From 6e75f2828932129482400433faec83916d5b6d02 Mon Sep 17 00:00:00 2001 From: "moyai-devin-berriai[bot]" <336287033+moyai-devin-berriai[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:48:10 +0000 Subject: [PATCH] feat(ui): support native decisions endpoint in decision playground (#44664) * feat(ui): support native decisions endpoint in decision playground * fix(ui): preserve decision playground defaults and test late cancellation results * test(ui): remove redundant cancellation test comment * Update ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --------- Co-authored-by: moyai-devin-berriai[bot] <336287033+moyai-devin-berriai[bot]@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../systemOneUI/QuestionBreakdown.tsx | 12 +- .../systemOneUI/ResponseView.test.tsx | 20 +++ .../components/systemOneUI/ResponseView.tsx | 6 +- .../SystemOneUI.integration.test.tsx | 131 +++++++++++++++++- .../components/systemOneUI/SystemOneUI.tsx | 64 +++++++-- .../systemOneUI/lib/decisions.test.ts | 83 +++++++++++ .../components/systemOneUI/lib/example.ts | 9 +- .../components/systemOneUI/lib/schemas.ts | 49 ++++++- .../systemOneUI/lib/validatePayload.ts | 19 ++- .../playground/llm_calls/system_one.ts | 9 +- 10 files changed, 361 insertions(+), 41 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/QuestionBreakdown.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/QuestionBreakdown.tsx index 9b5c9eee6ea..66f7f411886 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/QuestionBreakdown.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/QuestionBreakdown.tsx @@ -2,7 +2,7 @@ import { Badge } from "@/components/ui/badge"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { cn } from "@/lib/cva.config"; import { ROOT_BLOCK_STYLES } from "./lib/rootBlocks"; -import type { SystemOneQuestion, SystemOneRequest } from "./lib/schemas"; +import type { PlaygroundQuestion, PlaygroundRequest } from "./lib/schemas"; function formatState(state: unknown): string { if (typeof state === "string") { @@ -11,14 +11,14 @@ function formatState(state: unknown): string { return JSON.stringify(state, null, 2) ?? String(state); } -function QuestionCriteria({ question }: { question: SystemOneQuestion }) { +function QuestionCriteria({ question }: { question: PlaygroundQuestion }) { if (question.type === "choice") { return (
{Object.entries(question.criteria).map(([label, description]) => (
{label}
-
{description}
+
{formatState(description)}
))}
@@ -48,14 +48,14 @@ function QuestionCriteria({ question }: { question: SystemOneQuestion }) { {question.criteria.map((description, index) => (
  • {index} - {description} + {formatState(description)}
    • ))} ); } -export default function QuestionBreakdown({ payload }: { payload?: SystemOneRequest }) { +export default function QuestionBreakdown({ payload }: { payload?: PlaygroundRequest }) { if (!payload) { return ( @@ -94,7 +94,7 @@ export default function QuestionBreakdown({ payload }: { payload?: SystemOneRequ {question.type} -

    {question.instructions}

    + {question.instructions != null &&

    {formatState(question.instructions)}

    }

    Criteria

    diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.test.tsx index 9e7358b177b..6cde78fbaf3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.test.tsx @@ -59,6 +59,26 @@ describe("ResponseView", () => { expect(screen.getByRole("meter", { name: "1 probability" })).toHaveAttribute("aria-valuetext", "0%"); }); + it("renders structured decision score legends without coercing objects to strings", () => { + const response: SystemOneResponse = { + answers: { + severity: { + type: "score", + score: 0, + probabilities: { "0": 1 }, + legend: { "0": { description: "Low severity" } }, + }, + }, + usage: null, + }; + render(); + expect(screen.getByRole("meter", { name: '0: {"description":"Low severity"} probability' })).toHaveAttribute( + "aria-valuenow", + "100", + ); + expect(screen.queryByText(/\[object Object\]/)).not.toBeInTheDocument(); + }); + it("shows an inline error message", () => { render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx index 04ba8340fa4..fe58d565772 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx @@ -92,7 +92,11 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
    {levels.map(([level, probability]) => { - const label = answer.legend?.[level] ? `${level}: ${answer.legend[level]}` : level; + const description = answer.legend?.[level]; + const label = + description === undefined + ? level + : `${level}: ${typeof description === "string" ? description : JSON.stringify(description)}`; return ( { expect(resetButton).toBeDisabled(); }); - it("flags the tab as a TypeSafe-only beta without announcing it as an alert", () => { + it("explains the native decision endpoint after opting in without announcing it as an alert", async () => { + const user = userEvent.setup(); render(); + screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); + await user.keyboard("{ArrowDown}"); + await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); - expect(screen.getByRole("note", { name: "System One beta notice" })).toHaveTextContent( - "Support for more System One-compatible models is in progress.", + expect(screen.getByRole("note", { name: "Decision endpoint notice" })).toHaveTextContent( + "omit model to use the proxy's configured default.", ); expect(screen.getByRole("link", { name: "Give us feedback on what you want for decision models" })).toHaveAttribute( "href", @@ -87,7 +98,7 @@ describe("SystemOneUI integration", () => { expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); }); - it("posts the request with the session key and renders calibrated answers", async () => { + it("preserves the untouched TypeSafe default and renders calibrated answers", async () => { const user = userEvent.setup(); sessionStorage.setItem("customProxyBaseUrl", "https://stale.example.com/"); render(); @@ -123,7 +134,7 @@ describe("SystemOneUI integration", () => { expect(screen.getByText("Selected choice")).toBeInTheDocument(); }); - it("keeps the preview usable when question values have invalid types", () => { + it("keeps legacy validation when question values have invalid types", async () => { render(); fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), { @@ -146,6 +157,114 @@ describe("SystemOneUI integration", () => { expect(screen.getByText("Enter a valid request to preview its state and questions.")).toBeInTheDocument(); }); + it("preserves each endpoint draft and sends legacy requests to TypeSafe", async () => { + const user = userEvent.setup(); + render(); + screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); + await user.keyboard("{ArrowDown}"); + await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + const editor = screen.getByRole("textbox", { name: "System One JSON payload" }); + const draft = JSON.stringify({ + model: "my-decider", + state: {}, + questions: { + route: { type: "choice", criteria: { support: { text: "Help" }, other: null } }, + }, + }); + fireEvent.change(editor, { target: { value: draft } }); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + expect(screen.getByText(/"text": "Help"/)).toBeInTheDocument(); + + screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); + await user.keyboard("{ArrowDown}"); + await user.click(await screen.findByRole("option", { name: "TypeSafe · /typesafe/v1/systemone" })); + await user.click(screen.getByRole("button", { name: "Send" })); + expect(await screen.findByText("Selected choice")).toBeInTheDocument(); + expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/typesafe\/v1\/systemone$/); + expect(JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string).model).toBe("jev-latest"); + + screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); + await user.keyboard("{ArrowDown}"); + await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + expect(editor).toHaveValue(draft); + expect(screen.queryByText("Selected choice")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Send" })); + expect(await screen.findByText("Selected choice")).toBeInTheDocument(); + expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/decisions$/); + expect(JSON.parse(mockFetch.mock.calls[1]?.[1]?.body as string)).toEqual(JSON.parse(draft)); + }); + + it("sends a native request without model so the proxy can select its default", async () => { + const user = userEvent.setup(); + render(); + screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); + await user.keyboard("{ArrowDown}"); + await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + const payload = { state: "An outage", questions: { urgent: { type: "noul", instructions: "Is this urgent?" } } }; + fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), { + target: { value: JSON.stringify(payload) }, + }); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + await user.click(screen.getByRole("button", { name: "Send" })); + expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument(); + expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/decisions$/); + const body = JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string); + expect(body).toEqual(payload); + expect(body).not.toHaveProperty("model"); + }); + + it.each(["success", "error"])("ignores a late %s after switching endpoints", async (outcome) => { + const user = userEvent.setup(); + const pending = Promise.withResolvers(); + mockFetch.mockReturnValueOnce(pending.promise); + const queryClient = new QueryClient({ defaultOptions: { mutations: { retry: false } } }); + rtlRender( + + + , + ); + await user.click(screen.getByRole("button", { name: "Send" })); + await screen.findByRole("button", { name: "Cancel request" }); + screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); + await user.keyboard("{ArrowDown}"); + await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + + await act(async () => { + if (outcome === "success") { + pending.resolve(createResponse({ model: "stale-model", answers: { stale: { type: "noul", noul: 1 } } })); + } else { + pending.reject(new Error("Stale request failed")); + } + await pending.promise.catch(() => undefined); + }); + await waitFor(() => expect(queryClient.isMutating()).toBe(0)); + expect(screen.queryByText("stale-model")).not.toBeInTheDocument(); + expect(screen.queryByText("Stale request failed")).not.toBeInTheDocument(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.queryByText("100% yes")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + + await user.click(screen.getByRole("button", { name: "Send" })); + expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument(); + expect(screen.getByText("Selected choice")).toBeInTheDocument(); + expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/decisions$/); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it("uses a custom virtual key when personal key creation is disabled", async () => { + const user = userEvent.setup(); + render(); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); + fireEvent.change(screen.getByLabelText("Virtual Key", { exact: true }), { + target: { value: "test-virtual-key" }, + }); + await user.click(screen.getByRole("button", { name: "Send" })); + expect(await screen.findByText("Selected choice")).toBeInTheDocument(); + expect(Object.values(mockFetch.mock.calls[0]?.[1]?.headers ?? {})).toContain("Bearer test-virtual-key"); + }); + it("renders upstream errors inline", async () => { const user = userEvent.setup(); mockFetch.mockResolvedValueOnce(createResponse(responseBody, 401, "Virtual key rejected")); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx index 60bd7276d0e..7b312eab081 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx @@ -8,8 +8,8 @@ import { useMutation } from "@tanstack/react-query"; import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { makeSystemOneRequest } from "../../llm_calls/system_one"; -import { SYSTEM_ONE_EXAMPLE } from "./lib/example"; -import type { SystemOneRequest } from "./lib/schemas"; +import { DECISIONS_EXAMPLE, SYSTEM_ONE_EXAMPLE } from "./lib/example"; +import type { DecisionEndpoint, PlaygroundRequest } from "./lib/schemas"; import JsonEditor from "./JsonEditor"; import QuestionBreakdown from "./QuestionBreakdown"; import ResponseView from "./ResponseView"; @@ -23,12 +23,14 @@ interface SystemOneUIProps { type ApiKeySource = "session" | "custom"; interface SystemOneSendVariables { - payload: SystemOneRequest; + payload: PlaygroundRequest; + endpoint: DecisionEndpoint; apiKey: string; signal: AbortSignal; } const EXAMPLE_PAYLOAD = JSON.stringify(SYSTEM_ONE_EXAMPLE, null, 2); +const DECISIONS_EXAMPLE_PAYLOAD = JSON.stringify(DECISIONS_EXAMPLE, null, 2); const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231"; function getCustomProxyBaseUrl(): string | undefined { @@ -38,15 +40,21 @@ function getCustomProxyBaseUrl(): string | undefined { export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = false }: SystemOneUIProps) { const [apiKeySource, setApiKeySource] = useState(disabledPersonalKeyCreation ? "custom" : "session"); const [customApiKey, setCustomApiKey] = useState(""); - const [rawPayload, setRawPayload] = useState(EXAMPLE_PAYLOAD); + const [endpoint, setEndpoint] = useState("/typesafe/v1/systemone"); + const [payloads, setPayloads] = useState>({ + "/v1/decisions": DECISIONS_EXAMPLE_PAYLOAD, + "/typesafe/v1/systemone": EXAMPLE_PAYLOAD, + }); + const rawPayload = payloads[endpoint]; + const examplePayload = endpoint === "/v1/decisions" ? DECISIONS_EXAMPLE_PAYLOAD : EXAMPLE_PAYLOAD; const activeController = useRef(null); - const validation = useMemo(() => validateSystemOnePayload(rawPayload), [rawPayload]); + const validation = useMemo(() => validateSystemOnePayload(rawPayload, endpoint), [rawPayload, endpoint]); const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim(); const hasSyntaxError = validation.issues.some((issue) => issue.path === "syntax"); const systemOne = useMutation({ - mutationFn: ({ payload, apiKey, signal }: SystemOneSendVariables) => - makeSystemOneRequest(payload, apiKey, getCustomProxyBaseUrl(), signal), + mutationFn: ({ payload, apiKey, signal, endpoint }: SystemOneSendVariables) => + makeSystemOneRequest(payload, apiKey, getCustomProxyBaseUrl(), { signal, endpoint }), }); const isLoading = systemOne.isPending; const { reset: resetSystemOne } = systemOne; @@ -68,7 +76,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = function handlePayloadChange(value: string) { if (value !== rawPayload) { clearRequestState(); - setRawPayload(value); + setPayloads((current) => ({ ...current, [endpoint]: value })); } } @@ -85,12 +93,35 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = clearRequestState(); const controller = new AbortController(); activeController.current = controller; - systemOne.mutate({ payload: validation.payload, apiKey: effectiveApiKey, signal: controller.signal }); + const variables = { payload: validation.payload, apiKey: effectiveApiKey, signal: controller.signal, endpoint }; + systemOne.mutate(variables); } return (
    +
    + Endpoint + +
    Virtual Key Source @@ -127,8 +158,8 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation =
    - + - Beta: TypeSafe Jev only for now + + {endpoint === "/v1/decisions" ? "Decision models · Jev format" : "TypeSafe Jev · System One"} + - Sends System One requests (choice, noul, score) through /typesafe/v1/systemone and requires TYPESAFE_API_KEY - on the proxy. Support for more System One-compatible models is in progress.{" "} + {endpoint === "/v1/decisions" + ? "Sends choice, noul, and score questions through /v1/decisions. Replace the example model with a decision model configured on your proxy, or omit model to use the proxy's configured default." + : "Sends requests through /typesafe/v1/systemone and requires TYPESAFE_API_KEY on the proxy."}{" "} Give us feedback on what you want for decision models diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts new file mode 100644 index 00000000000..1ae9b2a130f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { validateSystemOnePayload } from "./validatePayload"; +import { systemOneResponseSchema } from "./schemas"; + +const request = { + model: "configured-decision-model", + state: { message: "Please help" }, + questions: { + category: { type: "choice", criteria: { support: { description: "Help" }, other: null } }, + urgent: { type: "noul", criteria: { true: ["An outage"], false: null } }, + severity: { type: "score", instructions: { task: "Rate severity" }, criteria: [["Low"]] }, + }, + provider_option: { enabled: true }, +}; +const validate = (value: unknown) => validateSystemOnePayload(JSON.stringify(value), "/v1/decisions"); + +describe("native decisions validation", () => { + it("accepts structured Jev criteria, optional instructions, and provider extensions without dropping fields", () => { + expect(validate(request)).toMatchObject({ isValid: true, payload: request, issues: [] }); + }); + + it("accepts an omitted model without inserting one", () => { + const payload = { state: request.state, questions: request.questions }; + expect(validate(payload)).toMatchObject({ isValid: true, payload, issues: [] }); + expect(validate(payload).payload).not.toHaveProperty("model"); + }); + + it.each([null, "", " ", 1])("rejects an invalid explicit model (%s)", (model) => { + expect(validate({ ...request, model }).isValid).toBe(false); + }); + + it.each([null, 1, true])("rejects a scalar state (%s)", (state) => { + expect(validate({ ...request, state }).isValid).toBe(false); + }); + + it.each([ + {}, + { "": request.questions.category }, + Object.fromEntries(Array.from({ length: 129 }, (_, i) => [`q${i}`, request.questions.category])), + { q: { type: "noul" } }, + { q: { type: "noul", criteria: { unexpected: "value" } } }, + { q: { type: "choice", criteria: {} } }, + { q: { type: "choice", criteria: { invalid: 1 } } }, + { q: { type: "choice", criteria: Object.fromEntries(Array.from({ length: 256 }, (_, i) => [`c${i}`, null])) } }, + { q: { type: "score", criteria: [] } }, + { q: { type: "score", criteria: Array(11).fill("level") } }, + ])("rejects invalid questions %#", (questions) => { + expect(validate({ ...request, questions }).isValid).toBe(false); + }); + + it("allows backend boundary values for score levels and question counts", () => { + expect( + validate({ ...request, questions: { q: { type: "score", criteria: Array(10).fill("level") } } }).isValid, + ).toBe(true); + expect( + validate({ + ...request, + questions: Object.fromEntries(Array.from({ length: 128 }, (_, i) => [`q${i}`, request.questions.category])), + }).isValid, + ).toBe(true); + }); + + it("does not loosen legacy TypeSafe request validation", () => { + expect(validateSystemOnePayload(JSON.stringify(request)).isValid).toBe(false); + }); + + it("parses structured score legends and nullable usage from the native endpoint", () => { + const response = { + answers: { + severity: { + type: "score", + score: 0, + confidence: 1, + probabilities: { "0": 1 }, + legend: { "0": { description: "Low" } }, + }, + urgent: { type: "noul", noul: 0.2 }, + }, + usage: null, + }; + expect(systemOneResponseSchema.parse(response)).toEqual(response); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.ts index 26fdc503e20..2632b2e1a9b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.ts @@ -1,6 +1,6 @@ -import type { SystemOneRequest } from "./schemas"; +import type { DecisionRequest, SystemOneRequest } from "./schemas"; -export const SYSTEM_ONE_EXAMPLE: SystemOneRequest = { +export const SYSTEM_ONE_EXAMPLE = { model: "jev-latest", state: "Since upgrading to the latest release, streaming responses stop halfway through whenever a fallback model takes over. Non-streaming requests still work. I haven't narrowed down which change caused it, but it happens on most long prompts.", @@ -35,4 +35,9 @@ export const SYSTEM_ONE_EXAMPLE: SystemOneRequest = { ], }, }, +} satisfies SystemOneRequest; + +export const DECISIONS_EXAMPLE: DecisionRequest = { + ...SYSTEM_ONE_EXAMPLE, + model: "your-decision-model", }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts index 1c4a9f071f2..585a2a36921 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts @@ -1,5 +1,50 @@ import { z } from "zod"; +export type DecisionEndpoint = "/v1/decisions" | "/typesafe/v1/systemone"; + +const decisionsJson = z.union([z.string(), z.record(z.string(), z.unknown()), z.array(z.unknown())]); +const decisionInstructions = decisionsJson.nullish(); +const decisionQuestionSchema = z.discriminatedUnion("type", [ + z.looseObject({ + type: z.literal("choice"), + instructions: decisionInstructions, + criteria: z.record(z.string(), decisionsJson.nullable()).refine((value) => { + const count = Object.keys(value).length; + return count >= 1 && count <= 255; + }, "Choice criteria must contain between 1 and 255 options."), + }), + z + .looseObject({ + type: z.literal("noul"), + instructions: decisionInstructions, + criteria: z.partialRecord(z.enum(["true", "false"]), decisionsJson.nullable()).nullish(), + }) + .refine((value) => value.instructions != null || value.criteria != null, { + message: "A noul question requires instructions or criteria.", + }), + z.looseObject({ + type: z.literal("score"), + instructions: decisionInstructions, + criteria: z.array(decisionsJson).min(1).max(10), + }), +]); + +export const decisionsRequestSchema = z.looseObject({ + model: z + .string() + .refine((value) => value.trim().length > 0, "Model must be a non-empty string.") + .optional(), + state: decisionsJson, + questions: z.record(z.string().min(1), decisionQuestionSchema).refine((value) => { + const count = Object.keys(value).length; + return count >= 1 && count <= 128; + }, "Questions must contain between 1 and 128 entries."), +}); + +export type DecisionRequest = z.infer; +export type PlaygroundRequest = SystemOneRequest | DecisionRequest; +export type PlaygroundQuestion = SystemOneQuestion | z.infer; + const MAX_CHOICE_OPTIONS = 255; const nonEmptyString = (message: string) => @@ -89,7 +134,7 @@ const scoreAnswerShape = { type: z.literal("score"), score: z.number().finite(), confidence: probability.optional(), - legend: z.record(z.string(), z.string()).optional(), + legend: z.record(z.string(), decisionsJson).optional(), probabilities, }; const scoreAnswerSchema = z.looseObject(scoreAnswerShape); @@ -102,7 +147,7 @@ export const systemOneResponseSchema = z.looseObject({ z.string(), z.discriminatedUnion("type", [noulAnswerSchema, choiceAnswerSchema, scoreAnswerSchema]), ), - usage: z.object({ input_tokens: tokenCount, output_tokens: tokenCount }).optional(), + usage: z.looseObject({ input_tokens: tokenCount, output_tokens: tokenCount }).nullish(), }); export type SystemOneRequest = z.infer; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts index fc06db3c705..eac96e67045 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts @@ -1,4 +1,9 @@ -import { systemOneRequestSchema, type SystemOneRequest } from "./schemas"; +import { + decisionsRequestSchema, + systemOneRequestSchema, + type DecisionEndpoint, + type PlaygroundRequest, +} from "./schemas"; const RECOMMENDED_MAX_SCORE_LEVELS = 10; @@ -10,7 +15,7 @@ export interface SystemOnePayloadIssue { export interface SystemOnePayloadValidation { isValid: boolean; - payload?: SystemOneRequest; + payload?: PlaygroundRequest; issues: SystemOnePayloadIssue[]; } @@ -27,7 +32,7 @@ function parseJson(raw: string): { ok: true; value: unknown } | { ok: false; mes } } -const scoreLevelWarnings = (payload: SystemOneRequest): SystemOnePayloadIssue[] => +const scoreLevelWarnings = (payload: PlaygroundRequest): SystemOnePayloadIssue[] => Object.entries(payload.questions) .filter(([, question]) => question.type === "score" && question.criteria.length > RECOMMENDED_MAX_SCORE_LEVELS) .map(([id]) => ({ @@ -36,7 +41,10 @@ const scoreLevelWarnings = (payload: SystemOneRequest): SystemOnePayloadIssue[] severity: "warning", })); -export function validateSystemOnePayload(raw: string): SystemOnePayloadValidation { +export function validateSystemOnePayload( + raw: string, + endpoint: DecisionEndpoint = "/typesafe/v1/systemone", +): SystemOnePayloadValidation { if (!raw.trim()) { return invalid("root", "Payload cannot be empty."); } @@ -46,7 +54,8 @@ export function validateSystemOnePayload(raw: string): SystemOnePayloadValidatio return invalid("syntax", `Invalid JSON syntax: ${json.message}`); } - const result = systemOneRequestSchema.safeParse(json.value); + const schema = endpoint === "/v1/decisions" ? decisionsRequestSchema : systemOneRequestSchema; + const result = schema.safeParse(json.value); if (!result.success) { return { isValid: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts index 5eb375bca1e..b71e1268ac2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/system_one.ts @@ -3,7 +3,8 @@ import { withRequiredHeaders } from "@/components/llm_calls/request_headers"; import { createApiClient } from "@/lib/http/client"; import { systemOneResponseSchema, - type SystemOneRequest, + type DecisionEndpoint, + type PlaygroundRequest, type SystemOneResponse, } from "../components/systemOneUI/lib/schemas"; @@ -13,10 +14,10 @@ export interface SystemOneResult { } export async function makeSystemOneRequest( - payload: SystemOneRequest, + payload: PlaygroundRequest, accessToken: string, customBaseUrl?: string, - signal?: AbortSignal, + { signal, endpoint = "/typesafe/v1/systemone" }: { signal?: AbortSignal; endpoint?: DecisionEndpoint } = {}, ): Promise { const proxyBaseUrl = customBaseUrl || getProxyBaseUrl(); const normalizedBaseUrl = proxyBaseUrl.endsWith("/") ? proxyBaseUrl.slice(0, -1) : proxyBaseUrl; @@ -29,7 +30,7 @@ export async function makeSystemOneRequest( }, ); const client = createApiClient({ getBaseUrl: () => normalizedBaseUrl }); - const body = await client.post("/typesafe/v1/systemone", { body: payload, headers, signal }); + const body = await client.post(endpoint, { body: payload, headers, signal }); const parsed = systemOneResponseSchema.safeParse(body); if (!parsed.success) { throw new Error("System One response has an invalid shape.");