From 06a50d055ac7d58ae71d4b161cf2abda99cdcc5f Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 18:16:24 -0700 Subject: [PATCH] feat(lens): guide setup through the first investigation --- .../lens/LensNavigation.integration.test.tsx | 36 ++- .../lens/LensSetup.integration.test.tsx | 191 +++++++++++++++ .../lens/LensWorkspace.integration.test.tsx | 6 +- .../src/components/lens/LensWorkspace.tsx | 129 +++++++--- .../InvestigationsView.integration.test.tsx | 8 +- .../investigations/InvestigationsWelcome.tsx | 41 ++-- .../lens/setup/LensGettingStarted.tsx | 226 ++++++++++++++++++ .../lens/setup/LensIntroduction.tsx | 71 ++++++ .../components/lens/setup/LensSetupSteps.tsx | 201 ++++++++++++++++ .../src/components/lens/setup/useLensSetup.ts | 56 +++++ .../view_logs/TraceView/TracingSetupCard.tsx | 62 ++--- 11 files changed, 933 insertions(+), 94 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts diff --git a/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx index bdfe49c1d00..6e19be5b33c 100644 --- a/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx @@ -1,7 +1,7 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import LensPage from "@/app/(dashboard)/lens/page"; const { auth } = vi.hoisted(() => ({ auth: vi.fn() })); @@ -16,32 +16,44 @@ vi.mock("./investigations/InvestigationsView", () => ({ })); describe("Lens navigation", () => { - beforeEach(() => auth.mockReturnValue({ accessToken: "test-token", userRole: "Admin", isViewOnly: false })); + beforeEach(() => { + testQueryClient.clear(); + auth.mockReturnValue({ accessToken: "test-token", userRole: "Admin", isViewOnly: false }); + vi.stubGlobal( + "fetch", + vi.fn(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") return Response.json({ data: [{}] }); + if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: true }); + return Response.json({ traces: true, requests: false, data: [] }); + }), + ); + }); it("opens traces by default and pauses polling while viewing investigations", async () => { const user = userEvent.setup(); const onUrlUpdate = vi.fn(); renderWithProviders(, { onUrlUpdate }); expect(screen.getByRole("tab", { name: "Traces" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); await user.click(screen.getByRole("tab", { name: "Investigations" })); - expect(screen.getByText("Manage investigations")).toBeVisible(); + expect(await screen.findByText("Manage investigations")).toBeVisible(); expect(screen.getByText("Trace polling paused")).not.toBeVisible(); expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("tab")).toBe("investigations"); await user.click(screen.getByRole("tab", { name: "Traces" })); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); }); - it("opens existing lens links on findings", () => { + it("opens existing lens links on findings", async () => { renderWithProviders(, { searchParams: "?lens=saved-lens" }); expect(screen.getByRole("tab", { name: "Findings" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByText("Manage investigations")).toBeVisible(); + expect(await screen.findByText("Manage investigations")).toBeVisible(); }); - it("honors an explicit traces tab even when a saved investigation is in the URL", () => { + it("honors an explicit traces tab even when a saved investigation is in the URL", async () => { renderWithProviders(, { searchParams: "?tab=traces&lens=saved-lens" }); expect(screen.getByRole("tab", { name: "Traces" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); }); it.each(["Internal User", "Internal Viewer", "Org Admin"])( @@ -50,7 +62,7 @@ describe("Lens navigation", () => { auth.mockReturnValue({ accessToken: "test-token", userRole, isViewOnly: false }); const user = userEvent.setup(); renderWithProviders(); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); await user.click(screen.getByRole("tab", { name: "Investigations" })); expect(screen.getByText(/Investigations require proxy administrator access/)).toBeVisible(); expect(screen.queryByText("Manage investigations")).not.toBeInTheDocument(); @@ -60,10 +72,10 @@ describe("Lens navigation", () => { it.each([ { userRole: "Admin Viewer", isViewOnly: false }, { userRole: "Admin", isViewOnly: true }, - ])("preserves read-only investigation access for $userRole with isViewOnly=$isViewOnly", (session) => { + ])("preserves read-only investigation access for $userRole with isViewOnly=$isViewOnly", async (session) => { auth.mockReturnValue({ accessToken: "test-token", ...session }); renderWithProviders(, { searchParams: "?tab=investigations" }); - expect(screen.getByText("Read-only investigations")).toBeVisible(); + expect(await screen.findByText("Read-only investigations")).toBeVisible(); expect(screen.queryByText("Manage investigations")).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx new file mode 100644 index 00000000000..ee33eab3964 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx @@ -0,0 +1,191 @@ +import { fireEvent, screen, within, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { LensWorkspace } from "./LensWorkspace"; +import { createLensDemoData } from "./demo/createLensDemo"; +import type { LensList } from "./model/types"; + +const network = vi.fn(); +const list = vi.fn<() => Promise>(); +const data = createLensDemoData(); +const worker = () => ({ + id: "setup-worker", + analysis_key_id: "a".repeat(64), + revoked: false, + last_seen: new Date().toISOString(), + scope: data.lenses[0].scope, +}); + +function serve({ enabled = false, traces = false, requests = false, connected = false } = {}) { + list.mockResolvedValue({ lenses: [], workers: connected ? [worker()] : [], tracing_enabled: enabled }); + network.mockImplementation(async (input, init) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") + return enabled + ? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] }) + : Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); + if (path === "/lens/activity/available") return Response.json({ traces, requests }); + if (path === "/lens" && init?.method === "POST") { + const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...JSON.parse(String(init.body)) } }; + list.mockResolvedValue({ lenses: [saved], workers: [worker()], tracing_enabled: true }); + return Response.json(saved); + } + if (path === "/lens") return Response.json(await list()); + if (path === "/key/generate") return Response.json({ token_id: worker().analysis_key_id }); + if (path === "/lens/workers/register") { + list.mockResolvedValue({ lenses: [], workers: [worker()], tracing_enabled: true }); + return Response.json({ worker: worker(), token: "test-worker-token", image: "test-worker-image" }); + } + if (path === "/models") return Response.json({ data: [{ id: "analysis" }] }); + if (path === "/model_group/info") + return Response.json({ data: [{ model_group: "analysis", providers: ["OpenAI"], mode: "chat" }] }); + if (path === "/key/info") return Response.json({ info: { models: ["analysis"], max_budget: 100 } }); + if (path === "/lens/agents") return Response.json(["support_agent"]); + if (path === "/lens/preview/sample") return Response.json({ eligible: 1, selected: 1, executions: [] }); + if (path.endsWith("/runs")) return Response.json(data.lenses[0].jobs); + return Response.json({ data: [] }); + }); +} + +beforeEach(() => { + testQueryClient.clear(); + network.mockReset(); + list.mockReset(); + vi.stubGlobal("fetch", network); + Element.prototype.scrollIntoView = vi.fn(); + serve(); +}); + +describe("Lens setup journey", () => { + it("shares the introduction across tabs and stays in setup after the first trace", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { onUrlUpdate }); + expect(await screen.findByRole("heading", { name: "Before you start" })).toBeVisible(); + await user.click(screen.getByRole("tab", { name: "Investigations" })); + expect(screen.getByRole("heading", { name: "Before you start" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Set up Lens" })); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("setup")).toBe("lens")); + serve({ enabled: true }); + await user.click(screen.getByRole("button", { name: "Check setup" })); + expect(await screen.findByText("Trace storage is connected")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Continue to your agent" })); + expect(screen.getByRole("button", { name: "Check for traces" })).toBeVisible(); + serve({ enabled: true, traces: true }); + await user.click(screen.getByRole("button", { name: "Check for traces" })); + expect(await screen.findByText(/Your first trace is ready/)).toBeVisible(); + expect(screen.queryByRole("table", { name: "Agent runs" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Continue to worker" })); + const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); + await user.click(connection.getByRole("combobox", { name: "Analysis model" })); + await user.click(await screen.findByRole("option", { name: "analysis" })); + await user.click(connection.getByRole("button", { name: "Get install command" })); + const connected = within(await screen.findByRole("dialog", { name: "Worker connected" })); + await user.click(connected.getByRole("button", { name: "New investigation" })); + expect(await screen.findByRole("dialog", { name: "Which activity should we investigate?" })).toBeVisible(); + expect(screen.getByRole("heading", { name: "Get Lens running", hidden: true })).toBeInTheDocument(); + }); + + it("resumes setup after a reload and leaves only when the user chooses traces", async () => { + serve({ enabled: true, traces: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?tab=investigations&setup=lens", + }); + expect(await screen.findByRole("button", { name: "Connect worker" })).toBeVisible(); + await user.click(screen.getByRole("tab", { name: "Traces" })); + expect(screen.getByRole("heading", { name: "Get Lens running" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "View traces" })); + expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); + await user.click(screen.getByRole("tab", { name: "Investigations" })); + expect(await screen.findByRole("heading", { name: "Run your first investigation" })).toBeVisible(); + expect(screen.queryByRole("link", { name: "Set up traces" })).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Get Lens running" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Set up Lens" })); + expect(await screen.findByRole("heading", { name: "Get Lens running" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeVisible(); + }); + + it("allows request-only investigations without forcing agent instrumentation", async () => { + serve({ enabled: true, requests: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?tab=investigations", + }); + expect(await screen.findByText("Request logs received")).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeEnabled(); + expect(screen.queryByRole("heading", { name: "Before you start" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Set up Lens" })); + expect(await screen.findByRole("heading", { name: "Before you start" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "View investigations" })); + expect(await screen.findByRole("heading", { name: "Run your first investigation" })).toBeVisible(); + }); + + it("keeps setup recoverable when checking for a first trace fails", async () => { + serve({ enabled: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?setup=lens", + }); + await screen.findByRole("button", { name: "Check for traces" }); + const normal = network.getMockImplementation()!; + network.mockImplementation(async (input, init) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") return Response.json({ detail: "Trace storage unavailable" }, { status: 503 }); + return normal(input, init); + }); + await user.click(screen.getByRole("button", { name: "Check for traces" })); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not check setup"); + expect(screen.queryByRole("button", { name: "Continue to worker" })).not.toBeInTheDocument(); + serve({ enabled: true, traces: true }); + await user.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByRole("button", { name: "Continue to worker" })).toBeEnabled(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + + it("keeps administrator-only setup unavailable to trace viewers", async () => { + serve({ enabled: true, traces: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?setup=lens", + }); + expect(await screen.findByText(/A gateway administrator can connect a worker/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeDisabled(); + await user.click(screen.getByRole("tab", { name: "Investigations" })); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeDisabled(); + expect(network.mock.calls.some(([input]) => new URL(String(input), "http://localhost").pathname === "/lens")).toBe( + false, + ); + }); + + it("opens the saved investigation directly when guided setup finishes", async () => { + serve({ enabled: true, traces: true, connected: true }); + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { + searchParams: "?setup=lens", + onUrlUpdate, + }); + await user.click(await screen.findByRole("button", { name: "New investigation" })); + const dialog = within(screen.getByRole("dialog")); + fireEvent.change(dialog.getByRole("textbox", { name: "Investigation name" }), { + target: { value: "My first review" }, + }); + await user.click(dialog.getByRole("button", { name: "Continue" })); + await user.click(dialog.getByRole("button", { name: "Continue" })); + await waitFor(() => expect(dialog.getByRole("button", { name: "Run and monitor" })).toBeEnabled()); + await user.click(dialog.getByRole("button", { name: "Run and monitor" })); + expect(await screen.findByRole("heading", { name: "My first review" })).toBeVisible(); + expect(screen.queryByRole("heading", { name: "Get Lens running" })).not.toBeInTheDocument(); + expect( + within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Findings" }), + ).toHaveAttribute("aria-selected", "true"); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("setup")).toBeNull()); + const create = network.mock.calls.find( + ([input, init]) => new URL(String(input), "http://localhost").pathname === "/lens" && init?.method === "POST", + ); + expect(create).toBeDefined(); + expect(JSON.parse(String(create?.[1]?.body))).toEqual(expect.objectContaining({ name: "My first review" })); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index 87adbe2adb3..3b268546344 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -25,7 +25,7 @@ describe("Lens interactive demo", () => { renderWithProviders(, { onUrlUpdate, }); - expect(await screen.findByText("Enable tracing")).toBeVisible(); + expect(await screen.findByRole("heading", { name: "The gateway that helps your agents improve" })).toBeVisible(); await user.click(screen.getByRole("button", { name: "Preview sample" })); expect(await screen.findByText("Where is order #1042?")).toBeVisible(); expect(screen.getByText("You’re viewing demo data")).toBeVisible(); @@ -46,7 +46,7 @@ describe("Lens interactive demo", () => { expect(network).not.toHaveBeenCalled(); expect(onUrlUpdate).not.toHaveBeenCalled(); await user.click(screen.getByRole("button", { name: "Exit demo" })); - expect(await screen.findByText("Enable tracing")).toBeVisible(); + expect(await screen.findByRole("heading", { name: "The gateway that helps your agents improve" })).toBeVisible(); expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); expect(screen.queryByText("Can I return my headphones?")).not.toBeInTheDocument(); }); @@ -84,7 +84,7 @@ describe("Lens interactive demo", () => { expect(network).not.toHaveBeenCalled(); expect(onUrlUpdate).not.toHaveBeenCalled(); await user.click(screen.getByRole("button", { name: "Exit demo" })); - expect(await screen.findByRole("heading", { name: "Find what needs attention" })).toBeVisible(); + expect(await screen.findByRole("heading", { name: "The gateway that helps your agents improve" })).toBeVisible(); }); it("has no demo entry for existing investigations, populated traces, or connecting another agent", async () => { diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx index 9912fc37c21..35f209c096c 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx @@ -1,7 +1,7 @@ "use client"; import { useEffect, useState } from "react"; -import { Aperture } from "lucide-react"; +import { Aperture, Loader2 } from "lucide-react"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { parseAsString, parseAsStringLiteral, useQueryState } from "nuqs"; import AgentTracesPage from "@/components/view_logs/TraceView/AgentTracesPage"; @@ -13,6 +13,9 @@ import { LensPreviewTarget } from "./LensPreviewButton"; import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles"; import { InvestigationsView } from "./investigations/InvestigationsView"; import { createLensDemo } from "./demo/createLensDemo"; +import { Button } from "@/components/ui/button"; +import { LensGettingStarted } from "./setup/LensGettingStarted"; +import { useLensSetup, type LensSetupState } from "./setup/useLensSetup"; type Tab = "traces" | "findings" | "investigations"; type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean }; @@ -61,12 +64,51 @@ function LensContent({ "tab", parseAsStringLiteral(["traces", "findings", "investigations"]).withOptions({ history: "push" }), ); - const [lensId] = useQueryState("lens", parseAsString); + const [lensId, setLensId] = useQueryState("lens", parseAsString); + const [setup, setSetup] = useQueryState("setup", parseAsStringLiteral(["lens"]).withOptions({ history: "push" })); const [demoTab, setDemoTab] = useState(initialTab); const [previewTarget, setPreviewTarget] = useState(null); const defaultTab = lensId ? "findings" : "traces"; const activeTab = demo ? demoTab : tab ?? defaultTab; const openDemo = onDemo ? () => onDemo(activeTab) : undefined; + const canInvestigate = isProxyAdminTierRole(userRole); + const setupState = useLensSetup(accessToken, !demo, canInvestigate, setup === "lens"); + const setupLocation = { tab: activeTab, canInvestigate, selected: !!lensId, requested: setup === "lens" }; + const showSetup = !demo && needsSetup(setupState, setupLocation); + const showSetupButton = !demo && !showSetup && isProxyAdminRole(userRole); + const startSetup = () => { + if (setup !== "lens") void setSetup("lens"); + }; + const showTraces = () => { + void setSetup(null); + void setTab(setupState.tracesReady ? "traces" : "investigations"); + }; + const checkingSetup = !demo && setupState.loading; + const showCreated = (id: string) => { + void setSetup(null); + void setLensId(id); + void setTab("findings"); + }; + const sharedSetup = checkingSetup ? ( +

+

+ ) : ( + + + + ); return (
@@ -94,36 +136,65 @@ function LensContent({ -
-
- - - - {(["findings", "investigations"] as const).map((view) => ( - - {demo || isProxyAdminTierRole(userRole) ? ( - - ) : ( -

- Investigations require proxy administrator access. You can still view your traces. -

+
+ {showSetupButton && ( + )} - - ))} +
+
+
+ {checkingSetup || showSetup ? ( + sharedSetup + ) : ( + <> + + + + {(["findings", "investigations"] as const).map((view) => ( + + {demo || isProxyAdminTierRole(userRole) ? ( + + ) : ( +

+ Investigations require proxy administrator access. You can still view your traces. +

+ )} + + ))} + + )}
); } + +function needsSetup( + state: LensSetupState, + { + tab, + canInvestigate, + selected, + requested, + }: { tab: Tab; canInvestigate: boolean; selected: boolean; requested: boolean }, +) { + if (requested) return true; + if (!state.missingTraces) return false; + if (tab === "traces") return true; + const hasActivity = state.hasInvestigations || state.requestsReady || selected; + return canInvestigate && !hasActivity; +} diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx index b7343717aa1..b7a713b034e 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx @@ -303,12 +303,10 @@ it("guides a first-time administrator into worker connection and lens setup", as }); const user = userEvent.setup(); renderWithProviders(); - const guide = within(await screen.findByRole("region", { name: "Find what needs attention" })); + const guide = within(await screen.findByRole("region", { name: "Run your first investigation" })); expect(apiClient.get).toHaveBeenCalledWith("/lens/activity/available", { accessToken: "test" }); - expect(await guide.findByRole("link", { name: "View traces" })).toHaveAttribute( - "href", - expect.stringMatching(/^\/ui\/lens\/?\?tab=traces$/), - ); + expect(guide.getByText("Traces received")).toBeVisible(); + expect(guide.queryByRole("link", { name: "View traces" })).not.toBeInTheDocument(); expect(await guide.findByRole("button", { name: "Preview sample" })).toBeVisible(); await user.click(guide.getByRole("button", { name: "Connect worker" })); const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx index a9886499fd3..669c658b202 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx @@ -34,36 +34,45 @@ export function InvestigationsWelcome({ const canConnect = activityReady && !readOnly; const traceStatus = activityStatus(tracesReady, requestsReady, checking); const waitingForWorker = activityReady && !connected; - const firstStepTitle = requestsReady && !tracesReady ? "Recorded activity" : "Set up traces"; + const stepOffset = Number(!activityReady); const traceButtonClass = buttonVariants({ - variant: activityReady ? "ghost" : "default", + variant: "default", className: "col-start-2 w-fit sm:col-start-auto", }); return (
{onDemo && }

- Find what needs attention + {activityReady ? "Run your first investigation" : "Find what needs attention"}

- Check how your agents behave. Get findings you can trace back to what happened. + {activityReady + ? "Your recorded activity is ready. Connect a worker and choose what Lens should look for." + : "Check how your agents behave. Get findings you can trace back to what happened."}

+ {activityReady && ( +

+ {traceStatus} +

+ )}
    -
  1. - -
    -

    {firstStepTitle}

    -

    {traceStatus}

    -
    - - {tracesReady ? "View traces" : "Set up traces"} - -
    2. + {!activityReady && ( +
  2. + +
    +

    Set up traces

    +

    {traceStatus}

    +
    + + Set up traces + +
    3. + )}
  3. - +

    Connect a worker

    @@ -83,7 +92,7 @@ export function InvestigationsWelcome({ data-state={workerReady ? "active" : "inactive"} className="grid grid-cols-[28px_minmax(0,1fr)] items-center gap-x-4 gap-y-3 py-5 text-muted-foreground data-[state=active]:text-foreground sm:grid-cols-[28px_minmax(0,1fr)_auto]" > - +

    Run an investigation

    diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx b/ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx new file mode 100644 index 00000000000..887aea1f20d --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx @@ -0,0 +1,226 @@ +"use client"; + +import { useRef, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { Check, ShieldCheck } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import type { TraceSummary } from "@/components/view_logs/TraceView/traceTypes"; +import { lensQueries } from "../api/queries"; +import { useSaveLens } from "../api/mutations"; +import { useLensApi } from "../services"; +import { TraceSheet } from "../investigations/TraceSheet"; +import { LensPreviewButton } from "../LensPreviewButton"; +import { useAnalysisKeyInfo } from "./worker/AnalysisKeyDetails"; +import { WorkerDialog } from "./worker/WorkerDialog"; +import { InvestigationSetupDialog } from "./InvestigationSetupDialog"; +import { LensIntroduction } from "./LensIntroduction"; +import type { LensSetupState } from "./useLensSetup"; +import type { Settings } from "../model/types"; +import { initialSetupStep, LensSetupSteps } from "./LensSetupSteps"; + +export function LensGettingStarted({ + accessToken, + state, + readOnly, + canInvestigate, + canMintTracingKey, + onStart, + onExit, + onCreated, + onDemo, +}: { + accessToken: string; + state: LensSetupState; + readOnly: boolean; + canInvestigate: boolean; + canMintTracingKey: boolean; + onStart: () => void; + onExit: () => void; + onCreated: (id: string) => void; + onDemo?: () => void; +}) { + const setupRef = useRef(null); + const [step, setStep] = useState(() => initialSetupStep(state)); + const [workerOpen, setWorkerOpen] = useState(false); + const [investigationOpen, setInvestigationOpen] = useState(false); + const [trace, setTrace] = useState(null); + const selectStep = (index: number) => { + onStart(); + setStep(index); + setupRef.current?.querySelector(`[aria-controls="lens-setup-step-${index}"]`)?.focus(); + }; + const canLeave = state.tracesReady || state.requestsReady || state.hasInvestigations; + const start = () => { + onStart(); + setupRef.current?.scrollIntoView({ block: "start" }); + setupRef.current?.focus({ preventScroll: true }); + }; + const connectWorker = () => { + selectStep(2); + setWorkerOpen(true); + }; + const createInvestigation = () => { + selectStep(3); + setInvestigationOpen(true); + }; + + return ( +
    + {onDemo && } + +
    +
    +
    +
    +

    + Get Lens running +

    +

    + We’ll check each connection. Stay here until your first investigation is ready. +

    +
    + {canLeave && ( + + )} +
    + + {state.error && ( +
    +

    Could not check setup. {state.error}

    + +
    + )} + {(readOnly || !canInvestigate) && ( +

    + A gateway administrator can connect a worker and run investigations. +

    + )} +
    + +
    + {workerOpen && ( + setWorkerOpen(false)} + onChanged={state.refresh} + onReady={ + state.ready + ? () => { + setWorkerOpen(false); + createInvestigation(); + } + : undefined + } + /> + )} + {investigationOpen && ( + setInvestigationOpen(false)} + onCreated={onCreated} + /> + )} + {trace && ( + setTrace(null)} + /> + )} +
    + ); +} + +function FirstInvestigation({ + accessToken, + state, + onClose, + onCreated, +}: { + accessToken: string; + state: LensSetupState; + onClose: () => void; + onCreated: (id: string) => void; +}) { + const api = useLensApi(accessToken); + const saveLens = useSaveLens(accessToken); + const models = useQuery({ ...lensQueries.models(api) }); + const modelDetails = useQuery({ ...lensQueries.modelDetails(api) }); + const workers = state.workers.filter((worker) => !worker.revoked); + const analysisAccess = useAnalysisKeyInfo( + accessToken, + workers.length === 1 ? workers[0].analysis_key_id ?? undefined : undefined, + ); + const defaultModel = analysisAccess.data?.models.length === 1 ? analysisAccess.data.models[0] : undefined; + const save = async (settings: Settings) => { + if (!state.ready) + throw new Error("Wait for recorded activity and a connected worker before starting an investigation"); + const saved = await saveLens.mutateAsync({ settings }); + state.refresh(); + onCreated(saved.id); + }; + + return ( + model.id) ?? []} + modelDetails={modelDetails.data?.data ?? []} + modelsLoading={models.isLoading} + modelsError={models.error?.message} + onClose={onClose} + onSave={save} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx b/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx new file mode 100644 index 00000000000..36fe67cdcaf --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx @@ -0,0 +1,71 @@ +import { Aperture, ArrowRight, ArrowUpRight, GitBranch } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { scenarios } from "../demo/scenarios"; + +export function LensIntroduction({ onStart, onDemo }: { onStart: () => void; onDemo?: () => void }) { + const example = scenarios[1]; + return ( +
    +

    + The gateway that helps your agents improve +

    +

    + Lens reviews your agents’ recorded activity and finds behavior worth fixing. Each finding links to the original + steps, so you can see what happened and what to change next. +

    +
    + {onDemo && ( + + )} + + + Docs +
    +
    +
    +
    +

    +

    + Sample · {example.agent} +
    +
    +
    +
    {example.tool}
    +
    {example.result}
    +
    +
    +
    Final answer
    +
    “{example.answer}”
    +
    +
    +
    +
    +

    +

    +

    Performance claim has no supporting benchmark

    +

    + The answer claims a 40% improvement, but the retrieved documentation contains no comparative benchmark. +

    +

    + Suggested change: Require a benchmark source for numerical performance + claims, or remove the comparison. +

    +
    +
    +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx b/ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx new file mode 100644 index 00000000000..dff5f747541 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx @@ -0,0 +1,201 @@ +import { ArrowRight, Check, ChevronDown } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { TracingSetupFields } from "@/components/view_logs/TraceView/TracingSetupCard"; +import type { TraceSummary } from "@/components/view_logs/TraceView/traceTypes"; +import type { LensSetupState } from "./useLensSetup"; + +type StepProps = { + accessToken: string; + state: LensSetupState; + readOnly: boolean; + canInvestigate: boolean; + canMintTracingKey: boolean; + onStep: (step: number) => void; + onTrace: (trace: TraceSummary) => void; + onConnect: () => void; + onCreate: () => void; +}; + +export function initialSetupStep(state: LensSetupState) { + if (!state.tracingEnabled) return 0; + if (!state.tracesReady) return 1; + return state.connected ? 3 : 2; +} + +function StorageStep({ state, onStep, ...props }: StepProps) { + if (!state.tracingEnabled) + return ( + + ); + return ( +
    +

    + Trace storage is connected +

    + +
    + ); +} + +function continuationLabel(state: LensSetupState) { + if (state.connected) return "Continue to investigation"; + return state.tracesReady ? "Continue to worker" : "Continue with request logs"; +} + +function AgentStep({ state, onConnect, onCreate, ...props }: StepProps) { + if (!state.tracingEnabled) + return

    Connect trace storage in step 1 before sending a trace.

    ; + const activityReady = state.tracesReady || state.requestsReady; + return ( + <> + + {state.tracesReady && ( +

    + Your first trace is ready. Continue setup so Lens can investigate your agent’s behavior. +

    + )} + {activityReady && ( +
    + + {state.requestsReady && !state.tracesReady && ( +

    + Request logs are already available. You can investigate them now and add agent traces later. +

    + )} +
    + )} + + ); +} + +function WorkerStep({ state, onConnect, onCreate, readOnly, canInvestigate }: StepProps) { + const activityReady = state.tracesReady || state.requestsReady; + return ( +
    +

    + {state.connected + ? "Worker connected. You’re ready to create an investigation." + : "The worker reviews recorded activity using a model on your gateway. You choose its analysis model and spending limit."} +

    + {!activityReady && ( +

    Record activity in step 2 before connecting a worker.

    + )} + +
    + ); +} + +function InvestigationStep({ state, onCreate, readOnly, canInvestigate }: StepProps) { + return ( +
    +

    + Choose the activity to review and describe how your agent should behave. Lens will show findings with evidence + and suggested changes. +

    + {!state.ready && ( +

    + Recorded activity and a connected worker are required before you can run an investigation. +

    + )} + +
    + ); +} + +export function LensSetupSteps({ step, ...props }: StepProps & { step: number }) { + const items = [ + { + title: "Enable tracing on the gateway", + description: "Connect ClickHouse and restart the gateway.", + complete: props.state.tracingEnabled, + content: , + }, + { + title: "Send your first trace", + description: "Capture your agent’s inputs, outputs, and tool calls.", + complete: props.state.tracesReady, + content: , + }, + { + title: "Connect a worker", + description: "Choose a model and run the worker on your infrastructure.", + complete: props.state.connected, + content: , + }, + { + title: "Run your first investigation", + description: "Describe the expected behavior and review a sample of activity.", + complete: props.state.hasInvestigations, + content: , + }, + ]; + return ( +
      + {items.map((item, index) => ( +
    1. +

      + +

      + +
      2. + ))} +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts b/ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts new file mode 100644 index 00000000000..3d770b8cadc --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts @@ -0,0 +1,56 @@ +import { useQuery } from "@tanstack/react-query"; +import { useNow } from "@/hooks/useNow"; +import { isTracingNotEnabled, useTraceAvailability } from "@/components/view_logs/TraceView/useAgentTraces"; +import { lensQueries } from "../api/queries"; +import { workerConnected } from "../model/status"; +import { useLensApi } from "../services"; + +function traceSetupState(traces: ReturnType, configured: boolean) { + const disabled = isTracingNotEnabled(traces.error); + return { + tracesReady: traces.data === true && !traces.error, + tracingEnabled: !disabled && (traces.isSuccess || configured), + missingTraces: disabled || traces.data === false, + error: disabled ? null : traces.error, + }; +} + +export function useLensSetup(accessToken: string, enabled: boolean, canInvestigate: boolean, settingUp: boolean) { + const api = useLensApi(accessToken); + const now = useNow(2000); + const traces = useTraceAvailability(accessToken, enabled); + const list = useQuery({ ...lensQueries.list(api, false, settingUp), enabled: enabled && canInvestigate }); + const activity = useQuery(lensQueries.activity(api, enabled && canInvestigate && list.isSuccess, false)); + const data = list.data ?? { lenses: [], workers: [], tracing_enabled: false }; + const traceState = traceSetupState(traces, data.tracing_enabled); + const requestsReady = activity.data?.requests === true && !activity.error; + const connected = data.workers.some((worker) => workerConnected(worker, now)); + const hasInvestigations = data.lenses.length > 0; + const error = traceState.error || list.error || activity.error; + const activityReady = traceState.tracesReady || requestsReady; + const loadingActivity = list.isSuccess && activity.isPending; + const loadingInvestigations = canInvestigate && (list.isPending || loadingActivity); + const refresh = () => { + void traces.refetch(); + if (canInvestigate) { + void list.refetch(); + void activity.refetch(); + } + }; + return { + tracingEnabled: traceState.tracingEnabled, + tracesReady: traceState.tracesReady, + requestsReady, + connected, + hasInvestigations, + missingTraces: traceState.missingTraces, + loading: traces.isPending || loadingInvestigations, + checking: traces.isFetching || list.isFetching || activity.isFetching, + ready: activityReady && connected && !error, + workers: data.workers, + error: error?.message, + refresh, + }; +} + +export type LensSetupState = ReturnType; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx index 7c6155446ea..e4ef3591af6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -645,17 +645,7 @@ function ConnectAgent({ ); } -export function TracingSetupCard({ - detail, - accessToken, - onOpenTrace, - connected = false, - onCheck, - checking = false, - readOnly = false, - canMintTracingKey = false, - onDemo, -}: { +type TracingSetupProps = { detail: string | null; accessToken: string; onOpenTrace: (trace: TraceSummary) => void; @@ -664,20 +654,47 @@ export function TracingSetupCard({ checking?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; - onDemo?: () => void; -}) { +}; + +export function TracingSetupFields({ + detail, + accessToken, + onOpenTrace, + connected = false, + onCheck, + checking = false, + readOnly = false, + canMintTracingKey = false, +}: TracingSetupProps) { const [checked, setChecked] = useState(false); - const enabled = detail === null; const check = () => { setChecked(true); onCheck?.(); }; + return detail === null ? ( + + ) : ( + + ); +} + +export function TracingSetupCard({ onDemo, ...props }: TracingSetupProps & { onDemo?: () => void }) { + const enabled = props.detail === null; return (
    {onDemo && }
    -

    {setupTitle(enabled, connected)}

    +

    {setupTitle(enabled, props.connected ?? false)}

    {enabled ? ( @@ -700,20 +717,7 @@ export function TracingSetupCard({ ? "Send your agent’s runs to LiteLLM to see its inputs, outputs, and tool calls." : "Tracing needs ClickHouse and a small update to your LiteLLM proxy configuration."}

    - {enabled ? ( - - ) : ( - - )} +
    ); }