From 22bd49e2311ec0314da6ed4ed25e685baf1f1358 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:54:13 -0700 Subject: [PATCH 01/42] fix(ui): make the Lens traces refresh button always clickable (#44252) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Ishaan Jaffer <155045088+ishaan-berri@users.noreply.github.com> --- .../TraceView/AgentTracesSection.test.tsx | 37 +++++++++++++++++-- .../TraceView/AgentTracesSection.tsx | 7 +++- .../view_logs/TraceView/TimeRangeControls.tsx | 21 ++++++----- 3 files changed, 50 insertions(+), 15 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx index 29acaec7f74..e90544ca0a6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -7,7 +7,7 @@ import { ApiError } from "@/lib/http/client"; import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; import traceList from "./__fixtures__/trace_list.json"; import AgentTracesPage from "./AgentTracesPage"; -import { AgentTracesSection, filterRuns } from "./AgentTracesSection"; +import { AgentTracesSection, filterRuns, type TimeControls } from "./AgentTracesSection"; import type { TracePage, TraceSummary } from "./traceTypes"; vi.mock("../../networking", () => ({ @@ -45,7 +45,7 @@ const renderSection = () => ); // A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone. -const renderWindowed = () => +const renderWindowed = (timeControls?: TimeControls) => renderWithProviders( endTime="2026-10-01T00:00Z" isCustomDate isLiveTail={false} + timeControls={timeControls} />, ); @@ -425,6 +426,25 @@ describe("AgentTracesSection", () => { expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); expect(rowCount()).toBe(runs.length); }); + + it("clears timeline zoom when refreshed", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed({ rangeHours: 24, onRangeHoursChange: () => {}, onLiveChange: () => {} }); + await screen.findAllByTestId("agent-trace-row"); + const area = screen.getByTestId("timeline-area"); + const x = (bucket: number) => bucket * 10 + 5; + + fireEvent.pointerDown(area, { clientX: x(0), pointerId: 1 }); + fireEvent.pointerMove(area, { clientX: x(1), pointerId: 1 }); + fireEvent.pointerUp(area, { clientX: x(1), pointerId: 1 }); + expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); + expect(screen.queryAllByTestId("agent-trace-row")).toHaveLength(0); + + fireEvent.click(screen.getByRole("button", { name: "Refresh" })); + + expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + }); }); describe("AgentTracesPage", () => { @@ -453,7 +473,18 @@ describe("AgentTracesPage", () => { expect(live).toHaveAttribute("aria-pressed", "true"); fireEvent.click(live); expect(live).toHaveAttribute("aria-pressed", "false"); - expect(screen.getByRole("button", { name: "Reset zoom" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "Refresh" })).toBeEnabled(); + }); + + it("refreshes the trace list", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const callsBeforeRefresh = vi.mocked(agentTraceListCall).mock.calls.length; + fireEvent.click(screen.getByRole("button", { name: "Refresh" })); + + await waitFor(() => expect(vi.mocked(agentTraceListCall).mock.calls.length).toBeGreaterThan(callsBeforeRefresh)); }); it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 4ec8fe7e0ec..1b25ca19051 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -211,8 +211,11 @@ export function AgentTracesSection({ onRangeHoursChange={(hours) => changeRange(hours, timeControls.onRangeHoursChange)} live={isLiveTail} onLiveChange={timeControls.onLiveChange} - zoomed={zoom !== null} - onResetZoom={() => setZoom(null)} + onRefresh={() => { + setZoom(null); + checkTraces(); + }} + refreshing={traces.isFetching} /> )} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx index 0d3cf77a1e5..03d950a7725 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx @@ -35,30 +35,31 @@ interface TimeRangeControlsProps { onRangeHoursChange: (hours: number) => void; live: boolean; onLiveChange: (live: boolean) => void; - zoomed: boolean; - onResetZoom: () => void; + onRefresh: () => void; + refreshing: boolean; } -/** Joined control group: reset zoom, the actual time range (opens presets), and Live. */ +/** Joined control group: refresh, the actual time range (opens presets), and Live. */ export function TimeRangeControls({ range, rangeHours, onRangeHoursChange, live, onLiveChange, - zoomed, - onResetZoom, + onRefresh, + refreshing, }: TimeRangeControlsProps) { return (
From f498176a276b1f16b9c8ae6a265efe8c56ffd188 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 2 Oct 2026 17:27:13 -0700 Subject: [PATCH 02/42] feat(lens): add sample previews and improve setup and worker feedback (#44268) * feat(lens): add interactive traces and investigations demo * fix(lens): limit sample previews to setup screens * fix(lens): simplify tracing setup and align sample previews * fix(ui): unify Lens and ROI demo notices * fix(lens): distinguish preparation from zero selected runs * fix(lens): report incompatible workers and address setup review --- deploy/lens/README.md | 6 +- deploy/lens/compose.yaml | 2 +- litellm/proxy/lens/worker.py | 31 +- tests/unit/proxy/lens/test_worker.py | 40 ++ .../public/assets/logos/google-adk.png | Bin 0 -> 11976 bytes .../public/assets/logos/hermes.png | Bin 0 -> 1100 bytes .../public/assets/logos/openclaw.png | Bin 0 -> 1015 bytes .../public/assets/logos/strands.svg | 4 + .../hooks/models/useModelCostMap.ts | 3 +- .../lens/_components/LensOverview.tsx | 58 -- .../lens/_components/LensProgress.tsx | 2 +- .../_components/LensView.integration.test.tsx | 35 +- .../(dashboard)/lens/_components/LensView.tsx | 37 +- .../lens/_components/LensWelcome.tsx | 13 +- .../LensWorkspace.integration.test.tsx | 137 +++++ .../lens/_components/LensWorkspace.tsx | 117 ++++ .../lens/_components/WorkerSetup.tsx | 2 +- .../lens/_components/lensData.test.ts | 22 + .../(dashboard)/lens/_components/lensData.ts | 9 + .../lens/_components/lensDemoData.test.ts | 55 ++ .../lens/_components/lensDemoData.ts | 483 +++++++++++++++++ .../src/app/(dashboard)/lens/page.tsx | 54 +- .../ROICalculatorView.integration.test.tsx | 26 + .../_components/ROICalculatorView.tsx | 25 +- .../src/components/lens/LensDemoContext.tsx | 12 + .../src/components/lens/LensPreviewButton.tsx | 18 + .../src/components/shared/DemoNotice.tsx | 16 + .../src/components/shared/PageHeader.tsx | 22 +- .../view_logs/TraceView/AgentTracesPage.tsx | 9 +- .../AgentTracesSection.integration.test.tsx | 509 ++++++++++++++++++ .../TraceView/AgentTracesSection.test.tsx | 502 +---------------- .../TraceView/AgentTracesSection.tsx | 36 +- .../view_logs/TraceView/DetailContent.tsx | 17 +- .../view_logs/TraceView/DetailPane.tsx | 17 +- .../view_logs/TraceView/TimeRangeControls.tsx | 28 +- .../view_logs/TraceView/TraceDrawer.tsx | 17 +- .../view_logs/TraceView/TracePreview.tsx | 70 --- ... => TracingSetupCard.integration.test.tsx} | 111 ++-- .../view_logs/TraceView/TracingSetupCard.tsx | 498 +++++++---------- .../view_logs/TraceView/spanProvider.ts | 4 +- .../view_logs/TraceView/tracingSetupGuides.ts | 501 +++++++++++++++++ .../view_logs/TraceView/useAgentTraces.ts | 12 +- 42 files changed, 2427 insertions(+), 1133 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/google-adk.png create mode 100644 ui/litellm-dashboard/public/assets/logos/hermes.png create mode 100644 ui/litellm-dashboard/public/assets/logos/openclaw.png create mode 100644 ui/litellm-dashboard/public/assets/logos/strands.svg create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts create mode 100644 ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/DemoNotice.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx rename ui/litellm-dashboard/src/components/view_logs/TraceView/{TracingSetupCard.test.tsx => TracingSetupCard.integration.test.tsx} (63%) create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/tracingSetupGuides.ts diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 315d072b501..4b65c88afcd 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -19,11 +19,13 @@ The URL, database, and retention settings can also come from `CLICKHOUSE_URL`, ` Retention changes require a proxy restart. ClickHouse removes expired rows during background merges, not immediately at startup. Enable request/response logging to analyze LLM requests. Lens can only inspect content you actually retain -In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in +In **Lens > Investigations**, click **Connect worker**, choose an analysis model and monthly limit, then **Get install command**. Use **Advanced options** to select an existing virtual key or change the proxy URL if the server running Docker needs a different network address. Copy the command and run it on your server. The dashboard shows **Worker connected** when the container checks in The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis -The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds +The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. CI also publishes immutable `:sha-` tags for successful worker builds on `main`. Keep the worker image compatible with your gateway version + +After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected: diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml index 4d1224fd41e..fc9850fcb04 100644 --- a/deploy/lens/compose.yaml +++ b/deploy/lens/compose.yaml @@ -1,6 +1,6 @@ services: lens-worker: - image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:67eba741c1b97c749975c5c38e2370a603e1105babc908d613c1b79d7b995393} + image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b} environment: LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} diff --git a/litellm/proxy/lens/worker.py b/litellm/proxy/lens/worker.py index 62f8295e7d3..051b4a09392 100644 --- a/litellm/proxy/lens/worker.py +++ b/litellm/proxy/lens/worker.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import Final import httpx +from pydantic import BaseModel, ConfigDict, ValidationError from .analysis import analyze_sample from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult, Progress, Result, Sample @@ -15,6 +16,17 @@ from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult logger: Final = logging.getLogger("litellm.lens.worker") +class ClaimedJobIdentity(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + id: str + + +class ClaimIdentity(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + lens_id: str + job: ClaimedJobIdentity + + def failure_message(error: Exception) -> str: if isinstance(error, (OSError, sqlite3.Error)): return "Worker temporary storage failed. Increase its capacity or reduce analysis parallelism." @@ -74,9 +86,24 @@ class LensWorker: async def run_once(self) -> bool: response: Final = await self.client.post("/lens/worker/claim", params=MappingProxyType({"protocol_version": 2})) response.raise_for_status() - if response.json() is None: + payload: Final = response.json() + if payload is None: return False - claim: Final = Claim.model_validate(response.json()) + try: + claim: Final = Claim.model_validate(payload) + except ValidationError: + identity: Final = ClaimIdentity.model_validate(payload) + failure: Final = await self.client.post( + f"/lens/worker/{identity.lens_id}/{identity.job.id}/result", + json=Result( + coverage=Coverage(), + error="The worker could not read this investigation. Update the worker to match the gateway, then retry.", + ).model_dump(), + ) + if failure.status_code != 409: + failure.raise_for_status() + logger.warning("Worker could not read a claimed investigation; reported a version compatibility failure") + return True prefix: Final = f"/lens/worker/{claim.lens_id}/{claim.job.id}" async def model(body: ModelRequest) -> ModelResult: diff --git a/tests/unit/proxy/lens/test_worker.py b/tests/unit/proxy/lens/test_worker.py index dce3fc04d45..0e64b0f10c8 100644 --- a/tests/unit/proxy/lens/test_worker.py +++ b/tests/unit/proxy/lens/test_worker.py @@ -3,6 +3,7 @@ from typing import Final import httpx import pytest +from pydantic import ValidationError from litellm.proxy.lens.models import ( Claim, @@ -81,6 +82,45 @@ async def test_idle_worker_does_not_start_an_analysis() -> None: assert await LensWorker(client).run_once() is False +@pytest.mark.asyncio +@pytest.mark.parametrize("result_status", (200, 409)) +async def test_incompatible_claim_reports_failure_instead_of_leaving_the_investigation_running( + result_status: int, +) -> None: + claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=()) + payload: Final = claim.model_dump(mode="json") | { + "job": claim.job.model_dump(mode="json") | { + "settings": claim.job.settings.model_dump() | {"future_setting": "private content"}, + }, + } + saved: Final = SimpleQueue[Result]() + + def handle(request: httpx.Request) -> httpx.Response: + if request.url.path == "/lens/worker/claim": + return httpx.Response(200, json=payload) + assert request.url.path == "/lens/worker/lens/job/result" + saved.put(Result.model_validate_json(request.content)) + return httpx.Response(result_status, json=True) + + async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client: + assert await LensWorker(client).run_once() is True + assert saved.get_nowait().error == ( + "The worker could not read this investigation. Update the worker to match the gateway, then retry." + ) + assert saved.empty() + + +@pytest.mark.asyncio +async def test_claim_without_an_identity_does_not_report_failure_for_another_investigation() -> None: + def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/lens/worker/claim" + return httpx.Response(200, json={"job": {"settings": {"future_setting": True}}}) + + async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client: + with pytest.raises(ValidationError): + await LensWorker(client).run_once() + + @pytest.mark.asyncio @pytest.mark.parametrize("model_status", (200, 402, 503)) async def test_worker_reads_claimed_activity_and_reports_analysis_or_failure(model_status: int) -> None: diff --git a/ui/litellm-dashboard/public/assets/logos/google-adk.png b/ui/litellm-dashboard/public/assets/logos/google-adk.png new file mode 100644 index 0000000000000000000000000000000000000000..9f967caa3008bdb13b446e1545cb16eef4e29149 GIT binary patch literal 11976 zcmdUV`9IWO^#7e1%h<-)q7aE}p^_!*5ZRJRcFI;LYh_K$*g`7VQKYs#;_MksZKKj=80bt?!`wIgvGx(rF*nLwyZBW)Lun7G_=c;9_1wchS%Yh?3 z0J7H&bhOL@VJnlyPck|}=GSrRCfnE}2Cnb!@|AK}8UIM`y61Ay#cwR?fxgPR@kQI( zBChC;4EVrSsP7S`@`*Bi#Ak zrR2HA`8Q??u5L{|(hXU~53*Jn39dR5oWmw2qs`Q?J2TH-l})ro32|1 z2{pQd^C?FcdP+Pl!<`xTzGCmjyM;BUKdwu4Q&wGVX>om_miWp6 z9=tSE$viQqJFj)Wl(50bVYE!S`Q?hMm*m4`>tS6Fict5FBl>1TacEM$h8dHfHv zjY82p`Gxx@KVD8-iLlZWAss_C*Pjn^zq$B``$2tKMDs^-{DPapaylML4C9pH2WWwYl zqyqRHMw%71@kMBV7V>cw@Vu>j`a*9B&a-o4l&&bozpOH-5=LAALt|f})Lk#fKOZ5h zC?k>PdC3}SPq|&kC9(G0KtbVMwch-u%bGXS$gKd_SR8(g4PRU#&Qwr;p(PQcUO|Vw z_*MqTKh$M-L7E|Hc9bK zL6@-7r(r{p*H@d>D}7@L)07ZuiqPi`p%2oSTaxL&;(v&LrXNDkW7vpOngWUlhdR0= zJ=08B^B?e~2r>NQ{XO?MXI5&PV-+x=?FA>^wC5lcd&rW!cpm;Pb(Yb}zY5G=e5hCA z?ICOG7|^6Q{&<_1(h&wN+QM+Vx!%UiQgLuXj>llL#E_a}^Ndl`&aq4|)KWBfZ?Qb| zkBn?vQIQ$;0!0mXottJv?sjk|uh3kI!!Bc6IGkuwI3Z$G4DGeqI9XeNd4!?D&&{Pu?w`xr@h8M9OboS<#NK)bE=hG7z) z-z<>9^W4Q=@L{q}Ookn_3$?qKa*vIcYr>?q_q~m4D@6s3 znA6X%4$rSBs=QGtJyq6x;ID_4VaCU>UD>Hh+cWchn{7Hc@V?1L^iJ94;q~hwDU*UQsU?!FTy4sNWNF^H=wD+Z!V?Mc z_rtIcA(_5ATN7#37Cg#1_ixo^uc~ys)L6RN&-2{(8#i1|U%{Wpw@j!(`9useJ?3U{ z#b|bz5YgjAf7+*H)4jmuF2vmQ%zrg$p$}}(Y|ygWoBuWl%ox_jBx{`Ha1M5`gNEP; zp0($BaH=oBSbh!R-r#*(c;pF5`=U|@6aA+!xG?-|G!?lGv#qPV@1XJN7EC*AovcDe zzG$Icei+ewd)xiO;S_FKp@q!u(X%htw^b(C@UD-wuy6%0#ZjSabpJx^1Ib+31wi+^ z-#zZyTvt$ROy<_B@h&FHI>JPDR&rCG zQqUfuh9O(QkyZtpn;bRNVBJAJ*EPof-i0o&+-J*NjdL?_s4lN8s!#K~2tyaOq-x4| z#^Q~4c*lG1+EDSfiu?`;f0xXVOSAnNYi-_HoOH2bc%WPJ)1>RR%OPeL)wnh5D>~0W zWeZ`&+H(=VqYW4c)}Wy4*lF#RYUdY2H86l-@r=61bQf+8Ge5q zwO~l_xI4Jhs3m+x*cZU>D($4I|x%AqddUr^=ATj80<9yjVHO%N_G(mOd4E-;r6Yx>EFo49H_V@7U$@+#wb$# zk_kx}A}ROE6{)F!EWyyjk-U*kF{WoXhM98I-5It`((>Au{xjZHzWf`BUzp6i_4X@g7A36i zh6hXIYp^D1e8tdV`J;yAjd67uAk{V{%J+j_+ZWcD1J-_g=C7N8O;JX>Hm`q$6YgyY z?)u_iW9q?iNOlg3pB-FY|DbAAs7N&U66wqBFvr6uU+LE#!AXdQ)#>W*1)`!hSGe{N z#hz$pZa<6G!O(xkb5R1QA=ieQt;@FPms;jv-D%J0^5hOBwV(>ak*uZ*#pYOd8iknM z??MLz?kpTcMpknZ{ul(uhY`%Uftb*Q;FujA3Lnky4x7Ch!>WC)Ro03c-p^$f2 z3@^g4InP^hbTnW^!3#5qp^JQd3Ue)v(qjykJ8>LTX^2{>oLLT%a4|O?xGj7eEITrSWG7)(iv$nU5ZS(H`e{ z_jKe6E@+VaNfW#pc}^|aQ8Gl4WSBwp257|@<{9mD7{03JW0Ngc;LG%KEKZ${T*EssD0!Io*OepD|@ zOW+V!7BVk>ee>uDSNyW;!dQOi)i5q@@Qk&Wwq1S_5JRQ{&#&+>l@SMP{`V*@R;!2@8n$!S}(%VL&8J#ikMXo_GTML29KC5Q<}G% zn%(Vu`d*J7DNIiMZW|&?sX6bSAa4m3TcFvpwtWtMvNvxf1R2}R_Nk7>U;lv>W?TU8 zaz48Ie0+#MyKnEf$>(A)*QlP{$;x0`%Q>g!&5$fV_sg2$5plgST6*A~xipR*FR7u- zImW>Z^cq(IcH{NXe?O{^e^99OEs*$;%&OOQpo06c_3^MFwW{j=*WkwNv}c;+poZ5e z=aQI0?9>KA8`x7WA^XExI3O?@1ZD)|mle9}UKwr9eRZ~qXi<;a4W7EA=skveB#cz^ z4lv76nLVlpZ_t8~@mH$1L*6!QgnF`oeXS2An?W7dUyR&K=(|R@)3NC*DUi>3J5*QoeoUA<8pf-(dQYOy_Hj*MwwXC6J)S=_0Jse&9jlyZvMYgk zx0i7U1u4WN3AsnjUq5cn-OL*gfe+?K&)_7OJYxI6MJ80tFRy)~yF@>Ew9WKR?ELj( zGFusDW`gv1t}mMp%nw7YEL|dG)(aJO{yIP5XHK!i0-0;lA4HhM3f1EQDL3 zMVVon2Nn#To}P%i0Z*X&lOnSxbLrm;s{%Kz|^e<9&#|@M)9bQ3M>AqX(nU zAi{&yyd97dP?TGAf*F`;=`-CL9Q?iB7qgeMbUzTY^!URAKR6s!56 zB^#Dc4oZ_w$P>G<{HUCkfT{arm%WMc~Cw@ciO}|H-FSIgf3T2kJB`NxB z*F$dw=rLb^U~7?%J5@#7Sa0mgO$p32d(mg+4^xH=WaRHyJ;SA7-P;MrQSF`Q`g?<^$M{*9yT;$eDL6GSWVosxJwUW>`s>GT!^sdH269=8bYl@nnA`SBUyhJEqY0y&IjSCnxzQR%?f~Wd^$II;*3zCY?v)ZC(SP z_Je-knxfRe+==k8ly*wVQNDui=uU$VA2CVle!a)lE)Hs<)K+KCWji}ZJPo8$cdKcF zSK;EQ5B~cZK|v;u_3V2q)$a$m>b{HrAaizLH!nz|PrB6h0kOlnG&18)_)*xA%Z&La z4RLOL4+@8clpr%XZVUFM4)xV4#bv1*V;`@Dzs^OFpZwk{EB8`(HFwd(g`VAN7q3~j z!NSwAIn{L9pd-1D+DxSH7t_W+%Y)fj-$^~u7##rGr3W^`cM3YeJ8c+QKlPUSdUr|Y zE^US$T~!52Hp)i#sI)MP+9bxUfiY4t|zA7%I$41E^1KE^jrvAzp$fPoV6$SRsl1C92rBJMl9Z~-K1{Q~y@#S72Cdk0uu zD;pG6iHbr_&>-iJ4bOXy8L1rVgidZQ9I#hZ;i@P??JbUm(jzpZzx98So<5j>J-u2F z!12OtQU8_ceYzGe+V=z6Ur=$${<&~JJy;V~-B+smEy;9I?;li&tYXFR0#d~CvSwr{ zE_Foed8g@C1q(qZ`i5SuNtB%HxQ0I+K#J{pvt6O7`rpkaaQgzod{|}bhdsuO=S;tae@2(nfA8^ESl9zw1FDIEu<4|lv;-SHCU)5iJX_&D#+&?-<>ox?_ySYq5c*j+^?GwEzK~&ZqD1D&>Bjt zdR@ex$isdQ-kalxxHBgFH)1JHQuM{_^dnkjAv2053E{R*m53i9*p%OHP%r`~D0-~P z6^%FVx-B2ibhb2wK%kmL;-?b2$b3N1u&|+B<3zXOLVZwp@o0Pdj59$0GQU<|?L^J#L|e~^XxBmf+JhnM7bl@6?13m&ww98M1R zv(+Bb<5#%TjMAt zDt%N09?rcqgJHMUXAVhj3kNt0)NaN8K}7Go&y2|nC`^_3fgy)niqq*4df*(}fB@{v zHBHju2T{j+uM6++o>Bo}Dt|POn4}kO7U&3Hz2`N0&)>iIv?CG}dJInASmv4?r^Xjv zvDc(1y3=4eP+%)+1bh47q=CwMRf`t`kkzT0!o@GWXu)ZtMN$wIgQB-w1p3{@a+xdXa`R#$ifdml4_5gi@Zzqnhc4v0Y?xs0M_kPCr(C^STCh%>7c>SS2F`~Q@ba+jxqdO*LtOp$S^&eB4UtodGT>sm;zI3fW zdF`dHR6!R>I&8r3(Vbg91AS10nsr16+GOp7Om-sb13{;!;yk|o%@E03GQEL1_%S!U z2ofN%fh9YSKfxRRiRzmE?gcZm?4D_=LGQ)!(qr-%USl%*EZ-ol28_Bra$XM}t%nzd z&rhk0EQxn@NXt(&#|s`@o6HdY!z&K)9yRv10!BJc?^tizcE756n_{QWKe)R)y|ToN zj*ZZyQeKSgvbgkn8%4S7v!7TOy1NS=g z5uV8qTu!TWG2*Q+)mfG6_8xX5H4AkdSU`-SZo6*?QT2hA(jA&AanY42|>K=&TL+L0u!ASy+e|oC{cfx-8y=asqE75?J zngYeCTBjMot17jm05%#ydWamiPYr zCCgxTJgG>j9Tm_`zE7jKI*|@5@4-cph+Ct z4hSqsn5I1)`W$Y&C%-ewE{K0AB7iLE$>M&uvm79ea}pli9pQ&c>(f{y{yYJJcs96V zDz(Zi-E+5U%HG=-_-ywJi@7c^0NnExs_Xlz?d-g8k1pEz=zTm2{P?mR(DSV{9Kx@F zHLt7BaI5GFM4cTya2$Y<5Ej2Vhz)eg>gXDmQJVMazU`YB`pvtz)XjstntvJbl;&x_ zGX253rEx6qAuB%=UA^ir*R_te1>j(O6M+q=qaM6#|8nY}r*dkM{YL#~m_y)c6=3yM znBOO$kYo+Jb81!wPvOk2Qk>}{$8Yt+NCji}5oTG&(kbn#V(?nR*m}ij>bNu~@5FEK z3qtpz+sMinu`XN~+^-`DfnYC*Y#3kOT32AQY46WKB5ljHmG z1<0MK$fsdK6M8VbxDyM^ASub04y)kL;_&VQUO zfp-F76%jP(aJK3F!nHg^+xOe_oL$I;23h-b1feV9l8#aDI|qAM89~c*brGFjuSjlb zoDPn?y(+)ZCe94#T4cFIb*GNx-#8-_@9fKnO-*#6`lo6Z>V}N%9l%&Y_{%EG z``H`qz7D>%V9tOzyoa4JvVjhes&uf$)Sq*AQ-1OL@gK-|)(aly1YH7PY5*oN;4tUS zNPFp?_cY`*J;2*T_|s|hfYB4DueoZe@&^gdLN*2diqiKupWlz^z_V5RMq`mvcN1exQOiaW=LbT{Qu@!LvR%KL-JhBFgXdJ#xS|7(Bbn$iOqJWm zye&fBPFqOkUpbF*t%8etg?@i;G16!966SKp4?mta?jo<=Fz>p$O?RJ8W6ivdskHT1fP^@+w!!HO73tHPljRV`%h)d%USPb<@?1+o0!i~2iwr{U5-*( z$JNQX1P{A2=Oe6LQYSJDj2`8>x@2>8Yt%*0hxK#7eLu5Oud!0yXZ(&9n8d4%{&o#% zXv-68ljkF8?C!OkZAn$;J1A|L{;J^)dxVFRPRQ?MbC=}%&kMiXps@?7L5Z^Fk2ODg z?iid%TgIOncD9?LA+>4E&`c;xM zx92Oe&swvdw7M||UR)Q7)=ZIAv@H&o2XI9@O6N?(rpYnciEs;C!ZiVxG{V=FcpYNzxOp1C1-vseA<9G?*<2B8n3_)Iye(76=P}}9&tE!Ro%~jPp?A%W z$OiEQtj`X_4(ep{Bj1MAwOk$Z{@U$uk=KFkxIKSs*w_(fz`NH?RdP}hwWPP5RbH;M zPPz#@?pLKAXk5@P1hqH^Mcv6xl;#AN#2zNa(mdj{%M%x)9)lJ{rRbBe!U8dVR{`OC!Pd0<@x}YnvkY*i z{nBVOIafQx7|G~l%nc->1P_dV;k1cm_}0)ObZb~(#v|}-yro(Hhg-x5?WM<9U&*%p z;9UV>UHs-%iUL!;d?hE!L+RBblNWcSE+e>7+IygP>Kuw{eUa$cTqCi_beu371WBJg z&J5n2PdcUA!E{(Evp}4-IbW!(QPRJ3F&3IQE0kV-mwejos{GUHiarMFP&<5ORzU|% z30CWoe8zq)_P4$rk`c-`o9MHRXfzlOl~_Bg4e0x?@Lx#GbQ)=HdeIizBX|Gt_qLbg zWQEwWF);xvZ^JXb<3ArAiSO=yHGiyVn3&xjZ2Mz*YP&2Gtu~v~#^|ZV`%8f=*c!HV zkr7B3_&PRse7=9>u3cP^r_)rEfim%t1i*9Z9Hj*7pE$QEq}q)hy|-N>=^}a~65;*` z4t!33BTggCRitK=Mp2}tCWg7 z{9UiQ>AE(p@`|xYTO5OAw1A!poY65y?z2fOp8jbP&$Tzv6s(I6k`eQYL`Xljs6aru zGi~0B{kT;{K2&-$Z_g zeLDL_lSXOEvrw|AfR#))w;*Jygv^=tC4--o4(ict~9opJ255QEw%*o~G7aXYeGyO2k3e(ST){$nylG*SfyM+el z$@oAQDkU%ZfmTjiXQb`4wb35PnT^mk0=v-Cmw}rBwt)gvfjEVa^LV;qL zQN6#HyF#qu(cy9b?LDV=&|4kDiHY|kGSwO~cTWn$7{^{%4$v$ewl4eR9}n&7c6Z9J z`Pus2HO7yU$i$O}VsSr97(55>u9d<>TI9JR`snU!)R1!bkg0O|V1IKi9kR@vfsie(xulFf`^I!EzGYhTpP`6hpTR^vfeZv>D3H)6 zbnWe41L(@5blf0=Aof3(OS3g>4Z7cUxd@=fFoW6A8DqcYP_O_5@k@~FJI6t}m<}1d zU(k0Ybl1lrL&JtfMY=~Gk`ez5MUiZ47@uNm=vovC5a}ZkXD>rlT>lJ>na~VIjj1&h z34@FnKcvzI$i#4S1Ck=N*%pvw{r|BPhmeoH1XZ2)1^5%tKwHp2F)t)lFpPvI$Onc- z9-_~w01_JNB?U=?j`x|H<}8e%8(R_1;W9^AsdF`4P&wbf`H+eqM^iy3&MmuJyM@Ne+=^OM5o0 zpUF0O32U^BdcN7{!{>IMr&Q-5UmnrM06kCQhH*M_oOvmuAl@@--J))NF3>nf9=TsJ zWp8^LM(%X0=v+Hnx|KY*_fXTmaU543bfZsmn6pIsn~y_+nq1)X#G)IJa)?ko=v< zFn<_dx&6|6Dai5Hm`aY?1uTyM>z^|`*EX_~4t;p(!G8mLW;f1z_p@2q`_5nlcNhI- z-C}bkM3$rXv4AA`mCfy^u152LSCaL2m1V|_sjkseA{Wlbl=bF@6Kj(*q9W zZw#ev@_sGMhm43QNq+Vrq`nPjEYb-OMMDl1dQFFNWAac$XMMj3t38$H}b;I1^*W{-8df4{vZwj8|zWg#4Hic|>#7lU8@oSmJwVH-TCYWJ?lFC+ec zkxw3UBi87kJmLBX>PdtbQ*G&i4`FGtGvEr#$Z+@OVr9KN)Bt+FWH+FVy%Q1%q6ry3 zgk!z}d~S*^Rp8d?0Z0*Npp@(2!GmV!CSN+AZcpX@1NHkM$wxNlCT(->uBrV+2#|n5 zZHGIa-KU~I8!NB8^3JsvWXN>9XK4NaHd zEq9KMmZi(Zv!CSqyoqNqB%4t-FHJOl*$iit$amq7uHu4=|2zG8yVbie&8Bf$yrVti>={CfT>Ts#Zvp{84nBcklqSqq4(x(Q|a5% zV*4qnsW+G$Pz-#*u}`nmCvsb+Rc79&JN$1pNLUnl2tw?ba?#ymabG*DnLU| zD`CV*JV%vc6DPW4@cl3O+m-h-TA=n98HF*5qHcy&*X9=0gsxj+I>SenwB?Qa&aNRa z?;w&~NrJ)jc3f&I=LdV~NmH8|QEnHK60SCjb6CdgE)>*fK|#I1#{Zo`_&-w+|5pK6 aa~f)e-P@me1|joTmVxdSoic5Q@c#!yYc4(j literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/public/assets/logos/hermes.png b/ui/litellm-dashboard/public/assets/logos/hermes.png new file mode 100644 index 0000000000000000000000000000000000000000..de47b728d12a648a0d3affd06e7d3ed75eacd562 GIT binary patch literal 1100 zcmV-S1he~zP)q7y%f8W&ke&%s_tx@FGChvw#-?7=i6_R^alj?rfw76x79^ zx##o|X}Y_ps(Y&d?m#jmnUXy7&*6OA)5jD@3X%o?dlV$a*x?obKUAF&&dq=Vz_fdM zXxkQT+oG;(e0_bPZCfa%pp=5v8lLCjO_Yw~EW4P$_FGXfj$_7g%-Cx>x!dgqb5cq+P4ki+n40OkR!aGBxYn9gRWVId=6Oz%lu}YkNze0G6a~{XeQCt6 zvFtF$004p@u!-mUK0MDuS(Z>rVY}U8zu)6ola=m7MIHfQcBcyZRe)&F(ot|T-UYNv)K$%%F&HZr_&(*LWqHJQ+P$LB91ix`n-_L=fC~Um7)|O&T(=Z4EDy6vHZYSy5 zr)>Yas;c*L!1sNmX^PEe17i%bEVE3=vJ52Qx~`3HAG6sE^Z9)AMaP~W^*|q5YerG@ z;Gw?OYpo}l(f2^7nD=;UKoA5k3rk0}u_1)8)b%{?y%0>5;3-VH_HLaq#wzSn8uoEK z9$}0@UDu;VJUe`hRbzCVG@!G#TQqi9|6W%&#=OvJNk`wON!V4dj*e5Y>r+rl`Ouhj zvF}T{5aM|bWLaii*1erE2K)W~sh8&SxvhL1uGi}z4;G7sZRI9IV2rUY?oS zC6>!2=JPpLtJPq0DdmF!V^ZBWdNc%46j2Casr8KGNs=%OLjqtJhW7f^NRs412ikrq zUa!~GT64GCQ7JXr4RtZj^ZaRvzSo2i1I(Z)ZJLH!Yx=%#7p&Lohs}Z@u-aZ#)ufSc zcp&=)AOo=OI9rw_4u=D>EJGNE`2G9$Xgrltmgm0j+xt>VD5Y#CWQ-Y9!!Qg7dEezh zIYlw8HG?1+ZONl3Vv;0mnubvnjke-xnp!Vvt@~5u{ij(y#rHm!`@BD&&#bD7d7eKc zS3j(2nsTvN^gHCb`!xHwH688k)9ib!)w-8q&+`z5Aw17RUDv4V`rfqzDej)XY0NhO z|90lTI^Z}?>o|@NUj|3GLC)Vz$bm0<)8omzPHZ9 z8x6UBeY5QB{F&G$0Twt9U@AxDsv5I5uwla|ROBz&(Q$~_Pxo;Up0>Bgo~>Hd`n0XB z>oM^`YisMh<;!Cr(02o#39@a4f7rT$o>B(kYD?F9t-88M#Rejr z?~qciC@nZ>a@;4<`$vj1W5NNprZjC$U;~u=FD0^41J{#jh=zKIcnmjE8Tg0gR@x}W zZ)HqRyin`^KnCg=`&{I^-qVVoVl7zuXcV9AIfftKZH7Po-8=KJ_`wL4>^_Q(%}s9H z&d!|}$(KGGR_|Jt>i03=K1 zm*D~KI9p810=I2siL~ipUN}4}6MiELqftbuma5B%X8yVlz@HieV)WuS5-$b~;tRu2 z$yLx@DUEe?#oDH8;`dA{HD5~tVR~S`AQ)qwVzOKySR6otbB_UPtRhR3wifT}dqlyz zrqAL0NF;zru=r>?4I`HW&kIjz40H?(JkbD13Tz_^6;xOHp&%RzspOiFI}XN7Q;Ca0 zBZcWW+myD3JWd-zd_{rHAGEZbh`N)nX1$<&j;CxEpoqCmj3~(8wy|#fLss*(4x_ulyuiu zq#6^0H2yu6{Eo9Wu$dCANlgHim{2|X3jki}k?e{CbkDHM1!1?UHQO`cULnLEQc5wU z31F&Tki>sfjj9f%g#CiCZ>763%2{q=)B!pH^i+2$cF~{BLN^6TRUG z(QWJchX5{Ado1@OP25$|>Z<@26PKv_i3zE`DSV3>8gL;?cE|!eyQL4cH$Y;WNNAHS lL~(o_$LW!U_jrp6@fsk=FmPoZ + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts index 2d82eedf25c..d9824b4753e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts @@ -4,8 +4,9 @@ import { createQueryKeys } from "../common/queryKeysFactory"; const modelCostMapKeys = createQueryKeys("modelCostMap"); -export const useModelCostMap = () => { +export const useModelCostMap = (enabled = true) => { return useQuery>({ + enabled, queryKey: modelCostMapKeys.list({}), queryFn: async () => await modelCostMap(), staleTime: 60 * 1000, // 1 minute diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx index 2e2a0318d85..c6cd8485cbe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx @@ -67,64 +67,6 @@ export function InvestigationList({ ); } -export function InvestigationExample({ onClose }: { onClose: () => void }) { - return ( - { - if (!open) onClose(); - }} - > - - - Example investigation - -
-
- - - 3 of 20 conversations -
-

- Failed lookups leave customers without answers -

- - The agent retries the same failed order lookup, then ends the conversation without an answer or a handoff. - -
-
-
-

After three failed lookups, the agent replies:

-
“I will check that for you.”
-
-
- - See the trace - -
    -
  1. - Customer · Where is my order? -
    2. -
  2. - Order lookup · Service unavailable -
    3. -
  3. - Two retries · Same error, no new information -
    4. -
  4. - Agent · I will check that for you. Conversation - ends. -
    5. -
-
-
- - - ); -} - export function MonitoringSetup({ settings, ready, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx index c2b9d227276..8249f284ada 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx @@ -53,7 +53,7 @@ export function LensProgress({ job, onCancel }: { job: Job; onCancel?: () => voi >
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx index a3ec7877223..b3d7366bbab 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx @@ -231,39 +231,18 @@ it("runs saved settings immediately without opening setup", async () => { expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); }); -it("lets a new user inspect example evidence and return to setup without starting an investigation", async () => { +it("offers the interactive demo without starting an investigation", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); vi.mocked(apiClient.get).mockImplementation(async (path) => { if (path === "/lens") return { lenses: [], workers: [], tracing_enabled: false }; return { traces: false, requests: false }; }); + const onDemo = vi.fn(); const user = userEvent.setup(); - renderWithProviders(); - - await user.click(await screen.findByRole("button", { name: "View an example" })); - const example = within(screen.getByRole("dialog", { name: "Example investigation" })); - expect(example.getByRole("heading", { name: "Failed lookups leave customers without answers" })).toBeVisible(); - const evidence = example.getByText(/Where is my order\?/); - expect(evidence).not.toBeVisible(); - await user.click(example.getByText("See the trace")); - expect(evidence).toBeVisible(); - expect(example.getByText(/Service unavailable/)).toBeVisible(); - await user.click(example.getByText("See the trace")); - expect(evidence).not.toBeVisible(); - - await user.click(example.getByRole("button", { name: "Close" })); - await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); - expect(screen.getByRole("link", { name: "Set up traces" })).toBeVisible(); - expect(screen.getByRole("button", { name: "Connect worker" })).toBeDisabled(); - expect(screen.getByRole("button", { name: "New investigation" })).toBeDisabled(); - - await user.click(screen.getByRole("button", { name: "View an example" })); - expect(screen.getByRole("dialog", { name: "Example investigation" })).toBeVisible(); - expect(screen.getByText(/Where is my order\?/)).not.toBeVisible(); - await user.keyboard("{Escape}"); - await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); - expect(screen.getByRole("button", { name: "View an example" })).toBeVisible(); + renderWithProviders(); + await user.click(await screen.findByRole("button", { name: "Preview sample" })); + expect(onDemo).toHaveBeenCalledOnce(); expect(apiClient.post).not.toHaveBeenCalled(); }); @@ -276,13 +255,14 @@ it("guides a first-time administrator into worker connection and lens setup", as return { traces: true, requests: false, data: [] }; }); const user = userEvent.setup(); - renderWithProviders(); + renderWithProviders(); const guide = within(await screen.findByRole("region", { name: "Find what needs attention" })); expect(apiClient.get).toHaveBeenCalledWith("/lens/activity/available", { accessToken: "test" }); expect(await guide.findByRole("link", { name: "View traces" })).toHaveAttribute( "href", expect.stringMatching(/^\/ui\/lens\/?\?tab=traces$/), ); + expect(await guide.findByRole("button", { name: "Preview sample" })).toBeVisible(); await user.click(guide.getByRole("button", { name: "Connect worker" })); const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); expect(connection.getByRole("button", { name: "Get install command" })).toBeVisible(); @@ -305,6 +285,7 @@ it("guides a first-time administrator into worker connection and lens setup", as }); }); await waitFor(() => expect(guide.getByRole("button", { name: "New investigation" })).toBeEnabled()); + expect(guide.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); await user.click(guide.getByRole("button", { name: "New investigation" })); expect(await screen.findByRole("dialog", { name: "Which activity should we investigate?" })).toBeVisible(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx index 09a8607813d..cd78effe0f8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import type { components } from "@/lib/http/schema"; import { useEffect, useState } from "react"; @@ -28,7 +29,7 @@ import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs"; import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetDescription } from "@/components/ui/sheet"; import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; import { LensFinding } from "./LensFinding"; -import { apiClient } from "@/components/networking"; +import { apiClient as liveApiClient } from "@/components/networking"; import { TracePanel } from "./TracePanel"; import { LensSetup } from "./LensSetup"; import { LensRuns } from "./LensRuns"; @@ -37,13 +38,7 @@ import { WorkerSetup } from "./WorkerSetup"; import { useAnalysisKeyInfo } from "./AnalysisKeyDetails"; import { uiHref } from "@/utils/uiHref"; import { ApiError } from "@/lib/http/client"; -import { - InvestigationList, - InvestigationExample, - MonitoringSetup, - InvestigationSummary, - InvestigationFailure, -} from "./LensOverview"; +import { InvestigationList, MonitoringSetup, InvestigationSummary, InvestigationFailure } from "./LensOverview"; import { LensWelcome } from "./LensWelcome"; import { workerConnected, @@ -70,10 +65,19 @@ function emptyFindingTitle(active: boolean, scanned: boolean, status?: string) { return scanned ? "No matching findings" : "Ready for the first analysis"; } -export function LensView({ accessToken, readOnly = false }: { accessToken: string; readOnly?: boolean }) { +export function LensView({ + accessToken, + readOnly = false, + onDemo, +}: { + accessToken: string; + readOnly?: boolean; + onDemo?: () => void; +}) { + const demo = useLensDemo(); + const apiClient = demo?.client ?? liveApiClient; const client = useQueryClient(); const [workerSetup, setWorkerSetup] = useState(false); - const [showExample, setShowExample] = useState(false); const [monitoring, setMonitoring] = useState(false); const [now, setNow] = useState(Date.now); useEffect(() => { @@ -85,6 +89,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryKey: key, queryFn: () => apiClient.get("/lens", { accessToken }), refetchInterval: (current) => { + if (demo) return false; const running = current.state.data?.lenses.some((item) => item.jobs.some((job) => ["queued", "running"].includes(job.status)), ); @@ -100,7 +105,10 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryFn: () => apiClient.get<{ data: import("./lensData").AnalysisModelInfo[] }>("/model_group/info", { accessToken }), }); - const [selected, setSelected] = useQueryState("lens", parseAsString.withOptions({ history: "push" })); + const [liveSelected, setLiveSelected] = useQueryState("lens", parseAsString.withOptions({ history: "push" })); + const [demoSelected, setDemoSelected] = useState(null); + const selected = demo ? demoSelected : liveSelected; + const setSelected = demo ? setDemoSelected : setLiveSelected; const [editing, setEditing] = useState<"new" | "edit" | "duplicate" | null>(null); const [batchId, setBatchId] = useState("latest"); const [historyOffset, setHistoryOffset] = useState(0); @@ -146,7 +154,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryKey: ["lens-activity-available", accessToken], queryFn: () => apiClient.get<{ traces: boolean; requests: boolean }>("/lens/activity/available", { accessToken }), enabled: loaded, - refetchInterval: 5000, + refetchInterval: demo ? (false as const) : 5000, }; const activity = useQuery(activityOptions); const tracesReady = activity.data?.traces === true && !activity.error; @@ -157,7 +165,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryKey: ["lens-history", lens?.id, historyOffset, accessToken], enabled: !!lens, queryFn: () => apiClient.get(`/lens/${lens?.id}/runs`, { accessToken, query: { offset: historyOffset } }), - refetchInterval: 10000, + refetchInterval: demo ? (false as const) : 10000, }; const history = useQuery(historyQuery); const historical = useQuery({ @@ -297,7 +305,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin }} onConnect={() => setWorkerSetup(true)} onCreate={() => setEditing("new")} - onExample={() => setShowExample(true)} + onDemo={activity.isSuccess && !ready ? onDemo : undefined} /> )} {showReadiness && !ready && ( @@ -687,7 +695,6 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin } /> )} - {showExample && setShowExample(false)} />} {monitoring && lens && ( void; onCreate: () => void; onRetry: () => void; - onExample: () => void; + onDemo?: () => void; }) { const activityReady = tracesReady || requestsReady; const workerReady = activityReady && connected; @@ -39,16 +40,14 @@ export function LensWelcome({ className: "col-start-2 w-fit sm:col-start-auto", }); return ( -
+
+ {onDemo && }

Find what needs attention

Check how your agents behave. Get findings you can trace back to what happened.

-
  1. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx new file mode 100644 index 00000000000..8b7531c9bc2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx @@ -0,0 +1,137 @@ +import { fireEvent, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { LensWorkspace } from "./LensWorkspace"; +import { createLensDemoData } from "./lensDemoData"; + +const network = vi.fn(); +beforeEach(() => { + testQueryClient.clear(); + vi.stubGlobal("fetch", network); + network.mockReset(); + network.mockImplementation(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); + if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: false }); + return Response.json({ data: [], traces: false, requests: false }); + }); +}); + +describe("Lens interactive demo", () => { + it("opens without tracing, filters sample runs, and restores the live view without mixing data", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { + onUrlUpdate, + }); + expect(await screen.findByText("Enable tracing")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Preview sample" })); + expect(await screen.findByText("Where is order #1042?")).toBeVisible(); + expect(screen.getByText("You’re viewing demo data")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Set up tracing" })).not.toBeInTheDocument(); + network.mockClear(); + await user.click(screen.getByRole("button", { name: "Refresh", exact: true })); + expect(screen.getByText("Where is order #1042?")).toBeVisible(); + fireEvent.change(screen.getByPlaceholderText("Search input or trace ID"), { target: { value: "headphones" } }); + expect(screen.getByText("Can I return my headphones?")).toBeVisible(); + expect(screen.queryByText("Where is order #1042?")).not.toBeInTheDocument(); + fireEvent.change(screen.getByRole("textbox", { name: "Search runs" }), { target: { value: "" } }); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by agent" }), "support_agent"); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by status" }), "Failed"); + expect(within(screen.getByRole("table", { name: "Agent runs" })).getAllByRole("row")).toHaveLength(4); + await user.click(screen.getByRole("tab", { name: "Investigations", exact: true })); + expect(await screen.findByRole("button", { name: /Support quality/ })).toBeVisible(); + expect(screen.queryByRole("button", { name: "New investigation" })).not.toBeInTheDocument(); + expect(network).not.toHaveBeenCalled(); + expect(onUrlUpdate).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(await screen.findByText("Enable tracing")).toBeVisible(); + expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); + expect(screen.queryByText("Can I return my headphones?")).not.toBeInTheDocument(); + }); + + it("connects findings and history to their original trace without live requests or URL changes", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { + searchParams: "?tab=investigations", + onUrlUpdate, + }); + await screen.findByRole("button", { name: "Preview sample" }); + await user.click(screen.getByRole("button", { name: "Preview sample" })); + await user.click(await screen.findByRole("button", { name: /Support quality/ })); + network.mockClear(); + await user.click(screen.getByRole("button", { name: /Repeated lookups leave customers without an answer/ })); + const finding = screen.getByRole("dialog"); + expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); + const summaries = within(finding).getAllByText("support_agent", { exact: true }); + await user.click(summaries[0]); + await user.click(within(finding).getAllByRole("button", { name: /Open original step/ })[0]); + expect(await screen.findByRole("complementary", { name: "Span details" })).toHaveTextContent( + "I will check that for you.", + ); + await user.click(screen.getByRole("button", { name: "Copy for agent" })); + expect(await navigator.clipboard.readText()).toContain("I will check that for you."); + expect(await navigator.clipboard.readText()).not.toContain("Authorization"); + await user.click(screen.getByRole("tab", { name: "Attributes" })); + expect(await screen.findByText("gen_ai.agent.name")).toBeVisible(); + await user.click( + within(screen.getByRole("dialog", { name: "Original run" })).getByRole("button", { name: "Close", exact: true }), + ); + await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Close", exact: true })); + await user.click(screen.getByRole("tab", { name: "History" })); + await user.click(screen.getAllByRole("button", { name: /runs reviewed/ })[1]); + expect(await screen.findByText(/1 linked run · high priority/)).toBeVisible(); + expect(network).not.toHaveBeenCalled(); + expect(onUrlUpdate).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(await screen.findByRole("heading", { name: "Find what needs attention" })).toBeVisible(); + }); + + it("has no demo entry for existing investigations, populated traces, or connecting another agent", async () => { + const user = userEvent.setup(); + const data = createLensDemoData(); + const saved = data.lenses[0]; + network.mockImplementation(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: true }); + if (path.endsWith("/runs")) return Response.json(saved.jobs); + if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) }); + return Response.json({ data: [], traces: true, requests: false }); + }); + renderWithProviders(, { + searchParams: `?tab=investigations&lens=${saved.id}`, + }); + expect(await screen.findByRole("heading", { name: saved.settings.name })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Traces" })); + expect(await screen.findByText("Where is order #1042?")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Set up tracing" })); + expect(await screen.findByRole("heading", { name: "Connect another agent" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + }); + + it("shows the header preview only for the active tab that still needs setup", async () => { + const user = userEvent.setup(); + const saved = createLensDemoData().lenses[0]; + network.mockImplementation(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: false }); + if (path.endsWith("/runs")) return Response.json(saved.jobs); + if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); + return Response.json({ data: [], traces: false, requests: false }); + }); + renderWithProviders(); + expect(await screen.findByRole("button", { name: "Preview sample" })).toBeVisible(); + const tabs = within(screen.getByRole("tablist", { name: "Lens" })); + await user.click(tabs.getByRole("tab", { name: "Investigations" })); + expect(await screen.findByRole("button", { name: new RegExp(saved.settings.name) })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(tabs.getByRole("tab", { name: "Traces" })); + await user.click(await screen.findByRole("button", { name: "Preview sample" })); + expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx new file mode 100644 index 00000000000..9c7d2ca3db8 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Aperture } from "lucide-react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { parseAsString, parseAsStringLiteral, useQueryState } from "nuqs"; +import AgentTracesPage from "@/components/view_logs/TraceView/AgentTracesPage"; +import { DemoNotice } from "@/components/shared/DemoNotice"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { LensDemoContext, useLensDemo } from "@/components/lens/LensDemoContext"; +import { LensPreviewTarget } from "@/components/lens/LensPreviewButton"; +import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles"; +import { LensView } from "./LensView"; +import { createLensDemo } from "./lensDemoData"; + +type Tab = "traces" | "investigations"; +type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean }; + +export function LensWorkspace(props: WorkspaceProps) { + const [demoTab, setDemoTab] = useState(null); + return demoTab ? ( + setDemoTab(null)} /> + ) : ( + + ); +} + +function DemoSession({ initialTab, onExit }: { initialTab: Tab; onExit: () => void }) { + const [demo] = useState(createLensDemo); + const [client] = useState( + () => new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity } } }), + ); + useEffect( + () => () => { + client.clear(); + }, + [client], + ); + return ( + + + + + + ); +} + +function LensContent({ + accessToken, + userRole, + readOnly, + initialTab = "traces", + onDemo, + onExit, +}: WorkspaceProps & { initialTab?: Tab; onDemo?: (tab: Tab) => void; onExit?: () => void }) { + const demo = useLensDemo(); + const [tab, setTab] = useQueryState( + "tab", + parseAsStringLiteral(["traces", "investigations"]).withOptions({ history: "push" }), + ); + const [lensId] = useQueryState("lens", parseAsString); + const [demoTab, setDemoTab] = useState(initialTab); + const [previewTarget, setPreviewTarget] = useState(null); + const defaultTab = lensId ? "investigations" : "traces"; + const activeTab = demo ? demoTab : tab ?? defaultTab; + const openDemo = onDemo ? () => onDemo(activeTab) : undefined; + return ( + +
    +
    +

    +

    +
    +
    + {demo && } + (demo ? setDemoTab(value as Tab) : void setTab(value as Tab))} + className="min-h-0 flex-1 gap-4" + > + + + Traces + + + Investigations + + + + + + + {demo || isProxyAdminTierRole(userRole) ? ( + + ) : ( +

    + Investigations require proxy administrator access. You can still view your traces. +

    + )} +     +     +
    +     + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx index 4554174f37b..931dd7c17d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx @@ -21,7 +21,7 @@ import { AnalysisKey, AnalysisAccessFields, createAnalysisKey, type AnalysisAcce import type { LensList, WorkerCreated } from "./lensData"; export const LENS_WORKER_IMAGE = - "ghcr.io/berriai/litellm-lens-worker@sha256:67eba741c1b97c749975c5c38e2370a603e1105babc908d613c1b79d7b995393"; + "ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b"; function initialProxyAddress(): string { const url = new URL(proxyBaseUrl || serverRootPath, window.location.origin); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts index 84e0fe7e6e6..3be6164fce2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts @@ -58,6 +58,28 @@ const job: Job = { }; describe("Analysis progress", () => { + it.each(["Collecting executions", "Reading executions"])( + "does not turn an unreported run count into zero runs during %s", + (stage) => { + expect(analysisProgress({ ...job, stage })).toEqual({ + step: -1, + title: "Preparing activity", + done: 0, + total: 0, + detail: "Loading the runs selected for this investigation.", + }); + }, + ); + + it("shows the selected count as soon as the worker reports it", () => { + expect(analysisProgress({ ...job, coverage: { ...coverage, selected: 7 } })).toMatchObject({ + step: 0, + done: 0, + total: 7, + detail: "0 of 7 selected runs reviewed", + }); + }); + it("measures review progress against the sample, not all eligible runs", () => { const expected = { step: 0, done: 7, total: 20, detail: "7 of 20 selected runs reviewed" }; expect( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts index c066577817b..1e30dc75a38 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts @@ -108,6 +108,15 @@ export function analysisProgress(job: Job) { : `${investigated} patterns checked against the original activity`, }; } + if (!selected) { + return { + step: -1, + title: "Preparing activity", + done: 0, + total: 0, + detail: "Loading the runs selected for this investigation.", + }; + } return { step: 0, title: "Reviewing activity", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts new file mode 100644 index 00000000000..eabae2928bd --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, vi } from "vitest"; +import { createLensDemo, createLensDemoData } from "./lensDemoData"; +import type { TracePage } from "@/components/view_logs/TraceView/traceTypes"; +import { evidenceTarget } from "./lensData"; + +describe("Lens demo data", () => { + it("links every finding to the quoted original step and assessed run", () => { + const data = createLensDemoData(); + for (const lens of data.lenses) { + for (const job of lens.jobs) { + for (const finding of job.findings ?? []) { + for (const evidence of finding.evidence) { + const target = evidenceTarget(evidence.execution_id); + const run = data.runs.find(({ trace }) => trace.summary.trace_id === target?.id); + const step = run?.details.find((span) => span.span_id === evidence.span_id); + expect(step?.output).toContain(evidence.quote); + expect(job.sample?.executions.map((execution) => execution.id)).toContain(evidence.execution_id); + expect( + job.assessments.find((assessment) => assessment.execution_id === evidence.execution_id)?.[ + finding.kind === "issue" ? "issue_checks" : "pattern_checks" + ], + ).toContain(finding.check_id); + } + } + } + } + }); + + it("keeps trace totals, timestamps, and agent names consistent", () => { + const data = createLensDemoData(); + for (const { trace } of data.runs) { + expect(trace.summary.span_count).toBe(trace.spans.length); + expect(trace.summary.agent_names).toContain(trace.agents[0].name); + expect(trace.summary.error_count).toBe(trace.spans.filter((span) => span.status === "error").length); + for (const span of trace.spans) { + expect(span.start_offset_ms + span.duration_ms).toBeLessThanOrEqual(trace.summary.duration_ms); + } + } + }); + + it("filters time windows locally and rejects writes or unknown reads without network access", async () => { + const network = vi.spyOn(globalThis, "fetch"); + const now = Date.now(); + const { client } = createLensDemo(now); + const all = await client.get("/v1/traces"); + const recent = await client.get("/v1/traces", { query: { start_ms: now - 3600_000, end_ms: now } }); + expect(recent.data.length).toBeGreaterThan(0); + expect(recent.data.length).toBeLessThan(all.data.length); + expect(recent.data.every((trace) => Date.parse(trace.start_time) >= now - 3600_000)).toBe(true); + await expect(client.post("/lens", { body: {} })).rejects.toMatchObject({ status: 403 }); + await expect(client.get("/lens/real-investigation")).rejects.toMatchObject({ status: 404 }); + expect(network).not.toHaveBeenCalled(); + network.mockRestore(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts new file mode 100644 index 00000000000..8b4e1645c9e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts @@ -0,0 +1,483 @@ +import { createApiClient } from "@/lib/http/client"; +import type { LensDemo } from "@/components/lens/LensDemoContext"; +import type { Trace, Span, SpanDetail } from "@/components/view_logs/TraceView/traceTypes"; +import type { Lens, Finding, Job, Settings } from "./lensData"; + +type Scenario = { + agent: string; + question: string; + tool: string; + result: string; + answer: string; + failed?: boolean; +}; + +const scenarios: Scenario[] = [ + { + agent: "support_agent", + question: "Where is order #1042?", + tool: "lookup_order", + result: "Order service timed out", + answer: "I will check that for you.", + failed: true, + }, + { + agent: "research_agent", + question: "Which database should we use for analytics?", + tool: "search_docs", + result: "Column stores are designed for analytical queries. No comparative benchmark is available.", + answer: "The column store is 40% faster than the row store for your workload.", + }, + { + agent: "support_agent", + question: "Can I return my headphones?", + tool: "lookup_return_policy", + result: "Returns are accepted within 30 days. This order was delivered 10 days ago.", + answer: "Yes. Your order is eligible for a return. I can help you start it.", + }, + { + agent: "release_agent", + question: "Is the checkout release ready to ship?", + tool: "read_test_results", + result: "Checkout: 120 passed, 0 failed. Rollback plan approved.", + answer: "All 120 checkout tests passed and the rollback plan is approved. The release is ready.", + }, + { + agent: "support_agent", + question: "My order #1088 has not arrived", + tool: "lookup_order", + result: "Order service timed out", + answer: "I will check that for you.", + failed: true, + }, + { + agent: "research_agent", + question: "How long do we retain audit logs?", + tool: "search_docs", + result: "Audit log retention is 90 days on the current plan.", + answer: "Audit logs are retained for 90 days, according to the plan documentation.", + }, + { + agent: "release_agent", + question: "Review the search release", + tool: "read_test_results", + result: "Search: 86 passed, 1 failed. Unicode query regression remains open.", + answer: "Hold the release. The Unicode query regression is still failing.", + }, + { + agent: "support_agent", + question: "Update the address on order #1091", + tool: "lookup_order", + result: "Order #1091 has not shipped. Address changes are allowed.", + answer: "Your order has not shipped. Please confirm the new delivery address.", + }, + { + agent: "research_agent", + question: "Can this plan store data in Europe?", + tool: "search_docs", + result: "Regional availability is not documented for this plan.", + answer: "I could not verify European data residency for this plan. Please confirm with support before choosing it.", + }, + { + agent: "support_agent", + question: "Check the delivery date for order #1107", + tool: "lookup_order", + result: "Order service timed out", + answer: "The order service is unavailable. I can connect you to support instead.", + failed: true, + }, + { + agent: "release_agent", + question: "Summarize the authentication release checks", + tool: "read_test_results", + result: "Authentication: 214 passed, 0 failed. Security review approved.", + answer: "All 214 tests passed and the security review is approved.", + }, + { + agent: "research_agent", + question: "Does the API support request retries?", + tool: "search_docs", + result: "Retry requests with exponential backoff after a 429 or 503 response.", + answer: "Yes. Use exponential backoff for 429 and 503 responses, as documented.", + }, +]; + +const executionId = (traceId: string) => btoa(JSON.stringify(["traces", "", traceId])); +const iso = (time: number) => new Date(time).toISOString(); + +function makeTrace(scene: Scenario, index: number, now: number) { + const traceId = (index + 1).toString(16).padStart(32, "0"); + const spanId = (step: number) => `${index + 1}${step}`.padStart(16, "0"); + const toolCount = scene.failed && index !== 9 ? 3 : 1; + const duration = toolCount === 3 ? 6340 : 3180 + index * 137; + const base: Span = { + agent: scene.agent, + duration_ms: duration, + error: null, + error_truncated: false, + framework: "", + input_preview: scene.question, + input_tokens: 0, + output_tokens: 0, + litellm_request_id: null, + model: null, + name: scene.agent, + parent_span_id: null, + span_id: spanId(0), + spend: null, + start_offset_ms: 0, + status: "ok", + type: "agent", + }; + const tools: Span[] = Array.from({ length: toolCount }, (_, i) => ({ + ...base, + span_id: spanId(i + 1), + parent_span_id: base.span_id, + name: scene.tool, + type: "tool", + start_offset_ms: 150 + i * 1600, + duration_ms: scene.failed ? 1500 : 340, + status: scene.failed ? "error" : "ok", + error: scene.failed ? scene.result : null, + })); + const model: Span = { + ...base, + span_id: spanId(5), + parent_span_id: base.span_id, + name: "Generate response", + type: "llm", + model: "demo-chat-model", + start_offset_ms: toolCount * 1600, + duration_ms: 1200, + input_tokens: 520 + index * 41, + output_tokens: 48 + index * 7, + spend: 0.003 + index * 0.0002, + }; + const trace: Trace = { + summary: { + agent_count: 1, + agent_invocations: 1, + agent_names: [scene.agent], + duration_ms: duration, + error_count: scene.failed ? toolCount : 0, + frameworks: [], + input_preview: scene.question, + input_tokens: model.input_tokens, + output_tokens: model.output_tokens, + llm_calls: 1, + models: [model.model!], + name: scene.agent, + service: "demo-agents", + span_count: toolCount + 2, + spend: model.spend, + start_time: iso(now - (index + 1) * 35 * 60_000), + status: scene.failed ? "error" : "ok", + tool_calls: toolCount, + trace_id: traceId, + }, + agents: [ + { + name: scene.agent, + parent_agent: null, + duration_ms: duration, + invocations: 1, + llm_calls: 1, + tool_calls: toolCount, + spend: model.spend, + }, + ], + spans: [base, ...tools, model], + }; + const details: SpanDetail[] = trace.spans.map((span) => ({ + span_id: span.span_id, + input: + span.type === "tool" + ? JSON.stringify({ query: scene.question }) + : JSON.stringify([{ role: "user", content: scene.question }]), + output: span.type === "tool" ? scene.result : JSON.stringify([{ role: "assistant", content: scene.answer }]), + attributes: { + "gen_ai.agent.name": scene.agent, + "service.name": "demo-agents", + demo: "true", + ...(span.model ? { "gen_ai.request.model": span.model } : {}), + }, + })); + return { trace, details }; +} + +export function createLensDemoData(now = Date.now()) { + const runs = scenarios.map((scene, index) => makeTrace(scene, index, now)); + const finding = ({ + id, + check, + title, + description, + suggestion, + indices, + kind = "issue", + }: { + id: string; + check: string; + title: string; + description: string; + suggestion: string; + indices: number[]; + kind?: Finding["kind"]; + }): Finding => ({ + id, + check_id: check, + title, + description, + suggestion, + kind, + priority: kind === "issue" ? "high" : "low", + status: "open", + reason: "", + revision: 1, + first_seen: iso(now - 86_400_000), + last_seen: iso(now - 300_000), + limitation: "", + occurrences: indices.map((i) => executionId(runs[i].trace.summary.trace_id)), + evidence: indices.map((i) => ({ + execution_id: executionId(runs[i].trace.summary.trace_id), + span_id: runs[i].trace.spans.at(-1)!.span_id, + quote: scenarios[i].answer, + role: "support", + })), + }); + const findingInputs: Parameters[0][] = [ + { + id: "failed-lookups", + check: "recover", + title: "Repeated lookups leave customers without an answer", + description: + "The support agent retries the same unavailable order service three times, then promises to check without answering or offering a handoff.", + suggestion: "After repeated failures, explain the problem and offer a handoff.", + indices: [0, 4], + }, + { + id: "safe-handoff", + check: "recover", + title: "A clear handoff helps when the order service is unavailable", + description: + "The agent explains the service outage and offers a support handoff instead of promising an answer it cannot provide.", + suggestion: "Keep this fallback for unavailable services.", + indices: [9], + kind: "pattern", + }, + { + id: "unsupported-claim", + check: "grounding", + title: "Performance claim has no supporting benchmark", + description: + "The answer claims a 40% performance improvement, but the retrieved documentation contains no comparative benchmark.", + suggestion: "Require a benchmark source for numeric performance claims, or remove the comparison.", + indices: [1], + }, + { + id: "uncertainty", + check: "grounding", + title: "Missing information is acknowledged", + description: + "When documentation does not establish regional availability, the agent says so and asks the user to verify it.", + suggestion: "Keep stating when a source does not answer the question.", + indices: [8], + kind: "pattern", + }, + { + id: "release-blocked", + check: "release", + title: "Failing checks correctly block the release", + description: "The release agent identifies an unresolved regression and recommends holding the release.", + suggestion: "Continue requiring passing checks before recommending a release.", + indices: [6], + kind: "pattern", + }, + ]; + const findings = findingInputs.map(finding); + const definitions = [ + { + id: "support", + name: "Support quality", + agent: "support_agent", + check: "recover", + context: + "Answer the customer's question using order information. If a tool fails, explain the problem and offer a handoff.", + instruction: "Look for repeated failed calls and conversations that end without an answer or a handoff.", + }, + { + id: "research", + name: "Research accuracy", + agent: "research_agent", + check: "grounding", + context: "Answer questions using verified documentation. Acknowledge missing information.", + instruction: "Find claims that are not supported by the retrieved sources.", + }, + { + id: "release", + name: "Release readiness", + agent: "release_agent", + check: "release", + context: "Review test results and recommend a release only when all required checks pass.", + instruction: "Check whether failed tests are acknowledged before a release recommendation.", + }, + ]; + const lenses: Lens[] = definitions.map((definition) => { + const settings: Settings = { + name: definition.name, + agent_name: definition.agent, + context: definition.context, + checks: [{ id: definition.check, instruction: definition.instruction, enabled: true }], + source: "traces", + service: "", + filters: [], + team_id: "", + execution_ids: [], + lookback_hours: 24, + sample_size: 0, + sample_percent: 100, + concurrency: 8, + monthly_budget: 100, + interval_minutes: 30, + enabled: false, + model: "demo-analysis-model", + }; + const executions = runs + .filter(({ trace }) => trace.summary.name === definition.agent) + .map(({ trace }) => ({ + id: executionId(trace.summary.trace_id), + trace_ref: "", + metadata: [], + root_seen: true, + service: trace.summary.service, + source: "traces" as const, + trace_id: trace.summary.trace_id, + team_id: "", + name: trace.summary.name, + start_time: trace.summary.start_time, + span_count: trace.summary.span_count, + })); + const relevant = findings.filter((f) => f.check_id === definition.check); + const jobs: Job[] = [0, 1].map((day) => { + const sample = day === 0 ? executions : executions.slice(1); + const selectedIds = new Set(sample.map((item) => item.id)); + const snapshot = relevant + .map((f) => ({ + ...f, + occurrences: f.occurrences.filter((id) => selectedIds.has(id)), + evidence: f.evidence.filter((e) => selectedIds.has(e.execution_id)), + })) + .filter((f) => f.occurrences.length > 0); + return { + id: `${definition.id}-scan-${day}`, + findings: snapshot, + settings, + revision: 1, + assessments: sample.map((e) => ({ + execution_id: e.id, + cannot_assess: false, + issue_checks: snapshot.some((f) => f.kind === "issue" && f.occurrences.includes(e.id)) + ? [definition.check] + : [], + pattern_checks: snapshot.some((f) => f.kind === "pattern" && f.occurrences.includes(e.id)) + ? [definition.check] + : [], + })), + attempts: 1, + error: "", + cost: sample.length * 0.012, + coverage: { + eligible: sample.length, + selected: sample.length, + screened: sample.length, + investigated: sample.length, + inconclusive: 0, + grouping_batches: 1, + grouped_batches: 1, + candidates: snapshot.length, + partial: 0, + unassessable: 0, + }, + status: "completed", + stage: "Complete", + created_at: iso(now - 330_000 - day * 60_000), + finished_at: iso(now - 300_000 - day * 60_000), + start: iso(now - 86_400_000), + end: iso(now - 300_000 - day * 60_000), + sample: { eligible: sample.length, selected: sample.length, executions: sample }, + }; + }); + return { + id: definition.id, + version: 1, + revision: 1, + spent: jobs.reduce((sum, job) => sum + job.cost, 0), + scope: { all_teams: true, api_key_hash: "", team_id: "" }, + settings, + created_at: iso(now - 2 * 86_400_000), + next_run_at: iso(now), + budget_month: iso(now).slice(0, 7), + findings: relevant, + jobs, + }; + }); + return { runs, lenses }; +} + +function readDemoTrace(data: ReturnType, url: URL): unknown { + const parts = url.pathname.split("/").filter(Boolean); + if (parts[0] !== "v1" || parts[1] !== "traces") return undefined; + if (!parts[2]) { + const start = Number(url.searchParams.get("start_ms") ?? 0); + const end = Number(url.searchParams.get("end_ms") ?? Infinity); + return { + data: data.runs + .map((run) => run.trace.summary) + .filter((trace) => Date.parse(trace.start_time) >= start && Date.parse(trace.start_time) <= end), + next_cursor: null, + }; + } + const run = data.runs.find(({ trace }) => trace.summary.trace_id === parts[2]); + if (!parts[3]) return run?.trace; + if (parts[3] === "spans" && !parts[5]) return run?.details.find((span) => span.span_id === parts[4]); + if (parts[3] === "spans" && parts[5] === "error") { + const span = run?.trace.spans.find((item) => item.span_id === parts[4]); + return span + ? { span_id: span.span_id, message: span.error ?? "", total_chars: span.error?.length ?? 0, next_cursor: null } + : undefined; + } + return undefined; +} + +export function createLensDemo(now = Date.now()): LensDemo { + const data = createLensDemoData(now); + const read = (url: URL): unknown => { + const parts = url.pathname.split("/").filter(Boolean); + if (url.pathname === "/lens") return { lenses: data.lenses, workers: [], tracing_enabled: true }; + if (url.pathname === "/lens/activity/available") return { traces: true, requests: false }; + if (url.pathname === "/models" || url.pathname === "/model_group/info") return { data: [] }; + if (parts[0] === "lens" && parts[2] === "runs") { + const jobs = data.lenses.find((lens) => lens.id === parts[1])?.jobs; + return parts[3] + ? jobs?.find((job) => job.id === parts[3]) + : jobs?.slice(Number(url.searchParams.get("offset") ?? 0)); + } + return readDemoTrace(data, url); + }; + return { + client: createApiClient({ + getBaseUrl: () => "https://lens-demo.invalid", + fetchImpl: async (input, init) => { + if (init?.method !== "GET") return Response.json({ detail: "Demo data is read-only" }, { status: 403 }); + const result = read(new URL(String(input))); + return result === undefined + ? Response.json({ detail: "This item is not in the demo" }, { status: 404 }) + : Response.json(result); + }, + }), + copyTrace: (traceId, spanId) => { + const run = data.runs.find(({ trace }) => trace.summary.trace_id === traceId); + return JSON.stringify(spanId ? run?.details.find((span) => span.span_id === spanId) : run, null, 2); + }, + }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx index b29e498600f..da13f48165b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx @@ -1,60 +1,10 @@ "use client"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { Aperture } from "lucide-react"; -import { parseAsString, parseAsStringLiteral, useQueryState } from "nuqs"; -import AgentTracesPage from "@/components/view_logs/TraceView/AgentTracesPage"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles"; -import { LensView } from "./_components/LensView"; +import { LensWorkspace } from "./_components/LensWorkspace"; export default function LensPage() { const { accessToken, userRole, isViewOnly } = useAuthorized(); - const [tab, setTab] = useQueryState( - "tab", - parseAsStringLiteral(["traces", "investigations"]).withOptions({ history: "push" }), - ); - const [lensId] = useQueryState("lens", parseAsString); - const defaultTab = lensId ? "investigations" : "traces"; - const activeTab = tab ?? defaultTab; if (!accessToken) return null; - return ( -
    -

    -

    - void setTab(value as "traces" | "investigations")} - className="min-h-0 flex-1 gap-4" - > - - - Traces - - - Investigations - - - - - - - {isProxyAdminTierRole(userRole ?? "") ? ( - - ) : ( -

    - Investigations require proxy administrator access. You can still view your traces. -

    - )} -     -     -
    - ); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx index 3e71e9fb860..1501938e9c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx @@ -249,6 +249,32 @@ describe("ROICalculatorView", () => { expect(screen.getAllByText("Connect GitHub to get started")).toHaveLength(1); }); + it("clearly identifies the sample report and returns to setup when exiting", async () => { + const emptySettings = { ...settings, has_github_token: false, ready: false, repos: [], estimator_model: "" }; + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(emptySettings); + if (path === "/roi-calculator/report") { + return Promise.resolve({ report: options?.query?.mode === "demo" ? { ...summary, mode: "demo" } : null }); + } + return Promise.resolve(idleStatus); + }); + + render(); + fireEvent.click(await screen.findByRole("button", { name: "Preview sample report" })); + + expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); + expect(screen.getByText("Spend per estimated engineering hour")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Settings" })).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Exit demo" })); + + expect(screen.getByRole("heading", { name: "Connect GitHub to get started" })).toBeVisible(); + expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + expect(apiClient.put).not.toHaveBeenCalled(); + }); + it("returns to Overview and shows the last sync time when completion is polled from Settings", async () => { const runningStatus = { ...idleStatus, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx index 1f5b136dbb7..40c1104354d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx @@ -5,6 +5,7 @@ import { Calculator, RefreshCw } from "lucide-react"; import { apiClient } from "@/components/networking"; import { PageHeader } from "@/components/shared/PageHeader"; +import { DemoNotice } from "@/components/shared/DemoNotice"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; @@ -222,6 +223,13 @@ export default function ROICalculatorView({ } title="ROI Calculator" + titleAction={ + !liveSummary && showLiveStatus ? ( + + ) : undefined + } subtitle={ <> {summary @@ -238,22 +246,7 @@ export default function ROICalculatorView({ } /> - {!liveSummary && showLiveStatus && ( - - )} - {sampleSummary && ( - - Sample report - - Example data only. No GitHub or model requests were made. - - - - )} + {sampleSummary && setSampleSummary(null)} />} {liveSummary && showLiveStatus && (

    {status.next_update ? `Next update ${formatSyncedAt(status.next_update)}` : scheduleLabel} diff --git a/ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx b/ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx new file mode 100644 index 00000000000..dd34511bc09 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx @@ -0,0 +1,12 @@ +"use client"; + +import { createContext, useContext } from "react"; +import type { ApiClient } from "@/lib/http/client"; + +export interface LensDemo { + client: ApiClient; + copyTrace: (traceId: string, spanId?: string) => string; +} + +export const LensDemoContext = createContext(null); +export const useLensDemo = () => useContext(LensDemoContext); diff --git a/ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx b/ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx new file mode 100644 index 00000000000..845a6b8d279 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx @@ -0,0 +1,18 @@ +"use client"; + +import { createContext, useContext } from "react"; +import { createPortal } from "react-dom"; +import { Button } from "@/components/ui/button"; + +export const LensPreviewTarget = createContext(undefined); + +export function LensPreviewButton({ onClick }: { onClick: () => void }) { + const target = useContext(LensPreviewTarget); + const button = ( + + ); + if (target === null) return null; + return target ? createPortal(button, target) : button; +} diff --git a/ui/litellm-dashboard/src/components/shared/DemoNotice.tsx b/ui/litellm-dashboard/src/components/shared/DemoNotice.tsx new file mode 100644 index 00000000000..e435d725db5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DemoNotice.tsx @@ -0,0 +1,16 @@ +import { Info } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +export function DemoNotice({ onExit }: { onExit?: () => void }) { + return ( +

    +

    +

    + +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx index 81092821efc..6fc5b0f3238 100644 --- a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx +++ b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx @@ -14,11 +14,12 @@ interface PageHeaderProps { subtitle: React.ReactNode; icon: React.ReactNode; primaryAction?: React.ReactNode; + titleAction?: React.ReactNode; tabs?: React.ReactNode | ((slots: EmbeddedTabsSlots) => React.ReactNode); utilities?: React.ReactNode; } -export function PageHeader({ title, subtitle, icon, primaryAction, tabs, utilities }: PageHeaderProps) { +export function PageHeader({ title, subtitle, icon, primaryAction, titleAction, tabs, utilities }: PageHeaderProps) { const leadingControls = primaryAction == null ? null : (
    @@ -31,14 +32,17 @@ export function PageHeader({ title, subtitle, icon, primaryAction, tabs, utiliti return (
    -
    - -

    {title}

    +
    +
    + +

    {title}

    +
    + {titleAction}

    {subtitle}

    diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx index d0c5964a4c3..d46fc372e78 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import moment from "moment"; import { useMemo, useState } from "react"; @@ -13,12 +14,15 @@ export default function AgentTracesPage({ isActive = true, readOnly = false, canMintTracingKey = false, + onDemo, }: { accessToken: string; isActive?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; }) { + const demo = useLensDemo(); const [rangeHours, setRangeHours] = useState(DEFAULT_RANGE_HOURS); const [live, setLive] = useState(true); const [anchor, setAnchor] = useState(() => moment()); @@ -36,16 +40,17 @@ export default function AgentTracesPage({ }; return ( -
    +
    diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx new file mode 100644 index 00000000000..816bf5f9267 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx @@ -0,0 +1,509 @@ +import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ApiError } from "@/lib/http/client"; + +import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import traceList from "./__fixtures__/trace_list.json"; +import AgentTracesPage from "./AgentTracesPage"; +import { AgentTracesSection, type TimeControls } from "./AgentTracesSection"; +import type { TracePage, TraceSummary } from "./traceTypes"; + +vi.mock("../../networking", () => ({ + apiClient: { get: vi.fn(), post: vi.fn() }, + agentTraceListCall: vi.fn(), + sendOtlpTraceCall: vi.fn(), + agentTraceCall: vi.fn(), + agentTraceSpanCall: vi.fn(), + getProxyBaseUrl: () => "http://localhost:4000", +})); + +vi.mock("./TraceDrawer", () => ({ + RunView: ({ traceId, onBack }: { traceId: string; onBack: () => void }) => ( +
    + run {traceId} + +
    + ), +})); + +import { agentTraceListCall, apiClient } from "../../networking"; + +const runs = (traceList as TracePage).data as TraceSummary[]; + +const renderSection = () => + renderWithProviders( + , + ); + +// A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone. +const renderWindowed = (timeControls?: TimeControls) => + renderWithProviders( + , + ); + +const bucketRunCounts = () => + screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-runs"))); + +describe("AgentTracesSection", () => { + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + beforeEach(() => { + vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({ + left: 0, + width: 600, + top: 0, + height: 56, + right: 600, + bottom: 56, + x: 0, + y: 0, + toJSON: () => ({}), + } as DOMRect); + testQueryClient.clear(); + vi.mocked(agentTraceListCall).mockReset(); + vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); + }); + + it.each([ + [401, "Your session is no longer valid. Sign out and sign in again."], + [403, "Your account does not have access to these traces."], + ])("stops live polling after HTTP %s and explains how to recover", async (status, message) => { + vi.useFakeTimers(); + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Private token details", Number(status), {})); + renderWithProviders( + , + ); + await act(async () => { + await vi.advanceTimersByTimeAsync(60_000); + }); + expect(screen.getByText(`Could not load runs: ${message}`)).toBeVisible(); + expect(screen.queryByText(/Private token details/)).not.toBeInTheDocument(); + expect(agentTraceListCall).toHaveBeenCalledOnce(); + expect(screen.getByTestId("runs-footer")).toHaveTextContent("Update failed"); + }); + + it("renders the setup snippet when the proxy answers 501", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue( + new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }), + ); + renderSection(); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Tracing is not enabled"); + expect(screen.getByRole("button", { name: "Preview sample" })).toBeVisible(); + expect(card).toHaveTextContent("type: clickhouse"); + expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); + expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); + expect(card).not.toHaveTextContent(/langsmith/i); + expect(card).toHaveTextContent("ClickHouse and proxy setup"); + expect(card).toHaveTextContent("Ask your proxy administrator"); + }); + + it("shows the waiting guide when tracing is on but no runs have arrived", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + renderSection(); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Connect your agent"); + expect(card).toHaveTextContent("Waiting for your first trace"); + expect(screen.getByRole("button", { name: "Preview sample" })).toBeVisible(); + expect(card).not.toHaveTextContent("store: clickhouse"); + }); + + it("keeps the trace list available when traces exist outside the current time window", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + vi.mocked(apiClient.get).mockResolvedValue(traceList); + renderSection(); + expect(await screen.findByText("No runs match these filters.")).toBeVisible(); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + expect(apiClient.get).toHaveBeenCalledWith("/v1/traces", { accessToken: "sk-test", query: { start_ms: 0 } }); + }); + + it("checks proxy readiness, waits for an agent, and confirms receipt using actual query results", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); + renderSection(); + const checkSetup = await screen.findByRole("button", { name: "Check setup" }); + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + fireEvent.click(checkSetup); + expect(await screen.findByRole("heading", { name: "Connect your agent" })).toBeVisible(); + expect(screen.getByText("Waiting for your first trace")).toBeVisible(); + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + fireEvent.click(screen.getByRole("button", { name: "Check for traces" })); + expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length); + expect(screen.getByText("Traces received. Select a run to inspect it.")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + }); + + it("keeps setup visible while checking and explains when tracing is still disabled", async () => { + const failure = new ApiError("Tracing is not enabled", 501, {}); + vi.mocked(agentTraceListCall).mockRejectedValue(failure); + renderSection(); + const checkSetup = await screen.findByRole("button", { name: "Check setup" }); + let rejectCheck: (error: Error) => void = () => {}; + vi.mocked(agentTraceListCall).mockImplementationOnce( + () => + new Promise((_, reject) => { + rejectCheck = reject; + }), + ); + fireEvent.click(checkSetup); + expect(await screen.findByRole("button", { name: "Checking…" })).toBeDisabled(); + expect(screen.getByRole("heading", { name: "Enable tracing" })).toBeVisible(); + await act(async () => rejectCheck(failure)); + expect(await screen.findByText(/Tracing is still unavailable/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); + }); + + it("keeps received traces and the open drawer visible during subsequent fetches", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); + renderSection(); + const checkSetup = await screen.findByRole("button", { name: "Check setup" }); + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + fireEvent.click(checkSetup); + const rows = await screen.findAllByTestId("agent-trace-row"); + fireEvent.click(rows[0]); + const drawer = screen.getByRole("complementary", { name: "Trace details" }); + + let finishRefresh: (page: TracePage) => void = () => {}; + vi.mocked(agentTraceListCall).mockImplementationOnce( + () => + new Promise((resolve) => { + finishRefresh = resolve; + }), + ); + await act(async () => { + void testQueryClient.invalidateQueries({ queryKey: ["agentTraces"] }); + }); + expect(await screen.findByText("Updating…")).toBeVisible(); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + await act(async () => finishRefresh(traceList as TracePage)); + }); + + it("separates a failed history check from the empty list and retries that check", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + vi.mocked(apiClient.get).mockRejectedValue(new ApiError("History unavailable", 503, {})); + renderSection(); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not check earlier traces. History unavailable"); + expect(screen.getByText("No runs match these filters.")).toBeVisible(); + expect(screen.queryByText(/Could not load runs/)).not.toBeInTheDocument(); + + vi.mocked(apiClient.get).mockResolvedValue(traceList); + fireEvent.click(screen.getByRole("button", { name: "Retry trace check" })); + await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); + expect(screen.getByText("No runs match these filters.")).toBeVisible(); + expect(agentTraceListCall).toHaveBeenCalledTimes(1); + }); + + it("treats a proxy without the trace routes (404) like tracing being off", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" })); + renderSection(); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Tracing is not enabled"); + expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); + }); + + it("lists every run with its input, counts and failed column", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + + const rows = await screen.findAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(runs.length); + const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans")); + expect(lead).toBeDefined(); + const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement; + expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument(); + expect(screen.getByText(`${runs.length} runs`)).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument(); + expect(within(failed).getByText("—")).toBeInTheDocument(); + }); + + it("shows the spend returned for a run", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + ...(traceList as TracePage), + data: [{ ...runs[0], spend: 0.025 }], + }); + renderSection(); + + const row = await screen.findByTestId("agent-trace-row"); + expect(within(row).getByText("$0.03")).toBeInTheDocument(); + }); + + it("filters by input text and by trace id", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + await screen.findAllByTestId("agent-trace-row"); + + const search = screen.getByLabelText("Search runs"); + fireEvent.change(search, { target: { value: "acme-404" } }); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); + + const lead = runs.find((r) => r.name === "research_lead") as TraceSummary; + fireEvent.change(search, { target: { value: lead.trace_id.slice(0, 10) } }); + const rows = screen.getAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans"); + }); + + it("uses recorded agent names for the column and filter even when services are shared", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + ...(traceList as TracePage), + data: [ + ...runs.slice(1).map((run) => ({ ...run, service: "shared-app", agent_names: ["research-agent"] })), + { ...runs[0], service: "shared-app", agent_names: ["billing-agent", "review-agent"] }, + ], + }); + const user = userEvent.setup(); + renderSection(); + await screen.findAllByTestId("agent-trace-row"); + + expect(screen.getByRole("columnheader", { name: "Agent" })).toBeInTheDocument(); + const agentFilter = screen.getByRole("combobox", { name: "Filter by agent" }); + expect(agentFilter).toHaveTextContent("All agents"); + + await chooseSelectOption(user, agentFilter, "billing-agent"); + const rows = screen.getAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("billing-agent"); + expect(rows[0]).not.toHaveTextContent("shared-app"); + + await chooseSelectOption(user, agentFilter, "review-agent"); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); + + await chooseSelectOption(user, agentFilter, "All agents"); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + }); + + it("shows each run's agent name with the logo of the SDK that produced it", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + ...(traceList as TracePage), + data: [ + { ...runs[0], agent_names: ["research-bot"], frameworks: ["claude-agent-sdk", "claude-code"] }, + { ...runs[1], agent_names: [], frameworks: ["claude-code"] }, + { ...runs[2], frameworks: [] }, + ], + }); + renderSection(); + const [sdkRun, cliRun, plainRun] = await screen.findAllByTestId("agent-trace-row"); + const agentCell = (row: HTMLElement) => within(row).getAllByRole("cell")[1]; + + expect(agentCell(sdkRun)).toHaveTextContent(/^research-bot$/); + expect(agentCell(sdkRun)).toHaveAttribute("title", "research-bot · Claude Agent SDK"); + expect(within(sdkRun).getByRole("img", { name: "Claude Agent SDK logo", hidden: true })).toHaveAttribute( + "src", + expect.stringContaining("anthropic.svg"), + ); + expect(agentCell(cliRun)).toHaveTextContent(/^Claude Code$/); + expect(within(plainRun).queryByRole("img", { hidden: true })).not.toBeInTheDocument(); + expect(agentCell(plainRun)).toHaveTextContent((runs[2].agent_names ?? [runs[2].service]).join(", ")); + }); + + it("opens a run in a side drawer over the list and swaps runs without closing it", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + const rows = await screen.findAllByTestId("agent-trace-row"); + + fireEvent.click(rows[0]); + const drawer = screen.getByRole("complementary", { name: "Trace details" }); + expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); + expect(screen.getByTestId("runs-table")).toBeInTheDocument(); + expect(rows[0]).toHaveAttribute("aria-selected", "true"); + + fireEvent.click(rows[1]); + expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); + expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); + expect(rows[1]).toHaveAttribute("aria-selected", "true"); + expect(rows[0]).toHaveAttribute("aria-selected", "false"); + }); + + it("closes the drawer when the open row is clicked again or Escape is pressed", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + const rows = await screen.findAllByTestId("agent-trace-row"); + + fireEvent.click(rows[0]); + fireEvent.click(rows[0]); + expect(rows[0]).toHaveAttribute("aria-selected", "false"); + + fireEvent.click(rows[1]); + fireEvent.keyDown(window, { key: "Escape" }); + expect(rows[1]).toHaveAttribute("aria-selected", "false"); + }); + + it("moves to the next and previous run with j / k and the header arrows", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + const rows = await screen.findAllByTestId("agent-trace-row"); + + fireEvent.click(rows[0]); + fireEvent.keyDown(window, { key: "j" }); + expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); + fireEvent.keyDown(window, { key: "k" }); + expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); + expect(screen.getByRole("button", { name: "Previous trace (K)" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Next trace (J)" })); + expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); + }); + + it("plots every loaded run on the timeline", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed(); + await screen.findAllByTestId("agent-trace-row"); + + expect(screen.getByTestId("traces-timeline")).toBeInTheDocument(); + const counts = bucketRunCounts(); + expect(counts).toHaveLength(60); + expect(counts.reduce((a, b) => a + b, 0)).toBe(runs.length); + }); + + it("zooms by dragging, resizes and pans the bracket, and clears with Esc", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed(); + await screen.findAllByTestId("agent-trace-row"); + const area = screen.getByTestId("timeline-area"); + const x = (bucket: number) => bucket * 10 + 5; + const drag = (target: HTMLElement, from: number, to: number) => { + fireEvent.pointerDown(target, { clientX: x(from), pointerId: 1 }); + fireEvent.pointerMove(area, { clientX: x(to), pointerId: 1 }); + fireEvent.pointerUp(area, { clientX: x(to), pointerId: 1 }); + }; + const rowCount = () => screen.queryAllByTestId("agent-trace-row").length; + const withRuns = bucketRunCounts().flatMap((count, i) => (count > 0 ? [i] : [])); + const first = withRuns[0]; + // The pan below moves a [0, first] bracket to the far right; it must end up clear of every run. + expect(first).toBeGreaterThan(1); + expect(first).toBeLessThan(30); + + drag(area, 0, 1); + expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); + expect(rowCount()).toBe(0); + + drag(screen.getByTestId("timeline-handle-hi"), 1, first); + expect(rowCount()).toBeGreaterThan(0); + + drag(screen.getByTestId("timeline-selection"), 1, 1 - first); + expect(rowCount()).toBeGreaterThan(0); + drag(screen.getByTestId("timeline-selection"), 0, 59); + expect(rowCount()).toBe(0); + + fireEvent.keyDown(screen.getByTestId("traces-timeline"), { key: "Escape" }); + expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); + expect(rowCount()).toBe(runs.length); + }); + + it("clears timeline zoom when refreshed", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed({ rangeHours: 24, onRangeHoursChange: () => {}, onLiveChange: () => {} }); + await screen.findAllByTestId("agent-trace-row"); + const area = screen.getByTestId("timeline-area"); + const x = (bucket: number) => bucket * 10 + 5; + + fireEvent.pointerDown(area, { clientX: x(0), pointerId: 1 }); + fireEvent.pointerMove(area, { clientX: x(1), pointerId: 1 }); + fireEvent.pointerUp(area, { clientX: x(1), pointerId: 1 }); + expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); + expect(screen.queryAllByTestId("agent-trace-row")).toHaveLength(0); + + fireEvent.click(screen.getByRole("button", { name: "Refresh" })); + + expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + }); +}); + +describe("AgentTracesPage", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(agentTraceListCall).mockReset(); + vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); + }); + + it("shows the actual range, switches presets from the popover, and toggles Live", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const trigger = screen.getByRole("button", { name: "Time range" }); + expect(trigger).toHaveTextContent(/ to /); + expect(screen.getByTestId("traces-timeline")).toHaveTextContent("Total 1d"); + + fireEvent.click(trigger); + fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last 7 days" })); + expect(await screen.findByText("Total 7d")).toBeInTheDocument(); + const last = vi.mocked(agentTraceListCall).mock.calls.at(-1)?.[0]; + expect((last?.endMs ?? 0) - (last?.startMs ?? 0)).toBeGreaterThanOrEqual(7 * 24 * 3600 * 1000 - 60_000); + + const live = screen.getByRole("button", { name: "Live" }); + expect(live).toHaveAttribute("aria-pressed", "true"); + fireEvent.click(live); + expect(live).toHaveAttribute("aria-pressed", "false"); + expect(screen.getByRole("button", { name: "Refresh" })).toBeEnabled(); + }); + + it("refreshes the trace list", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const callsBeforeRefresh = vi.mocked(agentTraceListCall).mock.calls.length; + fireEvent.click(screen.getByRole("button", { name: "Refresh" })); + + await waitFor(() => expect(vi.mocked(agentTraceListCall).mock.calls.length).toBeGreaterThan(callsBeforeRefresh)); + }); + + it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + fireEvent.click(screen.getByRole("button", { name: "Time range" })); + fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last hour" })); + + expect(await screen.findByText("No runs match these filters.")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Time range" })).toBeInTheDocument(); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + }); + + it("asks the proxy for the last 24 hours by default", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const { startMs, endMs } = vi.mocked(agentTraceListCall).mock.calls[0][0]; + expect(endMs - startMs).toBeGreaterThanOrEqual(24 * 3600 * 1000 - 60_000); + expect(endMs - startMs).toBeLessThan(24 * 3600 * 1000 + 120_000); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx index e90544ca0a6..062c6d30da6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -1,330 +1,11 @@ -import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { ApiError } from "@/lib/http/client"; - -import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import { describe, expect, it } from "vitest"; +import { filterRuns } from "./AgentTracesSection"; import traceList from "./__fixtures__/trace_list.json"; -import AgentTracesPage from "./AgentTracesPage"; -import { AgentTracesSection, filterRuns, type TimeControls } from "./AgentTracesSection"; import type { TracePage, TraceSummary } from "./traceTypes"; -vi.mock("../../networking", () => ({ - apiClient: { get: vi.fn(), post: vi.fn() }, - agentTraceListCall: vi.fn(), - sendOtlpTraceCall: vi.fn(), - agentTraceCall: vi.fn(), - agentTraceSpanCall: vi.fn(), - getProxyBaseUrl: () => "http://localhost:4000", -})); - -vi.mock("./TraceDrawer", () => ({ - RunView: ({ traceId, onBack }: { traceId: string; onBack: () => void }) => ( -
    - run {traceId} - -
    - ), -})); - -import { agentTraceListCall, apiClient } from "../../networking"; - const runs = (traceList as TracePage).data as TraceSummary[]; -const renderSection = () => - renderWithProviders( - , - ); - -// A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone. -const renderWindowed = (timeControls?: TimeControls) => - renderWithProviders( - , - ); - -const bucketRunCounts = () => - screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-runs"))); - -describe("AgentTracesSection", () => { - afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - beforeEach(() => { - vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({ - left: 0, - width: 600, - top: 0, - height: 56, - right: 600, - bottom: 56, - x: 0, - y: 0, - toJSON: () => ({}), - } as DOMRect); - testQueryClient.clear(); - vi.mocked(agentTraceListCall).mockReset(); - vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); - }); - - it.each([ - [401, "Your session is no longer valid. Sign out and sign in again."], - [403, "Your account does not have access to these traces."], - ])("stops live polling after HTTP %s and explains how to recover", async (status, message) => { - vi.useFakeTimers(); - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Private token details", Number(status), {})); - renderWithProviders( - , - ); - await act(async () => { - await vi.advanceTimersByTimeAsync(60_000); - }); - expect(screen.getByText(`Could not load runs: ${message}`)).toBeVisible(); - expect(screen.queryByText(/Private token details/)).not.toBeInTheDocument(); - expect(agentTraceListCall).toHaveBeenCalledOnce(); - expect(screen.getByTestId("runs-footer")).toHaveTextContent("Update failed"); - }); - - it("renders the setup snippet when the proxy answers 501", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue( - new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }), - ); - renderSection(); - - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Tracing is not enabled"); - expect(card).toHaveTextContent("type: clickhouse"); - expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); - expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); - expect(card).not.toHaveTextContent(/langsmith/i); - expect(card).toHaveTextContent("ClickHouse and proxy setup"); - expect(card).toHaveTextContent("Ask your proxy administrator"); - }); - - it("shows the waiting guide when tracing is on but no runs have arrived", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - renderSection(); - - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Connect your agent"); - expect(card).toHaveTextContent("Waiting for your first trace"); - expect(card).not.toHaveTextContent("store: clickhouse"); - }); - - it("keeps the trace list available when traces exist outside the current time window", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - vi.mocked(apiClient.get).mockResolvedValue(traceList); - renderSection(); - expect(await screen.findByText("No runs match these filters.")).toBeVisible(); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - expect(apiClient.get).toHaveBeenCalledWith("/v1/traces", { accessToken: "sk-test", query: { start_ms: 0 } }); - }); - - it("checks proxy readiness, waits for an agent, and confirms receipt using actual query results", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); - renderSection(); - const checkSetup = await screen.findByRole("button", { name: "Check setup" }); - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - fireEvent.click(checkSetup); - expect(await screen.findByRole("heading", { name: "Connect your agent" })).toBeVisible(); - expect(screen.getByText("Waiting for your first trace")).toBeVisible(); - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - fireEvent.click(screen.getByRole("button", { name: "Check for traces" })); - expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length); - expect(screen.getByText("Traces received. Select a run to inspect it.")).toBeVisible(); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - }); - - it("keeps setup visible while checking and explains when tracing is still disabled", async () => { - const failure = new ApiError("Tracing is not enabled", 501, {}); - vi.mocked(agentTraceListCall).mockRejectedValue(failure); - renderSection(); - const checkSetup = await screen.findByRole("button", { name: "Check setup" }); - let rejectCheck: (error: Error) => void = () => {}; - vi.mocked(agentTraceListCall).mockImplementationOnce( - () => - new Promise((_, reject) => { - rejectCheck = reject; - }), - ); - fireEvent.click(checkSetup); - expect(await screen.findByRole("button", { name: "Checking…" })).toBeDisabled(); - expect(screen.getByRole("heading", { name: "Enable tracing" })).toBeVisible(); - await act(async () => rejectCheck(failure)); - expect(await screen.findByText(/Tracing is still unavailable/)).toBeVisible(); - expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); - }); - - it("keeps received traces and the open drawer visible during subsequent fetches", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); - renderSection(); - const checkSetup = await screen.findByRole("button", { name: "Check setup" }); - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - fireEvent.click(checkSetup); - const rows = await screen.findAllByTestId("agent-trace-row"); - fireEvent.click(rows[0]); - const drawer = screen.getByRole("complementary", { name: "Trace details" }); - - let finishRefresh: (page: TracePage) => void = () => {}; - vi.mocked(agentTraceListCall).mockImplementationOnce( - () => - new Promise((resolve) => { - finishRefresh = resolve; - }), - ); - await act(async () => { - void testQueryClient.invalidateQueries({ queryKey: ["agentTraces"] }); - }); - expect(await screen.findByText("Updating…")).toBeVisible(); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); - expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - await act(async () => finishRefresh(traceList as TracePage)); - }); - - it("separates a failed history check from the empty list and retries that check", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - vi.mocked(apiClient.get).mockRejectedValue(new ApiError("History unavailable", 503, {})); - renderSection(); - expect(await screen.findByRole("alert")).toHaveTextContent("Could not check earlier traces. History unavailable"); - expect(screen.getByText("No runs match these filters.")).toBeVisible(); - expect(screen.queryByText(/Could not load runs/)).not.toBeInTheDocument(); - - vi.mocked(apiClient.get).mockResolvedValue(traceList); - fireEvent.click(screen.getByRole("button", { name: "Retry trace check" })); - await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); - expect(screen.getByText("No runs match these filters.")).toBeVisible(); - expect(agentTraceListCall).toHaveBeenCalledTimes(1); - }); - - it("treats a proxy without the trace routes (404) like tracing being off", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" })); - renderSection(); - - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Tracing is not enabled"); - expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); - }); - - it("lists every run with its input, counts and failed column", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - - const rows = await screen.findAllByTestId("agent-trace-row"); - expect(rows).toHaveLength(runs.length); - const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans")); - expect(lead).toBeDefined(); - const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement; - expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument(); - expect(screen.getByText(`${runs.length} runs`)).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument(); - expect(within(failed).getByText("—")).toBeInTheDocument(); - }); - - it("shows the spend returned for a run", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ - ...(traceList as TracePage), - data: [{ ...runs[0], spend: 0.025 }], - }); - renderSection(); - - const row = await screen.findByTestId("agent-trace-row"); - expect(within(row).getByText("$0.03")).toBeInTheDocument(); - }); - - it("filters by input text and by trace id", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - await screen.findAllByTestId("agent-trace-row"); - - const search = screen.getByLabelText("Search runs"); - fireEvent.change(search, { target: { value: "acme-404" } }); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); - - const lead = runs.find((r) => r.name === "research_lead") as TraceSummary; - fireEvent.change(search, { target: { value: lead.trace_id.slice(0, 10) } }); - const rows = screen.getAllByTestId("agent-trace-row"); - expect(rows).toHaveLength(1); - expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans"); - }); - - it("uses recorded agent names for the column and filter even when services are shared", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ - ...(traceList as TracePage), - data: [ - ...runs.slice(1).map((run) => ({ ...run, service: "shared-app", agent_names: ["research-agent"] })), - { ...runs[0], service: "shared-app", agent_names: ["billing-agent", "review-agent"] }, - ], - }); - const user = userEvent.setup(); - renderSection(); - await screen.findAllByTestId("agent-trace-row"); - - expect(screen.getByRole("columnheader", { name: "Agent" })).toBeInTheDocument(); - const agentFilter = screen.getByRole("combobox", { name: "Filter by agent" }); - expect(agentFilter).toHaveTextContent("All agents"); - - await chooseSelectOption(user, agentFilter, "billing-agent"); - const rows = screen.getAllByTestId("agent-trace-row"); - expect(rows).toHaveLength(1); - expect(rows[0]).toHaveTextContent("billing-agent"); - expect(rows[0]).not.toHaveTextContent("shared-app"); - - await chooseSelectOption(user, agentFilter, "review-agent"); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); - - await chooseSelectOption(user, agentFilter, "All agents"); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); - }); - - it("shows each run's agent name with the logo of the SDK that produced it", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ - ...(traceList as TracePage), - data: [ - { ...runs[0], agent_names: ["research-bot"], frameworks: ["claude-agent-sdk", "claude-code"] }, - { ...runs[1], agent_names: [], frameworks: ["claude-code"] }, - { ...runs[2], frameworks: [] }, - ], - }); - renderSection(); - const [sdkRun, cliRun, plainRun] = await screen.findAllByTestId("agent-trace-row"); - const agentCell = (row: HTMLElement) => within(row).getAllByRole("cell")[1]; - - expect(agentCell(sdkRun)).toHaveTextContent(/^research-bot$/); - expect(agentCell(sdkRun)).toHaveAttribute("title", "research-bot · Claude Agent SDK"); - expect(within(sdkRun).getByRole("img", { name: "Claude Agent SDK logo", hidden: true })).toHaveAttribute( - "src", - expect.stringContaining("anthropic.svg"), - ); - expect(agentCell(cliRun)).toHaveTextContent(/^Claude Code$/); - expect(within(plainRun).queryByRole("img", { hidden: true })).not.toBeInTheDocument(); - expect(agentCell(plainRun)).toHaveTextContent((runs[2].agent_names ?? [runs[2].service]).join(", ")); - }); - +describe("filterRuns", () => { it("status filter 'Failed' keeps only runs with errors", () => { const failed = filterRuns(runs, "", "all", "error"); expect(failed.length).toBeGreaterThan(0); @@ -333,181 +14,4 @@ describe("AgentTracesSection", () => { expect(ok.every((r) => r.error_count === 0)).toBe(true); expect(failed.length + ok.length).toBe(runs.length); }); - - it("opens a run in a side drawer over the list and swaps runs without closing it", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - const rows = await screen.findAllByTestId("agent-trace-row"); - - fireEvent.click(rows[0]); - const drawer = screen.getByRole("complementary", { name: "Trace details" }); - expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); - expect(screen.getByTestId("runs-table")).toBeInTheDocument(); - expect(rows[0]).toHaveAttribute("aria-selected", "true"); - - fireEvent.click(rows[1]); - expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); - expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); - expect(rows[1]).toHaveAttribute("aria-selected", "true"); - expect(rows[0]).toHaveAttribute("aria-selected", "false"); - }); - - it("closes the drawer when the open row is clicked again or Escape is pressed", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - const rows = await screen.findAllByTestId("agent-trace-row"); - - fireEvent.click(rows[0]); - fireEvent.click(rows[0]); - expect(rows[0]).toHaveAttribute("aria-selected", "false"); - - fireEvent.click(rows[1]); - fireEvent.keyDown(window, { key: "Escape" }); - expect(rows[1]).toHaveAttribute("aria-selected", "false"); - }); - - it("moves to the next and previous run with j / k and the header arrows", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - const rows = await screen.findAllByTestId("agent-trace-row"); - - fireEvent.click(rows[0]); - fireEvent.keyDown(window, { key: "j" }); - expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); - fireEvent.keyDown(window, { key: "k" }); - expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); - expect(screen.getByRole("button", { name: "Previous trace (K)" })).toBeDisabled(); - fireEvent.click(screen.getByRole("button", { name: "Next trace (J)" })); - expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); - }); - - it("plots every loaded run on the timeline", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWindowed(); - await screen.findAllByTestId("agent-trace-row"); - - expect(screen.getByTestId("traces-timeline")).toBeInTheDocument(); - const counts = bucketRunCounts(); - expect(counts).toHaveLength(60); - expect(counts.reduce((a, b) => a + b, 0)).toBe(runs.length); - }); - - it("zooms by dragging, resizes and pans the bracket, and clears with Esc", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWindowed(); - await screen.findAllByTestId("agent-trace-row"); - const area = screen.getByTestId("timeline-area"); - const x = (bucket: number) => bucket * 10 + 5; - const drag = (target: HTMLElement, from: number, to: number) => { - fireEvent.pointerDown(target, { clientX: x(from), pointerId: 1 }); - fireEvent.pointerMove(area, { clientX: x(to), pointerId: 1 }); - fireEvent.pointerUp(area, { clientX: x(to), pointerId: 1 }); - }; - const rowCount = () => screen.queryAllByTestId("agent-trace-row").length; - const withRuns = bucketRunCounts().flatMap((count, i) => (count > 0 ? [i] : [])); - const first = withRuns[0]; - // The pan below moves a [0, first] bracket to the far right; it must end up clear of every run. - expect(first).toBeGreaterThan(1); - expect(first).toBeLessThan(30); - - drag(area, 0, 1); - expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); - expect(rowCount()).toBe(0); - - drag(screen.getByTestId("timeline-handle-hi"), 1, first); - expect(rowCount()).toBeGreaterThan(0); - - drag(screen.getByTestId("timeline-selection"), 1, 1 - first); - expect(rowCount()).toBeGreaterThan(0); - drag(screen.getByTestId("timeline-selection"), 0, 59); - expect(rowCount()).toBe(0); - - fireEvent.keyDown(screen.getByTestId("traces-timeline"), { key: "Escape" }); - expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); - expect(rowCount()).toBe(runs.length); - }); - - it("clears timeline zoom when refreshed", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWindowed({ rangeHours: 24, onRangeHoursChange: () => {}, onLiveChange: () => {} }); - await screen.findAllByTestId("agent-trace-row"); - const area = screen.getByTestId("timeline-area"); - const x = (bucket: number) => bucket * 10 + 5; - - fireEvent.pointerDown(area, { clientX: x(0), pointerId: 1 }); - fireEvent.pointerMove(area, { clientX: x(1), pointerId: 1 }); - fireEvent.pointerUp(area, { clientX: x(1), pointerId: 1 }); - expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); - expect(screen.queryAllByTestId("agent-trace-row")).toHaveLength(0); - - fireEvent.click(screen.getByRole("button", { name: "Refresh" })); - - expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); - }); -}); - -describe("AgentTracesPage", () => { - beforeEach(() => { - testQueryClient.clear(); - vi.mocked(agentTraceListCall).mockReset(); - vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); - }); - - it("shows the actual range, switches presets from the popover, and toggles Live", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - const trigger = screen.getByRole("button", { name: "Time range" }); - expect(trigger).toHaveTextContent(/ to /); - expect(screen.getByTestId("traces-timeline")).toHaveTextContent("Total 1d"); - - fireEvent.click(trigger); - fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last 7 days" })); - expect(await screen.findByText("Total 7d")).toBeInTheDocument(); - const last = vi.mocked(agentTraceListCall).mock.calls.at(-1)?.[0]; - expect((last?.endMs ?? 0) - (last?.startMs ?? 0)).toBeGreaterThanOrEqual(7 * 24 * 3600 * 1000 - 60_000); - - const live = screen.getByRole("button", { name: "Live" }); - expect(live).toHaveAttribute("aria-pressed", "true"); - fireEvent.click(live); - expect(live).toHaveAttribute("aria-pressed", "false"); - expect(screen.getByRole("button", { name: "Refresh" })).toBeEnabled(); - }); - - it("refreshes the trace list", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - const callsBeforeRefresh = vi.mocked(agentTraceListCall).mock.calls.length; - fireEvent.click(screen.getByRole("button", { name: "Refresh" })); - - await waitFor(() => expect(vi.mocked(agentTraceListCall).mock.calls.length).toBeGreaterThan(callsBeforeRefresh)); - }); - - it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - fireEvent.click(screen.getByRole("button", { name: "Time range" })); - fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last hour" })); - - expect(await screen.findByText("No runs match these filters.")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Time range" })).toBeInTheDocument(); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - }); - - it("asks the proxy for the last 24 hours by default", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - const { startMs, endMs } = vi.mocked(agentTraceListCall).mock.calls[0][0]; - expect(endMs - startMs).toBeGreaterThanOrEqual(24 * 3600 * 1000 - 60_000); - expect(endMs - startMs).toBeLessThan(24 * 3600 * 1000 + 120_000); - }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 1b25ca19051..45f1b223088 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import moment from "moment"; import { useMemo, useState } from "react"; @@ -61,6 +62,7 @@ interface AgentTracesSectionProps { onRunOpenChange?: (open: boolean) => void; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; } function useTracingSetup(traces: AgentTracesResult, isActive: boolean, rangeChanged: boolean) { @@ -102,7 +104,9 @@ export function AgentTracesSection({ onRunOpenChange, readOnly = false, canMintTracingKey = false, + onDemo, }: AgentTracesSectionProps) { + const demo = useLensDemo(); const [openTrace, setOpenTrace] = useState(null); const [query, setQuery] = useState(""); const [agent, setAgent] = useState(ALL_AGENTS); @@ -160,10 +164,11 @@ export function AgentTracesSection({ checking: traces.isFetching, }; - if (setup.disabledDetail != null) return ; + if (setup.disabledDetail != null) + return ; // Onboarding only on the first, default view; an empty range the user picked keeps its controls. if (checkHistory && !history.error && history.data === false) - return ; + return ; if (showSetup) { return (
    @@ -185,11 +190,7 @@ export function AgentTracesSection({ return (
    {checkHistory && } - {setup.received && ( -

    - Traces received. Select a run to inspect it. -

    - )} + - + {!demo && ( + + )} {timeControls && ( changeRange(hours, timeControls.onRangeHoursChange)} live={isLiveTail} + showLive={!demo} onLiveChange={timeControls.onLiveChange} onRefresh={() => { setZoom(null); @@ -240,6 +244,16 @@ export function AgentTracesSection({ ); } +function TracesReceived({ received }: { received: boolean }) { + const demo = useLensDemo(); + if (!received || demo) return null; + return ( +

    + Traces received. Select a run to inspect it. +

    + ); +} + function RunsFooter({ count, zoom, diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx index 86ec3007f72..da2175cf0ff 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; import { useState } from "react"; @@ -29,9 +30,15 @@ const errorReason = (headline: string): string => /^([A-Za-z_][\w.]*)\(/.exec(he /** Shared lazy fetch of one span's full input / output / attributes. */ export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null, traceRef?: string) { + const demo = useLensDemo(); const queryOptions: UseQueryOptions = { queryKey: ["agentTraceSpan", traceId, traceRef, spanId, accessToken], - queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string, traceRef), + queryFn: () => + demo + ? demo.client.get( + `/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId as string)}`, + ) + : agentTraceSpanCall(accessToken, traceId, spanId as string, traceRef), enabled: spanId !== null, staleTime: Infinity, }; @@ -146,11 +153,17 @@ interface DetailContentProps { } function DiagnosticContent({ accessToken, traceId, traceRef, span }: DetailContentProps) { + const demo = useLensDemo(); const [opened, setOpened] = useState(false); const [cursor, setCursor] = useState(null); const queryOptions: UseQueryOptions = { queryKey: ["agentTraceSpanError", traceId, traceRef, span.span_id, accessToken, cursor], - queryFn: () => agentTraceSpanErrorCall(accessToken, traceId, span.span_id, { traceRef, cursor }), + queryFn: () => + demo + ? demo.client.get( + `/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(span.span_id)}/error`, + ) + : agentTraceSpanErrorCall(accessToken, traceId, span.span_id, { traceRef, cursor }), enabled: opened, staleTime: Infinity, gcTime: 0, diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx index 985de97cdc9..aa777b9861e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { PanelRightClose } from "lucide-react"; import { useState } from "react"; @@ -104,6 +105,7 @@ function SpanPane({ onClose: () => void; }) { const [tab, setTab] = useState("content"); + const demo = useLensDemo(); const traceId = trace.summary.trace_id; const detailQuery = useSpanDetail( accessToken, @@ -162,9 +164,13 @@ function SpanPane({
    @@ -186,6 +192,7 @@ function GroupMetric({ label, value }: { label: string; value: string }) { /** ×N group: rollup of every invocation plus the first failure's message. */ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; onClose: () => void }) { + const demo = useLensDemo(); const tokens = row.members.reduce((sum, m) => sum + m.input_tokens + m.output_tokens, 0); const firstFailure = row.members.find((m) => m.status === "error" && m.error); return ( @@ -222,7 +229,11 @@ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; o
    diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx index 03d950a7725..c9e25c2e033 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx @@ -34,6 +34,7 @@ interface TimeRangeControlsProps { rangeHours: number; onRangeHoursChange: (hours: number) => void; live: boolean; + showLive?: boolean; onLiveChange: (live: boolean) => void; onRefresh: () => void; refreshing: boolean; @@ -45,6 +46,7 @@ export function TimeRangeControls({ rangeHours, onRangeHoursChange, live, + showLive = true, onLiveChange, onRefresh, refreshing, @@ -84,18 +86,20 @@ export function TimeRangeControls({ - + {showLive && ( + + )}
    ); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx index a991c771a56..435ea51e8d2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { useQuery } from "@tanstack/react-query"; import { ArrowLeft, Check, Copy } from "lucide-react"; @@ -74,6 +75,7 @@ const toggle = (set: ReadonlySet, id: string): Set => { }; function CopyForAgent({ traceId, traceRef }: { traceId: string; traceRef?: string }) { + const demo = useLensDemo(); const [copied, setCopied] = useState(false); useEffect(() => { if (!copied) return; @@ -86,11 +88,16 @@ function CopyForAgent({ traceId, traceRef }: { traceId: string; traceRef?: strin size="xs" className="h-7 shrink-0 gap-1.5 rounded-md text-[12px] shadow-none" onClick={async () => - setCopied(await copyToClipboard(agentHandoffText(traceId, null, traceRef), "Command copied")) + setCopied( + await copyToClipboard( + demo ? demo.copyTrace(traceId) : agentHandoffText(traceId, null, traceRef), + demo ? "Trace copied" : "Command copied", + ), + ) } > {copied ? : } - {copied ? "Command copied" : "Copy for agent"} + {copied ? "Copied" : "Copy for agent"} ); } @@ -277,9 +284,13 @@ interface RunViewProps { /** One agent run: header with totals and "Copy for agent", span tree on the left, span details on the right. */ export function RunView({ traceId, traceRef, initialSpanId, accessToken, onBack, embedded = false }: RunViewProps) { + const demo = useLensDemo(); const traceQuery = useQuery({ queryKey: ["agentTrace", traceId, traceRef, accessToken], - queryFn: () => agentTraceCall(accessToken, traceId, traceRef), + queryFn: () => + demo + ? demo.client.get(`/v1/traces/${encodeURIComponent(traceId)}`) + : agentTraceCall(accessToken, traceId, traceRef), staleTime: 30_000, }); const trace = traceQuery.data; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx deleted file mode 100644 index c4fd04602ac..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx +++ /dev/null @@ -1,70 +0,0 @@ -"use client"; - -import { cn } from "@/lib/cva.config"; - -import previewTrace from "./previewTrace.json"; -import { SpanIcon } from "./SpanIcon"; -import type { SpanType } from "./traceTypes"; -import { fmtMs } from "./traceUtils"; - -interface PreviewRow { - id: string; - name: string; - type: SpanType; - model: string | null; - depth: number; - start_offset_ms: number; - duration_ms: number; - error: boolean; -} - -const preview = previewTrace as { - name: string; - input_preview: string; - span_count: number; - duration_ms: number; - rows: PreviewRow[]; -}; - -export function TracePreview() { - const total = preview.duration_ms; - return ( -
    -
    - - {preview.name} - {preview.input_preview} - - {preview.span_count} steps · {fmtMs(total)} - -
    -
    - {preview.rows.map((row) => ( -
    -
    - - {row.name} -
    -
    -
    -
    - - {fmtMs(row.duration_ms)} - -
    - ))} -
    -
    -
    - ); -} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.integration.test.tsx similarity index 63% rename from ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx rename to ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.integration.test.tsx index 65e5e9229ff..e1d128d59e9 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.integration.test.tsx @@ -12,6 +12,7 @@ import { tracingEnvSnippet, TracingSetupCard, } from "./TracingSetupCard"; +import { FRAMEWORKS } from "./tracingSetupGuides"; import type { Trace } from "./traceTypes"; vi.mock("../../networking", () => ({ @@ -31,6 +32,7 @@ const renderCard = ( onCheck?: () => void; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; } = {}, ) => { const onOpenTrace = vi.fn(); @@ -38,6 +40,7 @@ const renderCard = ( vi.clearAllMocks()); describe("TracingSetupCard", () => { - it("shows agent connection guidance and labels the example run as sample data", async () => { + it("offers the interactive demo while waiting for the first trace", async () => { const user = userEvent.setup(); - const { card } = renderCard(); + const onDemo = vi.fn(); + const { card } = renderCard({ onDemo }); expect(screen.getByRole("heading", { name: "Connect your agent" })).toBeVisible(); expect(screen.getByText("Tracing enabled")).toBeVisible(); expect(screen.getByText("Waiting for your first trace")).toBeVisible(); - expect(screen.getByTestId("trace-preview")).toBeVisible(); - expect(card).toHaveTextContent("sample data, not your runs"); + await user.click(screen.getByRole("button", { name: "Preview sample" })); + expect(onDemo).toHaveBeenCalledOnce(); + expect(sendOtlpTraceCall).not.toHaveBeenCalled(); expect(card).not.toHaveTextContent("store: clickhouse"); await user.click(screen.getByText("Set up manually")); - expect(screen.getByText(/^export OTEL_EXPORTER_OTLP_ENDPOINT=http:\/\/proxy.test/)).toBeVisible(); + expect(screen.getByText(/^export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=/)).toBeVisible(); expect(card).not.toHaveTextContent(/langsmith/i); }); - it("lists the OTEL endpoints before the framework picker, each copyable", async () => { + it("keeps connection details visible and copies the full trace endpoint", async () => { const user = userEvent.setup(); - const { card } = renderCard(); - const text = card.textContent ?? ""; - expect(text.indexOf("OpenTelemetry (OTEL) endpoints")).toBeLessThan(text.indexOf("Your agent framework")); + renderCard(); + expect(screen.getByRole("combobox", { name: "Your agent framework" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Copy http://proxy.test/v1/traces" })).toBeVisible(); await user.click(screen.getByRole("button", { name: "Copy http://proxy.test/v1/traces" })); expect(copyToClipboard).toHaveBeenLastCalledWith("http://proxy.test/v1/traces"); }); - it("hides the sample preview once traces are arriving", () => { + it("shows connection guidance for another agent without a demo", () => { renderCard({ connected: true }); expect(screen.getByRole("heading", { name: "Connect another agent" })).toBeVisible(); - expect(screen.queryByTestId("trace-preview")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); }); it("builds the coding agent command for the selected framework and keeps both manual installers", async () => { const user = userEvent.setup(); renderCard(); await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "CrewAI"); - const prompt = codingAgentPrompt("http://proxy.test", { - label: "CrewAI", - packages: "crewai openinference-instrumentation-crewai", - }); - const commandText = () => screen.getByText(/^claude |^codex /).textContent; - expect(commandText()).toBe(codingAgentCommand("Claude Code", prompt)); + const prompt = codingAgentPrompt( + "http://proxy.test", + FRAMEWORKS.find((guide) => guide.id === "crewai")!, + "openai/gpt-6-sol", + ); + expect(screen.getByText(/^claude /)).not.toBeVisible(); + await user.click(screen.getByRole("button", { name: "Copy setup command" })); + expect(copyToClipboard).toHaveBeenLastCalledWith(codingAgentCommand("Claude Code", prompt)); await user.click(screen.getByRole("tab", { name: "Codex" })); - expect(commandText()).toBe(codingAgentCommand("Codex", prompt)); + await user.click(screen.getByRole("button", { name: "Copy setup command" })); + expect(copyToClipboard).toHaveBeenLastCalledWith(codingAgentCommand("Codex", prompt)); + await user.click(screen.getByText("View command")); + expect(screen.getByText(/^codex /)).toBeVisible(); + expect(screen.getByText(/^codex /)).toHaveTextContent(codingAgentCommand("Codex", prompt), { + normalizeWhitespace: false, + }); await user.click(screen.getByText("Set up manually")); - expect(screen.getByText(/pip install -U opentelemetry-distro/)).toHaveTextContent( + expect(screen.getByText(/^pip install opentelemetry-distro/)).toHaveTextContent( "crewai openinference-instrumentation-crewai", ); await user.click(screen.getByRole("tab", { name: "uv" })); - expect(screen.getByText(/uv add opentelemetry-distro/)).toHaveTextContent( + expect(screen.getByText(/^uv add opentelemetry-distro/)).toHaveTextContent( "crewai openinference-instrumentation-crewai", ); }); - it("uses npm and a CommonJS-safe TypeScript entrypoint for the Vercel AI SDK", async () => { + it("uses the selected framework's tracing and agent name without asking for a model", async () => { const user = userEvent.setup(); const { card } = renderCard(); await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "Vercel AI SDK"); await user.click(screen.getByText("Set up manually")); - expect(card).toHaveTextContent("npm install ai @ai-sdk/openai-compatible @vercel/otel"); - expect(card).toHaveTextContent("my_agent.ts"); - expect(card).not.toHaveTextContent("opentelemetry-instrument python"); - const quickstart = screen.getByText(/registerOTel\(\{/).textContent ?? ""; - expect(quickstart).toContain("async function main()"); - expect(quickstart.split("\n").filter((line) => /^(const|let) .*= await /.test(line))).toEqual([]); + expect(screen.queryByRole("combobox", { name: "Model" })).not.toBeInTheDocument(); + expect(card).toHaveTextContent("npm install ai @ai-sdk/otel"); + expect(card).toHaveTextContent('const AGENT_NAME = "research_agent"'); + expect(card).toHaveTextContent("functionId: AGENT_NAME"); + expect(card).toHaveTextContent("Use a model configured on this proxy."); + expect(screen.getByText(/^import \{ createOpenAICompatible/)).toHaveTextContent( + 'const model = litellm("openai/gpt-6-sol")', + ); + expect(card).toHaveTextContent('baseURL: "http://proxy.test/v1"'); + }); + + it("keeps plugin model settings and uses a generated tracing key only for tracing", async () => { + const user = userEvent.setup(); + vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET }); + const { card } = renderCard(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "Hermes"); + await user.click(screen.getByText("Set up manually")); + expect(screen.queryByRole("combobox", { name: "Model" })).not.toBeInTheDocument(); + expect(card).toHaveTextContent("Keep your existing model settings"); + await user.click(screen.getByRole("button", { name: "Generate tracing key" })); + await screen.findByText("Your tracing key"); + expect(card).toHaveTextContent("gen_ai.agent.name: research_agent"); + expect(card).toHaveTextContent("endpoint: http://proxy.test/v1/traces"); + expect(card).toHaveTextContent('Authorization: "Bearer ${LITELLM_TRACING_KEY}"'); + expect(card).not.toHaveTextContent(SECRET); }); it("hides the actions a read-only viewer cannot perform", () => { @@ -121,7 +153,7 @@ describe("TracingSetupCard", () => { expect(screen.queryByRole("button", { name: "Send a test trace" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Generate tracing key" })).not.toBeInTheDocument(); expect(card).toHaveTextContent("ask a proxy admin for one"); - expect(card).toHaveTextContent("OpenTelemetry (OTEL) endpoints"); + expect(card).toHaveTextContent("Connection details"); }); it("offers a scoped tracing key only to callers allowed to set key routes", () => { @@ -135,7 +167,7 @@ describe("TracingSetupCard", () => { const user = userEvent.setup(); vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET }); const { card } = renderCard(); - + await user.click(screen.getByText("Set up manually")); await user.click(screen.getByRole("button", { name: "Generate tracing key" })); expect(await screen.findByText("Your tracing key")).toBeVisible(); @@ -193,24 +225,23 @@ describe("TracingSetupCard", () => { }); describe("setup snippets", () => { - it("uses OTLP HTTP with the proxy base URL and reads the key from the environment", () => { + it("uses the instance trace endpoint and keeps tracing and inference keys separate", () => { const env = tracingEnvSnippet("http://proxy.test"); - expect(env).toContain("OTEL_EXPORTER_OTLP_ENDPOINT=http://proxy.test\n"); - expect(env).toContain("OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"); - expect(env).not.toContain("/v1/traces"); + expect(env).toContain('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://proxy.test/v1/traces"'); + expect(env).toContain('OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf"'); expect(env).not.toContain("export LITELLM_API_KEY="); expect(env).toContain("Bearer $LITELLM_API_KEY"); const withKey = tracingEnvSnippet("http://proxy.test", SECRET); expect(withKey).toContain(`export LITELLM_TRACING_KEY=${SECRET}\n`); - expect(withKey).toContain('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_TRACING_KEY"'); - expect(withKey).not.toContain("LITELLM_API_KEY"); + expect(withKey).toContain('OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer $LITELLM_TRACING_KEY"'); + expect(withKey).not.toContain("export LITELLM_API_KEY="); - const prompt = codingAgentPrompt("http://proxy.test", { label: "LangChain", packages: "langchain" }); - expect(prompt).toContain("base_url=http://proxy.test/v1"); - expect(prompt).toContain("opentelemetry-distro opentelemetry-exporter-otlp-proto-http langchain"); - expect(prompt).toContain("OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"); + const prompt = codingAgentPrompt("http://proxy.test", FRAMEWORKS[0], "openai/gpt-6-sol"); + expect(prompt).toContain('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://proxy.test/v1/traces"'); + expect(prompt).toContain("Keep the existing model configuration"); + expect(prompt).toContain('AGENT_NAME = "research_agent"'); + expect(prompt).toContain("name=AGENT_NAME"); expect(prompt).toContain("Lens > Traces"); - expect(prompt).not.toMatch(/langsmith/i); }); it("builds a shell-safe command for each coding agent", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx index de776da0617..7c6155446ea 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -4,31 +4,23 @@ import { ArrowRight, ArrowUpRight, Check, Copy, KeyRound, Loader2, Send } from " import { useState } from "react"; import { cn } from "@/lib/cva.config"; +import { LensPreviewButton } from "@/components/lens/LensPreviewButton"; import { Button } from "@/components/ui/button"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { copyToClipboard } from "@/utils/dataUtils"; import anthropicLogo from "../../../../public/assets/logos/anthropic.svg"; -import crewaiLogo from "../../../../public/assets/logos/crewai-color.svg"; -import langchainLogo from "../../../../public/assets/logos/langchain.svg"; -import langgraphLogo from "../../../../public/assets/logos/langgraph-color.svg"; -import llamaindexLogo from "../../../../public/assets/logos/llamaindex-color.svg"; -import openaiAgentsLogo from "../../../../public/assets/logos/openai-agents.svg"; import openaiLogo from "../../../../public/assets/logos/openai_small.svg"; import otelLogo from "../../../../public/assets/logos/opentelemetry.svg"; -import pydanticAiLogo from "../../../../public/assets/logos/pydantic-ai-color.svg"; -import vercelLogo from "../../../../public/assets/logos/vercel.svg"; import { agentTraceCall, apiClient, getProxyBaseUrl, sendOtlpTraceCall } from "../../networking"; import { ActiveDot } from "./ActiveDot"; import { sampleTraceExport } from "./sampleTrace"; -import { TracePreview } from "./TracePreview"; +import { FRAMEWORKS, frameworkSnippet, type FrameworkGuide } from "./tracingSetupGuides"; import type { TraceSummary } from "./traceTypes"; const COPIED_RESET_MS = 1500; const DOCS_URL = "https://docs.litellm.ai/docs/proxy/lens"; -const OTEL_BASE_PACKAGES = "opentelemetry-distro opentelemetry-exporter-otlp-proto-http"; -const PY_RUN_SNIPPET = "opentelemetry-instrument python my_agent.py"; -const TS_RUN_SNIPPET = "npx tsx my_agent.ts"; +const EXAMPLE_MODEL = "openai/gpt-6-sol"; const SAMPLE_TRACE_POLL_MS = 1000; export const TRACING_KEY_REQUEST = { key_alias: "Agent tracing", @@ -41,177 +33,36 @@ type Installer = "pip" | "uv"; type CodingAgent = "Claude Code" | "Codex"; const PY_INSTALL: Record string> = { - pip: (packages) => `pip install -U ${packages}`, + pip: (packages) => `pip install ${packages}`, uv: (packages) => `uv add ${packages}`, }; -interface FrameworkGuide { - id: string; - label: string; - logo: string; - packages: string; - quickstart: string; - typescript?: boolean; -} - -const FRAMEWORKS: readonly FrameworkGuide[] = [ - { - id: "deep-agents", - label: "Deep Agents", - logo: langgraphLogo.src, - packages: "deepagents langchain-openai openinference-instrumentation-langchain", - quickstart: `from deepagents import create_deep_agent -from langchain_openai import ChatOpenAI - -llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = create_deep_agent(model=llm, tools=[], system_prompt="You are a careful researcher.") -agent.invoke({"messages": [{"role": "user", "content": "What is LiteLLM?"}]})`, - }, - { - id: "vercel-ai-sdk", - label: "Vercel AI SDK", - logo: vercelLogo.src, - typescript: true, - packages: "ai @ai-sdk/openai-compatible @vercel/otel @opentelemetry/api", - quickstart: `import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; -import { registerOTel } from "@vercel/otel"; -import { generateText } from "ai"; - -registerOTel({ serviceName: process.env.OTEL_SERVICE_NAME ?? "my-agent" }); - -const litellm = createOpenAICompatible({ name: "litellm", baseURL: "{PROXY}/v1", apiKey: process.env.LITELLM_API_KEY }); - -async function main() { - const { text } = await generateText({ - model: litellm("claude-sonnet-4-5"), - prompt: "What is LiteLLM?", - experimental_telemetry: { isEnabled: true, functionId: "my_agent" }, - }); - console.log(text); -} - -main();`, - }, - { - id: "langgraph", - label: "LangGraph", - logo: langgraphLogo.src, - packages: "langgraph langchain-openai openinference-instrumentation-langchain", - quickstart: `from langchain.agents import create_agent -from langchain_openai import ChatOpenAI - -llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = create_agent(model=llm, tools=[], name="my_agent") -agent.invoke({"messages": [{"role": "user", "content": "What is LiteLLM?"}]})`, - }, - { - id: "langchain", - label: "LangChain", - logo: langchainLogo.src, - packages: "langchain langchain-openai openinference-instrumentation-langchain", - quickstart: `from langchain_openai import ChatOpenAI - -llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -llm.invoke("What is LiteLLM?")`, - }, - { - id: "openai-agents", - label: "OpenAI Agents SDK", - logo: openaiAgentsLogo.src, - packages: "openai-agents openinference-instrumentation-openai-agents", - quickstart: `from agents import Agent, OpenAIChatCompletionsModel, Runner -from openai import AsyncOpenAI - -client = AsyncOpenAI(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = Agent(name="my_agent", model=OpenAIChatCompletionsModel(model="claude-sonnet-4-5", openai_client=client)) -print(Runner.run_sync(agent, "What is LiteLLM?").final_output)`, - }, - { - id: "crewai", - label: "CrewAI", - logo: crewaiLogo.src, - packages: "crewai openinference-instrumentation-crewai", - quickstart: `from crewai import LLM, Agent, Crew, Task - -llm = LLM(model="openai/claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = Agent(role="Researcher", goal="Answer questions", backstory="", llm=llm) -task = Task(description="What is LiteLLM?", expected_output="A short answer", agent=agent) -Crew(agents=[agent], tasks=[task]).kickoff()`, - }, - { - id: "pydantic-ai", - label: "Pydantic AI", - logo: pydanticAiLogo.src, - packages: "pydantic-ai openinference-instrumentation-pydantic-ai", - quickstart: `from pydantic_ai import Agent -from pydantic_ai.models.openai import OpenAIModel -from pydantic_ai.providers.openai import OpenAIProvider - -provider = OpenAIProvider(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = Agent(OpenAIModel("claude-sonnet-4-5", provider=provider), name="my_agent", instrument=True) -print(agent.run_sync("What is LiteLLM?").output)`, - }, - { - id: "llamaindex", - label: "LlamaIndex", - logo: llamaindexLogo.src, - packages: "llama-index llama-index-llms-openai-like openinference-instrumentation-llama-index", - quickstart: `from llama_index.llms.openai_like import OpenAILike - -llm = OpenAILike(model="claude-sonnet-4-5", api_base="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"], is_chat_model=True) -print(llm.complete("What is LiteLLM?"))`, - }, - { - id: "otel", - label: "OpenTelemetry", - logo: otelLogo.src, - packages: "", - quickstart: `# Any OTEL SDK works. Use the gen_ai.* semantic conventions: -# gen_ai.operation.name, gen_ai.agent.name, gen_ai.response.id, gen_ai.usage.* -from opentelemetry import trace - -tracer = trace.get_tracer("my-agent") -attrs = {"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "my_agent"} -with tracer.start_as_current_span("my_agent", attributes=attrs): - ...`, - }, -]; - -const installPackages = (guide: Pick): string => - guide.typescript ? guide.packages : [OTEL_BASE_PACKAGES, guide.packages].filter(Boolean).join(" "); - -/** The endpoint is the proxy base URL: OTLP exporters append /v1/traces themselves. */ export const tracingEnvSnippet = (proxyUrl: string, tracingKey: string | null = null): string => [ ...(tracingKey ? [`export LITELLM_TRACING_KEY=${tracingKey}`] : []), - `export OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`, - "export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf", - `export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $${tracingKey ? "LITELLM_TRACING_KEY" : "LITELLM_API_KEY"}"`, - "export OTEL_SERVICE_NAME=my-agent", + `export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${proxyUrl}/v1/traces"`, + `export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer $${tracingKey ? "LITELLM_TRACING_KEY" : "LITELLM_API_KEY"}"`, + 'export OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf"', + 'export OTEL_METRICS_EXPORTER="none"', + 'export OTEL_LOGS_EXPORTER="none"', ].join("\n"); -export const codingAgentPrompt = ( - proxyUrl: string, - guide: Pick, -): string => +export const codingAgentPrompt = (proxyUrl: string, guide: FrameworkGuide, model: string): string => [ `Send this ${guide.label} project's OpenTelemetry traces to LiteLLM.`, - "", - `1. Add these dependencies: ${installPackages(guide)}`, - "2. Set these env vars wherever the project loads config (.env, settings, deployment manifests):", - ` OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`, - " OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf", - ' OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"', - " OTEL_SERVICE_NAME=", - guide.typescript - ? "3. Call registerOTel() from @vercel/otel at startup and pass experimental_telemetry: { isEnabled: true } to every AI SDK call." - : "3. Start the app through OTEL auto-instrumentation: opentelemetry-instrument .", - `4. Point every LLM client at LiteLLM: base_url=${proxyUrl}/v1, api key from LITELLM_API_KEY.`, - "5. Give each agent and subagent a name so runs are easy to read.", - "6. Run the agent once and confirm the run shows up in the LiteLLM UI under Lens > Traces.", - "", - "Never hardcode the key. Read it from LITELLM_API_KEY.", - ].join("\n"); + "Keep the existing model configuration, authentication, and application behavior. Never hardcode a key; read it from LITELLM_API_KEY.", + "Set the trace destination wherever this project loads environment variables:", + tracingEnvSnippet(proxyUrl), + guide.install ?? `Install and enable the ${guide.plugin?.label}: ${guide.plugin?.url}`, + guide.plugin?.instruction ?? + "Initialize OpenTelemetry before creating the agent. If the app already configures a tracer provider, keep it and point its exporter at the destination above instead.", + "Adapt this example to the existing application, replacing research_agent with the agent's name:", + frameworkSnippet(guide, proxyUrl, model), + guide.note ?? "", + "Run the agent once and confirm its named run appears in Lens > Traces.", + ] + .filter(Boolean) + .join("\n\n"); const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; @@ -222,10 +73,9 @@ export const maskSecret = (secret: string): string => secret.length > 10 ? `${secret.slice(0, 5)}${"•".repeat(16)}${secret.slice(-4)}` : "•".repeat(secret.length); export const otlpEndpoints = (proxyUrl: string): readonly (readonly [string, string, boolean])[] => [ - ["Traces endpoint (POST)", `${proxyUrl}/v1/traces`, true], - ["OTEL_EXPORTER_OTLP_ENDPOINT", proxyUrl, true], + ["Traces endpoint", `${proxyUrl}/v1/traces`, true], ["Auth header", "Authorization: Bearer ", true], - ["Protocol", "OTLP/HTTP, protobuf or JSON (gRPC not supported)", false], + ["Protocol", "OTLP/HTTP (protobuf or JSON)", false], ]; export const PROXY_CONFIG_SNIPPET = [ @@ -270,8 +120,8 @@ function CodeBlock({ 
             {display}
@@ -322,7 +172,7 @@ function LineTabs({
 function Step({ title, children }: { title: React.ReactNode; children: React.ReactNode }) {
   return (
     
    
-      
    {title}
    
+      
    {title}
    
       {children}
     
    
   );
@@ -395,7 +245,7 @@ function SendTestTrace({
   const busy = state.kind === "sending" || state.kind === "waiting";
   return (
     
    
-      
-        {!connected && (
-          
    
-            Waiting for your first trace
-          
    
-        )}
       
    
       {missingAfterCheck && (
         
    
@@ -442,7 +290,7 @@ function TraceReceipt({
           logs for export errors.
         
    
       )}
-    
+
); } @@ -526,19 +374,19 @@ function EndpointValue({ value }: { value: string }) { ); } -function Endpoints({ proxyUrl }: { proxyUrl: string }) { +function Endpoints({ proxyUrl, children }: { proxyUrl: string; children?: React.ReactNode }) { return (
-

- - OpenTelemetry (OTEL) endpoints -

-

- Point any OpenTelemetry exporter here. The framework guides below set these for you. -

-
+
+

+ + Connection details +

+ {children} +
+
{otlpEndpoints(proxyUrl).map(([label, value, copyable]) => ( -
+
{label}
{copyable ? : {value}} @@ -574,7 +422,7 @@ interface ConnectAgentProps { function EnableTracing({ checked, checking, onCheck }: { checked: boolean; checking: boolean; onCheck: () => void }) { return ( <> - +

Set your ClickHouse URL, add this to config.yaml, then restart the proxy. Ask your proxy administrator if you don’t manage this deployment. @@ -594,14 +442,68 @@ function EnableTracing({ checked, checking, onCheck }: { checked: boolean; check Tracing is still unavailable. Check that the configuration was applied to this proxy and it has restarted.

)} - +
+ +
); } +function CodingAgentSetup({ proxyUrl, guide, model }: { proxyUrl: string; guide: FrameworkGuide; model: string }) { + const [codingAgent, setCodingAgent] = useState("Claude Code"); + const [copied, setCopied] = useState(null); + const command = codingAgentCommand(codingAgent, codingAgentPrompt(proxyUrl, guide, model)); + const copy = async () => { + if (await copyToClipboard(command)) { + setCopied(command); + window.setTimeout(() => setCopied(null), COPIED_RESET_MS); + } + }; + return ( +
+

+ Connect your project +

+
+
+ { + setCodingAgent(value); + setCopied(null); + }} + logos={CODING_AGENT_LOGOS} + /> +
+
+

+ Run the setup command in your agent’s project. It uses your LITELLM_API_KEY + . +

+ +
+ View command + 
+              {command}
+
+
+
+
+
+ ); +} + function ConnectAgent({ accessToken, onOpenTrace, @@ -615,27 +517,20 @@ function ConnectAgent({ const proxyUrl = getProxyBaseUrl().replace(/\/$/, ""); const [framework, setFramework] = useState(FRAMEWORKS[0].id); const [installer, setInstaller] = useState("pip"); - const [codingAgent, setCodingAgent] = useState("Claude Code"); const [tracingKey, setTracingKey] = useState(null); const guide = FRAMEWORKS.find((f) => f.id === framework) ?? FRAMEWORKS[0]; - const packages = installPackages(guide); - const install = guide.typescript ? `npm install ${packages}` : PY_INSTALL[installer](packages); - const quickstart = guide.quickstart.replace("{PROXY}", proxyUrl); + const install = guide.install?.startsWith("pip install ") + ? PY_INSTALL[installer](guide.install.slice("pip install ".length)) + : guide.install; + const model = EXAMPLE_MODEL; + const quickstart = frameworkSnippet(guide, proxyUrl, model, !!tracingKey); return ( <> - {!readOnly && ( - -

- Send a small sample run (an agent, an LLM call and a tool call) to confirm tracing works end to end. -

- - - )} - - - + + {!readOnly && } +
-
- - {canMintTracingKey && !readOnly ? ( - - ) : ( -

- Use any LiteLLM virtual key you already have, or ask a proxy admin for one. -

- )} - - - - Let Claude Code or - Codex connect it - - } - > -

- Run this in your agent’s project. It starts your coding agent with the setup task and reads the key from - LITELLM_API_KEY. -

- - } - /> - - -
- Set up manually - - npm - ) : ( - - ) - } - /> + +
+ Set up manually + + {canMintTracingKey && !readOnly ? ( + + ) : ( +

+ Use any LiteLLM virtual key you already have, or ask a proxy admin for one. +

+ )} + {install && ( + + npm + ) : ( + + ) + } + /> + + )} + {guide.plugin && ( + +

+ Install and enable the{" "} + + {guide.plugin.label} + + . {guide.plugin.instruction} +

+ + )} +

+ {tracingKey ? ( + <> + Tracing uses the key above. Set LITELLM_API_KEY to a key with model access. + + ) : ( + <> + Set LITELLM_API_KEY to your LiteLLM key. + + )} +

Shell} + wrap /> -

- Replace the example model with a model configured on your proxy. +

+ {guide.existingModel ? "Keep your existing model settings. " : "Save and run this example. "} + Replace research_agent with your agent’s name. + {!guide.existingModel && " Use a model configured on this proxy."}

- {guide.typescript ? "my_agent.ts" : "my_agent.py"}} - /> -
- Shell} /> -
+ {guide.fileName}} wrap /> + {guide.note &&

{guide.note}

} + {!guide.existingModel && ( +
+ Shell} + /> +
+ )} + + View in docs
- - {!connected && ( -
-

- Example run (sample data, not your runs) -

- -
- )} ); } @@ -755,6 +654,7 @@ export function TracingSetupCard({ checking = false, readOnly = false, canMintTracingKey = false, + onDemo, }: { detail: string | null; accessToken: string; @@ -764,6 +664,7 @@ export function TracingSetupCard({ checking?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; }) { const [checked, setChecked] = useState(false); const enabled = detail === null; @@ -773,9 +674,18 @@ export function TracingSetupCard({ }; return ( -
-
+
+ {onDemo && } +

{setupTitle(enabled, connected)}

+ + {enabled ? ( + + ) : ( + -

- {enabled && } - {enabled ? "Tracing enabled" : "Tracing is not enabled"} -

{enabled ? ( = { + MODEL: JSON.stringify(model), + OPENAI_MODEL: JSON.stringify(`openai/${model}`), + BASE_URL: JSON.stringify(`${proxyUrl}/v1`), + PROXY_URL: JSON.stringify(proxyUrl), + TRACE_URL: `${proxyUrl}/v1/traces`, + }; + const code = guide.quickstart.replace( + /\{(MODEL|OPENAI_MODEL|BASE_URL|PROXY_URL|TRACE_URL)\}/g, + (_, name: string) => values[name], + ); + return tracingKey && guide.existingModel ? code.replaceAll("${LITELLM_API_KEY}", "${LITELLM_TRACING_KEY}") : code; +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts index a50d57409de..46f4b4bed71 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts @@ -1,3 +1,4 @@ +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { useInfiniteQuery, useQuery, type UseQueryOptions } from "@tanstack/react-query"; import moment from "moment"; import { useMemo } from "react"; @@ -65,6 +66,7 @@ export function useAgentTraces({ isLiveTail, enabled, }: UseAgentTracesOptions): AgentTracesResult { + const demo = useLensDemo(); const fetchPage = (pageParam: unknown): Promise => { const nowMs = Date.now(); const listOptions: Parameters[0] = { @@ -73,7 +75,11 @@ export function useAgentTraces({ endMs: isCustomDate ? moment(endTime).valueOf() : nowMs, cursor: pageParam as string | null, }; - return agentTraceListCall(listOptions); + return demo + ? demo.client.get("/v1/traces", { + query: { start_ms: listOptions.startMs, end_ms: listOptions.endMs, cursor: listOptions.cursor }, + }) + : agentTraceListCall(listOptions); }; const queryOptions: Parameters>[0] = { queryKey: ["agentTraces", accessToken, startTime, endTime, isCustomDate], @@ -106,9 +112,11 @@ export function useAgentTraces({ } export function useTraceAvailability(accessToken: string, enabled: boolean) { + const demo = useLensDemo(); + const client = demo?.client ?? apiClient; const options: UseQueryOptions = { queryKey: ["trace-availability", accessToken], - queryFn: () => apiClient.get("/v1/traces", { accessToken, query: { start_ms: 0 } }), + queryFn: () => client.get("/v1/traces", { accessToken, query: { start_ms: 0 } }), select: (page: TracePage) => page.data.length > 0, enabled, retry: false, From 9e31afbc6db72dca0d024c99ed3845949cad6a9a Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:31:44 -0700 Subject: [PATCH 03/42] chore(openrouter): sync prices, limits and deprecation dates from the models API (#44287) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 128 +++++++++++------- model_prices_and_context_window.json | 128 +++++++++++------- 2 files changed, 152 insertions(+), 104 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 12e4760ed3a..2b613ff1aea 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42258,14 +42258,14 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4.1-flash": { - "cache_read_input_token_cost": 1e-08, - "input_cost_per_token": 3e-08, + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 1.2e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -43604,19 +43604,19 @@ "supports_web_search": false }, "openrouter/qwen/qwen3.5-35b-a3b": { - "input_cost_per_token": 1.625e-07, + "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 1.3e-06, + "output_cost_per_token": 1e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 1.5625e-07, + "cache_read_input_token_cost": 5e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_prompt_caching": true, @@ -43924,14 +43924,14 @@ }, "openrouter/z-ai/glm-5.1": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 1.7914e-07, - "input_cost_per_token": 9.646e-07, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, "litellm_provider": "openrouter", "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3.0316e-06, + "output_cost_per_token": 4.4e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -67419,13 +67419,13 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 2.219e-07, - "output_cost_per_token": 3.39e-06, - "cache_read_input_token_cost": 1.775e-07, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 1.4e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -67556,8 +67556,8 @@ "supports_prompt_caching": true }, "openrouter/deepseek/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 1.08e-08, - "input_cost_per_token": 1.08e-08, + "cache_read_input_token_cost": 5.1e-09, + "input_cost_per_token": 5.1e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, @@ -67609,6 +67609,7 @@ "input_cost_per_token": 9e-08, "output_cost_per_token": 1.8e-07, "cache_read_input_token_cost": 9e-09, + "deprecation_date": "2026-10-31", "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 131072, @@ -67626,6 +67627,7 @@ "supports_web_search": false }, "openrouter/poolside/laguna-s-2.1:free": { + "deprecation_date": "2026-10-31", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -67645,14 +67647,14 @@ "supports_web_search": false }, "openrouter/moonshotai/kimi-k3": { - "cache_read_input_token_cost": 4.357e-07, - "input_cost_per_token": 4.357e-07, + "cache_read_input_token_cost": 2.7e-07, + "input_cost_per_token": 2.7e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 1e-05, + "output_cost_per_token": 1.35e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -67668,6 +67670,7 @@ "input_cost_per_token": 6e-08, "output_cost_per_token": 1.2e-07, "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-10-31", "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 32768, @@ -67685,6 +67688,7 @@ "supports_web_search": false }, "openrouter/poolside/laguna-xs-2.1:free": { + "deprecation_date": "2026-10-31", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -67867,13 +67871,13 @@ "supports_web_search": false }, "openrouter/nvidia/nemotron-3-ultra-550b-a55b": { - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 182520, - "max_tokens": 182520, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -68131,14 +68135,14 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "cache_read_input_token_cost": 8.372e-09, - "input_cost_per_token": 4.186e-08, + "cache_read_input_token_cost": 5.6e-09, + "input_cost_per_token": 2.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 384000, + "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 8.372e-08, + "output_cost_per_token": 5.6e-08, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -68172,14 +68176,14 @@ "supports_web_search": false }, "openrouter/google/gemma-4-26b-a4b-it": { - "cache_read_input_token_cost": 4.25e-08, - "input_cost_per_token": 7.65e-08, + "cache_read_input_token_cost": 3.75e-08, + "input_cost_per_token": 6.75e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 235929, "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 2.55e-07, + "output_cost_per_token": 2.25e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -68958,11 +68962,11 @@ "openrouter/deepseek/deepseek-v3.1-terminus": { "cache_read_input_token_cost": 1.35e-07, "deprecation_date": "2026-09-28", - "input_cost_per_token": 3e-07, + "input_cost_per_token": 2.7e-07, "litellm_provider": "openrouter", "max_input_tokens": 163840, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 147456, + "max_tokens": 147456, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://openrouter.ai/api/v1/models", @@ -69006,12 +69010,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3-next-80b-a3b-thinking": { + "deprecation_date": "2026-10-09", "input_cost_per_token": 1.5e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -69187,13 +69192,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3-30b-a3b-instruct-2507": { - "input_cost_per_token": 4.815e-08, + "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 1.9305e-07, + "output_cost_per_token": 3e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -74319,14 +74324,14 @@ "supports_web_search": false }, "openrouter/inclusionai/ling-3.0-flash-fin": { - "cache_read_input_token_cost": 1.2e-08, - "input_cost_per_token": 6e-08, + "cache_read_input_token_cost": 8.4e-09, + "input_cost_per_token": 4.2e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.232e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76406,12 +76411,12 @@ "supports_web_search": false }, "openrouter/thinkingmachines/inkling": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://openrouter.ai/api/v1/models", @@ -79675,5 +79680,24 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "openrouter/inclusionai/ling-3.1-flash": { + "input_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": true, + "supports_vision": false, + "supports_web_search": false } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 12e4760ed3a..2b613ff1aea 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42258,14 +42258,14 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4.1-flash": { - "cache_read_input_token_cost": 1e-08, - "input_cost_per_token": 3e-08, + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token": 3e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 5e-07, + "output_cost_per_token": 1.2e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -43604,19 +43604,19 @@ "supports_web_search": false }, "openrouter/qwen/qwen3.5-35b-a3b": { - "input_cost_per_token": 1.625e-07, + "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 16384, - "max_tokens": 16384, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 1.3e-06, + "output_cost_per_token": 1e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "cache_read_input_token_cost": 1.5625e-07, + "cache_read_input_token_cost": 5e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_prompt_caching": true, @@ -43924,14 +43924,14 @@ }, "openrouter/z-ai/glm-5.1": { "cache_creation_input_token_cost": 0.0, - "cache_read_input_token_cost": 1.7914e-07, - "input_cost_per_token": 9.646e-07, + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, "litellm_provider": "openrouter", "max_input_tokens": 204800, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 3.0316e-06, + "output_cost_per_token": 4.4e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -67419,13 +67419,13 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.3": { - "input_cost_per_token": 2.219e-07, - "output_cost_per_token": 3.39e-06, - "cache_read_input_token_cost": 1.775e-07, + "input_cost_per_token": 1.4e-06, + "output_cost_per_token": 4.4e-06, + "cache_read_input_token_cost": 1.4e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 943718, - "max_tokens": 943718, + "max_output_tokens": 131072, + "max_tokens": 131072, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -67556,8 +67556,8 @@ "supports_prompt_caching": true }, "openrouter/deepseek/deepseek-v4-flash-0731": { - "cache_read_input_token_cost": 1.08e-08, - "input_cost_per_token": 1.08e-08, + "cache_read_input_token_cost": 5.1e-09, + "input_cost_per_token": 5.1e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, @@ -67609,6 +67609,7 @@ "input_cost_per_token": 9e-08, "output_cost_per_token": 1.8e-07, "cache_read_input_token_cost": 9e-09, + "deprecation_date": "2026-10-31", "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 131072, @@ -67626,6 +67627,7 @@ "supports_web_search": false }, "openrouter/poolside/laguna-s-2.1:free": { + "deprecation_date": "2026-10-31", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -67645,14 +67647,14 @@ "supports_web_search": false }, "openrouter/moonshotai/kimi-k3": { - "cache_read_input_token_cost": 4.357e-07, - "input_cost_per_token": 4.357e-07, + "cache_read_input_token_cost": 2.7e-07, + "input_cost_per_token": 2.7e-06, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 943718, "max_tokens": 943718, "mode": "chat", - "output_cost_per_token": 1e-05, + "output_cost_per_token": 1.35e-05, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -67668,6 +67670,7 @@ "input_cost_per_token": 6e-08, "output_cost_per_token": 1.2e-07, "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-10-31", "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 32768, @@ -67685,6 +67688,7 @@ "supports_web_search": false }, "openrouter/poolside/laguna-xs-2.1:free": { + "deprecation_date": "2026-10-31", "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", @@ -67867,13 +67871,13 @@ "supports_web_search": false }, "openrouter/nvidia/nemotron-3-ultra-550b-a55b": { - "input_cost_per_token": 6e-07, - "output_cost_per_token": 2.4e-06, - "cache_read_input_token_cost": 1.2e-07, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 2.2e-06, + "cache_read_input_token_cost": 1e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 182520, - "max_tokens": 182520, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -68131,14 +68135,14 @@ "supports_web_search": true }, "openrouter/deepseek/deepseek-v4-flash": { - "cache_read_input_token_cost": 8.372e-09, - "input_cost_per_token": 4.186e-08, + "cache_read_input_token_cost": 5.6e-09, + "input_cost_per_token": 2.8e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, - "max_output_tokens": 131072, - "max_tokens": 131072, + "max_output_tokens": 384000, + "max_tokens": 384000, "mode": "chat", - "output_cost_per_token": 8.372e-08, + "output_cost_per_token": 5.6e-08, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -68172,14 +68176,14 @@ "supports_web_search": false }, "openrouter/google/gemma-4-26b-a4b-it": { - "cache_read_input_token_cost": 4.25e-08, - "input_cost_per_token": 7.65e-08, + "cache_read_input_token_cost": 3.75e-08, + "input_cost_per_token": 6.75e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, "max_output_tokens": 235929, "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 2.55e-07, + "output_cost_per_token": 2.25e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -68958,11 +68962,11 @@ "openrouter/deepseek/deepseek-v3.1-terminus": { "cache_read_input_token_cost": 1.35e-07, "deprecation_date": "2026-09-28", - "input_cost_per_token": 3e-07, + "input_cost_per_token": 2.7e-07, "litellm_provider": "openrouter", "max_input_tokens": 163840, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 147456, + "max_tokens": 147456, "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://openrouter.ai/api/v1/models", @@ -69006,12 +69010,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3-next-80b-a3b-thinking": { + "deprecation_date": "2026-10-09", "input_cost_per_token": 1.5e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, @@ -69187,13 +69192,13 @@ "supports_web_search": false }, "openrouter/qwen/qwen3-30b-a3b-instruct-2507": { - "input_cost_per_token": 4.815e-08, + "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_output_tokens": 235929, + "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 1.9305e-07, + "output_cost_per_token": 3e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -74319,14 +74324,14 @@ "supports_web_search": false }, "openrouter/inclusionai/ling-3.0-flash-fin": { - "cache_read_input_token_cost": 1.2e-08, - "input_cost_per_token": 6e-08, + "cache_read_input_token_cost": 8.4e-09, + "input_cost_per_token": 4.2e-08, "litellm_provider": "openrouter", "max_input_tokens": 262144, - "max_output_tokens": 235929, - "max_tokens": 235929, + "max_output_tokens": 32768, + "max_tokens": 32768, "mode": "chat", - "output_cost_per_token": 1.8e-07, + "output_cost_per_token": 1.232e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -76406,12 +76411,12 @@ "supports_web_search": false }, "openrouter/thinkingmachines/inkling": { - "cache_read_input_token_cost": 1.7e-07, - "input_cost_per_token": 1e-06, + "cache_read_input_token_cost": 1.6e-07, + "input_cost_per_token": 9.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 524288, - "max_output_tokens": 471859, - "max_tokens": 471859, + "max_output_tokens": 262144, + "max_tokens": 262144, "mode": "chat", "output_cost_per_token": 4.05e-06, "source": "https://openrouter.ai/api/v1/models", @@ -79675,5 +79680,24 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "openrouter/inclusionai/ling-3.1-flash": { + "input_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "max_input_tokens": 262144, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 0.0, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": false, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_response_schema": false, + "supports_tool_choice": true, + "supports_vision": false, + "supports_web_search": false } } From dd86ca517584a6662859436aa3ecb0b13c276a0f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:01:27 -0700 Subject: [PATCH 04/42] refactor(traces): type the ClickHouse query help response (#44285) * refactor(traces): type the ClickHouse query help response Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(traces): include agent names and frameworks in named contract round trips Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(traces): cover native query help validation in the storage adapter Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Yujong Lee Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/storage-clickhouse/src/lib.rs | 2 +- .../crates/storage-clickhouse/src/read.rs | 22 +- .../crates/traces-clickhouse/src/lib.rs | 2 +- .../crates/traces-clickhouse/src/query.rs | 292 +++++++++++++----- .../traces-clickhouse/src/query/guide.rs | 5 +- .../traces-clickhouse/src/query_access.rs | 40 ++- .../crates/traces-clickhouse/src/table.rs | 10 +- .../templates/query_help.jinja | 20 +- .../traces-clickhouse/tests/migrations.rs | 8 +- .../traces-clickhouse/tests/query_access.rs | 2 +- .../crates/traces/tests/query/named.rs | 4 +- litellm/rust_bridge/_native.pyi | 2 +- litellm/rust_bridge/trace_query_responses.py | 35 ++- litellm/rust_bridge/traces.py | 13 +- tests/unit/proxy/test_tracing_endpoints.py | 69 ++++- 15 files changed, 392 insertions(+), 134 deletions(-) diff --git a/litellm-rust/crates/storage-clickhouse/src/lib.rs b/litellm-rust/crates/storage-clickhouse/src/lib.rs index 7cba0a11653..7ab2aa9bc0a 100644 --- a/litellm-rust/crates/storage-clickhouse/src/lib.rs +++ b/litellm-rust/crates/storage-clickhouse/src/lib.rs @@ -4,7 +4,7 @@ mod read; pub use error::Error; pub use insert::{insert_compressed_rows, insert_encoded_rows}; -pub use read::{Parameter, Query, execute_read, fetch, fetch_json}; +pub use read::{Parameter, Query, READ_LIMITS, ReadLimits, execute_read, fetch, fetch_json}; use url::Url; #[derive(Clone)] diff --git a/litellm-rust/crates/storage-clickhouse/src/read.rs b/litellm-rust/crates/storage-clickhouse/src/read.rs index 0dae94109bc..59d5b0de558 100644 --- a/litellm-rust/crates/storage-clickhouse/src/read.rs +++ b/litellm-rust/crates/storage-clickhouse/src/read.rs @@ -5,7 +5,18 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use crate::{Connection, Error}; -const MAX_RESPONSE_BYTES: usize = 4 * 1024 * 1024; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReadLimits { + pub result_rows: u64, + pub response_bytes: usize, + pub execution_seconds: u64, +} + +pub const READ_LIMITS: ReadLimits = ReadLimits { + result_rows: 1000, + response_bytes: 4 * 1024 * 1024, + execution_seconds: 10, +}; #[derive(Debug, Deserialize, Serialize)] #[serde(untagged)] @@ -78,9 +89,12 @@ pub async fn execute_read( .clear() .extend_pairs(existing_pairs) .append_pair("readonly", "1") - .append_pair("max_result_rows", "1000") + .append_pair("max_result_rows", &READ_LIMITS.result_rows.to_string()) .append_pair("result_overflow_mode", "throw") - .append_pair("max_execution_time", "10") + .append_pair( + "max_execution_time", + &READ_LIMITS.execution_seconds.to_string(), + ) .append_pair("wait_end_of_query", "1") .append_pair("default_format", "JSON"); @@ -101,7 +115,7 @@ pub async fn execute_read( let mut body = Vec::new(); while let Some(chunk) = response.chunk().await.map_err(|_| Error::Transport)? { - if body.len() + chunk.len() > MAX_RESPONSE_BYTES { + if body.len() + chunk.len() > READ_LIMITS.response_bytes { return Err(Error::ResponseTooLarge); } body.extend_from_slice(&chunk); diff --git a/litellm-rust/crates/traces-clickhouse/src/lib.rs b/litellm-rust/crates/traces-clickhouse/src/lib.rs index cd5877e4993..ba2f776b51a 100644 --- a/litellm-rust/crates/traces-clickhouse/src/lib.rs +++ b/litellm-rust/crates/traces-clickhouse/src/lib.rs @@ -12,7 +12,7 @@ pub use error::Error; pub use insert::{InsertRow, InsertTable, encode_rows, insert_rows, insert_shared_rows}; pub use litellm_storage_clickhouse::{Connection, Parameter}; pub use litellm_traces::{QueryScope, ReadQuery}; -pub use query::{execute_read, query_help, query_sql}; +pub use query::{QueryHelp, execute_read, query_help, query_sql}; pub use query_access::QueryReaders; pub use schema::{ NORMALIZED_FIELD_DEFINITIONS, NormalizedFieldDefinition, ensure_schema, schema_statements, diff --git a/litellm-rust/crates/traces-clickhouse/src/query.rs b/litellm-rust/crates/traces-clickhouse/src/query.rs index c862cebf556..459f0f95fe7 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query.rs @@ -6,11 +6,14 @@ use futures_util::{ stream::{self, TryStreamExt}, }; use litellm_http::Client; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde::{Deserialize, Serialize, Serializer}; +use serde_json::Value; use strum::IntoEnumIterator; -use super::{Connection, Error, NORMALIZED_FIELD_DEFINITIONS, Parameter}; +use super::{ + Connection, Error, NORMALIZED_FIELD_DEFINITIONS, NormalizedFieldDefinition, Parameter, + query_access::READER_LIMITS, +}; mod guide; pub mod lens; @@ -48,10 +51,42 @@ enum PathPart { Index(usize), } +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, strum::Display)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +enum JsonKind { + Array, + Boolean, + Integer, + Null, + Number, + Object, + String, +} + +impl JsonKind { + fn of(value: &Value) -> Self { + match value { + Value::Null => Self::Null, + Value::Bool(_) => Self::Boolean, + Value::Number(number) if number.is_i64() || number.is_u64() => Self::Integer, + Value::Number(_) => Self::Number, + Value::String(_) => Self::String, + Value::Array(_) => Self::Array, + Value::Object(_) => Self::Object, + } + } +} + +#[derive(Clone, Copy, Debug, Serialize, strum::Display)] +enum MapValueType { + String, +} + #[derive(Serialize)] struct MetadataField { path: Vec, - types: BTreeSet<&'static str>, + types: BTreeSet, expression: String, } @@ -66,42 +101,150 @@ struct ColumnSchema { #[derive(Serialize)] struct TableSchema { - name: &'static str, + name: TraceTable, columns: Vec, } +trait Unobserved { + fn unobserved() -> Self; +} + +enum Discovery { + Observed(T), + Unavailable(String), +} + +impl Serialize for Discovery { + fn serialize(&self, serializer: S) -> Result { + #[derive(Serialize)] + struct Unavailable<'a, T> { + #[serde(flatten)] + sample: T, + error: &'a str, + } + match self { + Self::Observed(sample) => sample.serialize(serializer), + Self::Unavailable(error) => Unavailable { + sample: T::unobserved(), + error, + } + .serialize(serializer), + } + } +} + #[derive(Serialize)] -struct MetadataCatalog { - table: &'static str, - column: &'static str, +struct MetadataSample { fields: Vec, sampled_rows: usize, invalid_json_rows: usize, truncated: bool, +} + +impl Unobserved for MetadataSample { + fn unobserved() -> Self { + Self { + fields: Vec::new(), + sampled_rows: 0, + invalid_json_rows: 0, + truncated: true, + } + } +} + +#[derive(Serialize)] +struct MetadataCatalog { + table: TraceTable, + column: &'static str, + #[serde(flatten)] + discovery: Discovery, sample_sql: &'static str, scope: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, } #[derive(Serialize)] struct AttributeField { key: String, #[serde(rename = "type")] - kind: &'static str, + kind: MapValueType, expression: String, } #[derive(Serialize)] -struct AttributeCatalog { - table: &'static str, - column: &'static str, +struct AttributeSample { fields: Vec, truncated: bool, +} + +impl Unobserved for AttributeSample { + fn unobserved() -> Self { + Self { + fields: Vec::new(), + truncated: true, + } + } +} + +#[derive(Serialize)] +struct AttributeCatalog { + table: TraceTable, + column: &'static str, + #[serde(flatten)] + discovery: Discovery, discovery_sql: String, scope: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - error: Option, +} + +#[derive(Serialize)] +struct NormalizedField { + table: TraceTable, + name: &'static str, + column: &'static str, + #[serde(rename = "type")] + kind: &'static str, + meaning: &'static str, +} + +impl From<&NormalizedFieldDefinition> for NormalizedField { + fn from(field: &NormalizedFieldDefinition) -> Self { + Self { + table: TraceTable::OtelTraces, + name: field.name, + column: field.clickhouse_column, + kind: field.clickhouse_type, + meaning: field.meaning, + } + } +} + +#[derive(Serialize)] +struct Relationship { + left: &'static str, + right: &'static str, + additional_predicates: &'static str, + meaning: &'static str, +} + +const RELATIONSHIPS: [Relationship; 1] = [Relationship { + left: "otel_traces.LiteLLMRequestId", + right: "spend_logs.response_id", + additional_predicates: "otel_traces.TeamId = spend_logs.team_id AND (otel_traces.TeamId != '' OR (otel_traces.UserId != '' AND otel_traces.UserId = spend_logs.user) OR (otel_traces.ApiKeyHash != '' AND otel_traces.ApiKeyHash = spend_logs.api_key))", + meaning: "The normalized ID is the response ID, not request_id. Cached requests can share response_id; joins may return multiple spend rows", +}]; + +#[derive(Serialize)] +pub struct QueryHelp { + dialect: &'static str, + access: &'static str, + response: &'static str, + tables: Vec, + normalized_fields: Vec, + metadata: MetadataCatalog, + attributes: Vec, + relationships: &'static [Relationship], + examples: [guide::Example; 5], + gotchas: [String; 11], + guide: String, } pub async fn execute_read( @@ -153,22 +296,16 @@ fn metadata_expression(path: &[PathPart]) -> String { fn discover( value: &Value, path: Vec, - fields: &mut BTreeMap, BTreeSet<&'static str>>, + fields: &mut BTreeMap, BTreeSet>, ) -> bool { if path.len() > MAX_DEPTH || (fields.len() >= MAX_FIELDS && !fields.contains_key(&path)) { return true; } if !path.is_empty() { - let kind = match value { - Value::Null => "null", - Value::Bool(_) => "boolean", - Value::Number(number) if number.is_i64() || number.is_u64() => "integer", - Value::Number(_) => "number", - Value::String(_) => "string", - Value::Array(_) => "array", - Value::Object(_) => "object", - }; - fields.entry(path.clone()).or_default().insert(kind); + fields + .entry(path.clone()) + .or_default() + .insert(JsonKind::of(value)); } match value { Value::Object(object) => object.iter().fold(false, |limited, (key, value)| { @@ -194,7 +331,7 @@ fn discover( } } -fn metadata_catalog(sample: &[MetadataRow]) -> MetadataCatalog { +fn metadata_sample(sample: &[MetadataRow]) -> MetadataSample { let (fields, limited, invalid_rows) = sample.iter().take(SAMPLE_ROWS).fold( (BTreeMap::new(), sample.len() > SAMPLE_ROWS, 0), |(fields, limited, invalid_rows), row| match serde_json::from_str::(&row.metadata) { @@ -214,24 +351,19 @@ fn metadata_catalog(sample: &[MetadataRow]) -> MetadataCatalog { types, }) .collect(); - MetadataCatalog { - table: "spend_logs", - column: "metadata", + MetadataSample { fields, sampled_rows: sample.len().min(SAMPLE_ROWS), invalid_json_rows: invalid_rows, truncated: limited, - sample_sql: METADATA_SQL, - error: None, - scope: METADATA_SCOPE, } } -pub async fn query_help(client: &Client, connection: &Connection) -> Result { +pub async fn query_help(client: &Client, connection: &Connection) -> Result { let tables = stream::iter(TraceTable::iter()) .then(|table| async move { Ok::<_, Error>(TableSchema { - name: table.into(), + name: table, columns: rows::( client, connection, @@ -242,13 +374,15 @@ pub async fn query_help(client: &Client, connection: &Connection) -> Result>() .await?; - let metadata = match rows::(client, connection, METADATA_SQL).await { - Ok(sample) => metadata_catalog(&sample), - Err(error) => MetadataCatalog { - error: Some(error.to_string()), - truncated: true, - ..metadata_catalog(&[]) + let metadata = MetadataCatalog { + table: TraceTable::SpendLogs, + column: "metadata", + discovery: match rows::(client, connection, METADATA_SQL).await { + Ok(sample) => Discovery::Observed(metadata_sample(&sample)), + Err(error) => Discovery::Unavailable(error.to_string()), }, + sample_sql: METADATA_SQL, + scope: METADATA_SCOPE, }; let attributes = stream::iter(["SpanAttributes", "ResourceAttributes"]) .then(|column| async move { @@ -257,27 +391,27 @@ pub async fn query_help(client: &Client, connection: &Connection) -> Result= now() - INTERVAL 7 DAY \ LIMIT 200) ORDER BY key LIMIT 201" ); - let (keys, error) = match rows::(client, connection, &sql).await { - Ok(keys) => (keys, None), - Err(error) => (Vec::new(), Some(error.to_string())), + let discovery = match rows::(client, connection, &sql).await { + Ok(keys) => Discovery::Observed(AttributeSample { + truncated: keys.len() > MAX_FIELDS, + fields: keys + .into_iter() + .take(MAX_FIELDS) + .map(|row| AttributeField { + expression: format!("{column}[{}]", literal(&row.key)), + key: row.key, + kind: MapValueType::String, + }) + .collect(), + }), + Err(error) => Discovery::Unavailable(error.to_string()), }; - let fields = keys - .iter() - .take(MAX_FIELDS) - .map(|row| AttributeField { - key: row.key.clone(), - kind: "String", - expression: format!("{column}[{}]", literal(&row.key)), - }) - .collect(); AttributeCatalog { - table: "otel_traces", + table: TraceTable::OtelTraces, column, - fields, - truncated: error.is_some() || keys.len() > MAX_FIELDS, + discovery, discovery_sql: sql, scope: ATTRIBUTE_SCOPE, - error, } }) .collect::>() @@ -287,33 +421,31 @@ pub async fn query_help(client: &Client, connection: &Connection) -> Result>(), - "metadata": metadata, - "attributes": attributes, - "relationships": [{ - "left": "otel_traces.LiteLLMRequestId", "right": "spend_logs.response_id", - "additional_predicates": "otel_traces.TeamId = spend_logs.team_id AND (otel_traces.TeamId != '' OR (otel_traces.UserId != '' AND otel_traces.UserId = spend_logs.user) OR (otel_traces.ApiKeyHash != '' AND otel_traces.ApiKeyHash = spend_logs.api_key))", - "meaning": "The normalized ID is the response ID, not request_id. Cached requests can share response_id; joins may return multiple spend rows" - }], - "examples": guide.examples()?, - "gotchas": guide.gotchas()?, - "guide": guide::render(&guide)?, - }).to_string()) + Ok(QueryHelp { + dialect: "ClickHouse SQL", + access: "Request-log visibility enforced by ClickHouse row policies; proxy admins see all rows, users see their own rows and permitted teams, and callers without user identity see their own key rows", + response: "ClickHouse JSON envelope: meta, data, rows, statistics; 64-bit integers may be strings", + examples: guide.examples()?, + gotchas: guide.gotchas()?, + guide: guide::render(&guide)?, + normalized_fields: NORMALIZED_FIELD_DEFINITIONS + .iter() + .map(NormalizedField::from) + .collect(), + relationships: &RELATIONSHIPS, + tables, + metadata, + attributes, + }) } #[cfg(test)] mod tests { use super::*; use rstest::rstest; + use serde_json::json; #[rstest] fn metadata_discovery_preserves_mixed_types_and_reports_invalid_rows() { @@ -328,7 +460,7 @@ mod tests { metadata: "invalid".into(), }, ]; - let catalog = json!(metadata_catalog(&sample)); + let catalog = json!(metadata_sample(&sample)); assert_eq!( catalog["fields"], json!([{ @@ -351,7 +483,7 @@ mod tests { metadata: json!(metadata).to_string(), }) .collect(); - let catalog = json!(metadata_catalog(&sample)); + let catalog = json!(metadata_sample(&sample)); assert_eq!(catalog["truncated"], true); assert_eq!(catalog["sampled_rows"], row_count.min(SAMPLE_ROWS)); assert_eq!( diff --git a/litellm-rust/crates/traces-clickhouse/src/query/guide.rs b/litellm-rust/crates/traces-clickhouse/src/query/guide.rs index 3bf7336648d..a0b60f9666f 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/guide.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/guide.rs @@ -1,8 +1,8 @@ use askama::Template; use serde::Serialize; -use super::{AttributeCatalog, MetadataCatalog, TableSchema}; -use crate::{Error, NormalizedFieldDefinition}; +use super::{AttributeCatalog, Discovery, MetadataCatalog, TableSchema}; +use crate::{Error, NormalizedFieldDefinition, query_access::ReaderLimits}; #[derive(Template)] #[template(path = "query_help.jinja", escape = "none", blocks = [ @@ -33,6 +33,7 @@ pub(super) struct QueryGuide<'a> { pub normalized_fields: &'a [NormalizedFieldDefinition], pub metadata: &'a MetadataCatalog, pub attributes: &'a [AttributeCatalog], + pub limits: &'a ReaderLimits, } #[derive(Serialize)] diff --git a/litellm-rust/crates/traces-clickhouse/src/query_access.rs b/litellm-rust/crates/traces-clickhouse/src/query_access.rs index 94ead18dd94..5d82a5e2e93 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query_access.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query_access.rs @@ -2,6 +2,7 @@ use std::{sync::Arc, time::Duration}; use hmac::{Hmac, Mac}; use litellm_http::Client; +use litellm_storage_clickhouse::READ_LIMITS; use litellm_traces::QueryScope; use moka::future::Cache; use strum::IntoEnumIterator; @@ -11,6 +12,33 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use super::{Connection, Error, TraceTable}; +const MIB: u64 = 1024 * 1024; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ReaderLimits { + pub result_rows: u64, + pub result_bytes: u64, + pub memory_bytes: u64, + pub execution_seconds: u64, +} + +impl ReaderLimits { + pub fn result_mib(&self) -> u64 { + self.result_bytes / MIB + } + + pub fn memory_mib(&self) -> u64 { + self.memory_bytes / MIB + } +} + +pub(crate) const READER_LIMITS: ReaderLimits = ReaderLimits { + result_rows: READ_LIMITS.result_rows, + result_bytes: READ_LIMITS.response_bytes as u64, + memory_bytes: 256 * MIB, + execution_seconds: READ_LIMITS.execution_seconds, +}; + #[derive(Clone)] pub struct QueryReaders { writer: Connection, @@ -75,13 +103,19 @@ impl QueryReaders { return Err(Error::InvalidScope); } let password_hash = format!("{:x}", Sha256::digest(password)); + let ReaderLimits { + result_rows, + result_bytes, + memory_bytes, + execution_seconds, + } = READER_LIMITS; self.execute( client, format!( "CREATE USER IF NOT EXISTS {user} IDENTIFIED WITH sha256_hash BY '{password_hash}' \ - SETTINGS readonly = 1 CONST, max_execution_time = 10 CONST, \ - max_result_rows = 1000 CONST, max_result_bytes = 4194304 CONST, \ - result_overflow_mode = 'throw' CONST, max_memory_usage = 268435456 CONST, \ + SETTINGS readonly = 1 CONST, max_execution_time = {execution_seconds} CONST, \ + max_result_rows = {result_rows} CONST, max_result_bytes = {result_bytes} CONST, \ + result_overflow_mode = 'throw' CONST, max_memory_usage = {memory_bytes} CONST, \ max_threads = 2 CONST, max_concurrent_queries_for_user = 8 CONST" ), ) diff --git a/litellm-rust/crates/traces-clickhouse/src/table.rs b/litellm-rust/crates/traces-clickhouse/src/table.rs index 5e3fe83fa82..4cbbf3cee89 100644 --- a/litellm-rust/crates/traces-clickhouse/src/table.rs +++ b/litellm-rust/crates/traces-clickhouse/src/table.rs @@ -1,6 +1,14 @@ #[derive( - Clone, Copy, Debug, strum::Display, strum::AsRefStr, strum::EnumIter, strum::IntoStaticStr, + Clone, + Copy, + Debug, + serde::Serialize, + strum::Display, + strum::AsRefStr, + strum::EnumIter, + strum::IntoStaticStr, )] +#[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] pub enum TraceTable { OtelTraces, diff --git a/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja b/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja index a79342c8343..6e746221022 100644 --- a/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja +++ b/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja @@ -11,18 +11,18 @@ Normalized span fields {% endfor %} Observed LLM call metadata {{ metadata.scope }} -Sampled rows: {{ metadata.sampled_rows }}; invalid JSON rows: {{ metadata.invalid_json_rows }}; truncated: {{ metadata.truncated }} -{% if let Some(error) = metadata.error %}Metadata discovery unavailable: {{ error }} -{% else if metadata.fields.is_empty() %}No metadata paths found in the sampled rows -{% else %}{% for field in metadata.fields %}{{ field.expression }}: {% for kind in field.types %}{{ kind }} {% endfor %} -{% endfor %}{% endif %} +{% match metadata.discovery %}{% when Discovery::Unavailable(error) %}Metadata discovery unavailable: {{ error }} +{% when Discovery::Observed(sample) %}Sampled rows: {{ sample.sampled_rows }}; invalid JSON rows: {{ sample.invalid_json_rows }}; truncated: {{ sample.truncated }} +{% if sample.fields.is_empty() %}No metadata paths found in the sampled rows +{% else %}{% for field in sample.fields %}{{ field.expression }}: {% for kind in field.types %}{{ kind }} {% endfor %} +{% endfor %}{% endif %}{% endmatch %} Observed span and resource attributes {% for catalog in attributes %}{{ catalog.table }}.{{ catalog.column }} {{ catalog.scope }} -{% if let Some(error) = catalog.error %}Attribute discovery unavailable: {{ error }} -{% else if catalog.fields.is_empty() %}No attribute keys found in the sampled spans -{% else %}{% for field in catalog.fields %}{{ field.expression }}: {{ field.kind }} -{% endfor %}{% endif %}{% endfor %} +{% match catalog.discovery %}{% when Discovery::Unavailable(error) %}Attribute discovery unavailable: {{ error }} +{% when Discovery::Observed(sample) %}{% if sample.fields.is_empty() %}No attribute keys found in the sampled spans +{% else %}{% for field in sample.fields %}{{ field.expression }}: {{ field.kind }} +{% endfor %}{% endif %}{% endmatch %}{% endfor %} Examples {% block recent_spans_name %}Recent normalized LLM spans{% endblock %} @@ -44,7 +44,7 @@ Gotchas {% block time_window %}Always bound Timestamp or start_time and use LIMIT; add TeamId/ApiKeyHash or team_id/api_key filters when investigating one tenant{% endblock %} -{% block reader_limits %}The reader enforces 1000 result rows, 4 MiB response bytes, 256 MiB memory and a 10 second query limit; exceeding limits fails instead of returning partial results{% endblock %} +{% block reader_limits %}The reader enforces {{ limits.result_rows }} result rows, {{ limits.result_mib() }} MiB response bytes, {{ limits.memory_mib() }} MiB memory and a {{ limits.execution_seconds }} second query limit; exceeding limits fails instead of returning partial results{% endblock %} {% block reader_profile %}LiteLLM provisions SELECT-only readers from the configured ClickHouse connection and enforces request-log visibility through row policies. Callers see their own user rows and permitted teams, or their own key rows when no user identity is available. Provisioning requires CREATE USER, ALTER USER, CREATE ROW POLICY, and GRANT SELECT permissions{% endblock %} diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index dff7d8d5940..9fdeb450c70 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -1467,8 +1467,8 @@ async fn query_help_discovers_live_schema_and_runs_its_examples( ) .await?; } - let help: serde_json::Value = serde_json::from_str( - &litellm_traces_clickhouse::query_help(&database.client, &reader).await?, + let help = serde_json::to_value( + litellm_traces_clickhouse::query_help(&database.client, &reader).await?, )?; let keys: std::collections::BTreeSet<_> = help .as_object() @@ -1654,8 +1654,8 @@ async fn query_help_preserves_schema_and_guide_when_discovery_hits_reader_limits }))).collect::, _>>()?; insert_rows(&database, "otel_traces", spans).await?; let reader = Connection::configured(&database.url, "trace_test", "help_reader", "")?; - let help: serde_json::Value = serde_json::from_str( - &litellm_traces_clickhouse::query_help(&database.client, &reader).await?, + let help = serde_json::to_value( + litellm_traces_clickhouse::query_help(&database.client, &reader).await?, )?; assert_eq!(help["tables"].as_array().ok_or("tables")?.len(), 3); assert!(!help["examples"].as_array().ok_or("examples")?.is_empty()); diff --git a/litellm-rust/crates/traces-clickhouse/tests/query_access.rs b/litellm-rust/crates/traces-clickhouse/tests/query_access.rs index d4c7c886bd0..be749351a72 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/query_access.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/query_access.rs @@ -94,7 +94,7 @@ async fn queries_and_help_are_scoped_by_the_database( .await?, )?; assert_eq!(summary["data"][0]["count"], json!(expected.len())); - let help = query_help(&database.client, &reader).await?; + let help = serde_json::to_string(&query_help(&database.client, &reader).await?)?; assert_eq!(help.contains("secret_b"), expected.contains(&"b")); assert_eq!(help.contains("secret-b"), expected.contains(&"b")); let recreated = QueryReaders::new(database.writer.clone(), "trace_test".to_owned()); diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index b20da7bd800..7db77df8a29 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -46,10 +46,10 @@ fn named_requests_preserve_all_access_cases(#[case] teams: Vec<&str>, #[case] ke #[rstest] fn result_contracts_preserve_public_field_names() { round_trip::( - json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "ok", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), + json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "ok", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), ); round_trip::( - json!({"span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "agent": "agent", "status": "error", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "agent": "agent", "framework": "framework", "status": "error", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), ); round_trip::( json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}), diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index a86daaa35ad..45bccad6179 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -41,7 +41,7 @@ class NativeTraceStorage: def ensure_schema(self) -> Future[None]: ... def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> Future[None]: ... def query_sql(self, sql: str, scope: QueryScope, secret: str) -> Future[str]: ... - def query_help(self, scope: QueryScope, secret: str) -> Future[str]: ... + def query_help(self, scope: QueryScope, secret: str) -> Future[JsonValue]: ... def query(self, query: ReadQueryName, parameters: Mapping[str, str | int | float | Sequence[str]]) -> Future[str]: ... @final diff --git a/litellm/rust_bridge/trace_query_responses.py b/litellm/rust_bridge/trace_query_responses.py index 914d9b6c8d4..ad7678fcd9b 100644 --- a/litellm/rust_bridge/trace_query_responses.py +++ b/litellm/rust_bridge/trace_query_responses.py @@ -1,9 +1,12 @@ from collections.abc import Mapping -from typing import Final +from typing import Final, Literal from pydantic import BaseModel, ConfigDict, JsonValue _RESPONSE_CONFIG: Final = ConfigDict(frozen=True, extra="allow") +_HELP_CONFIG: Final = ConfigDict(frozen=True, extra="forbid") +TraceTableName = Literal["otel_traces", "agent_traces_by_key", "spend_logs"] +MetadataValueType = Literal["array", "boolean", "integer", "null", "number", "object", "string"] class TraceQueryColumn(BaseModel): @@ -28,14 +31,14 @@ class TraceSQLResponse(BaseModel): class TraceQueryTable(BaseModel): - model_config = ConfigDict(frozen=True) - name: str + model_config = _HELP_CONFIG + name: TraceTableName columns: tuple[TraceQueryColumn, ...] class TraceQueryNormalizedField(BaseModel): - model_config = ConfigDict(frozen=True) - table: str + model_config = _HELP_CONFIG + table: TraceTableName name: str column: str type: str @@ -43,15 +46,15 @@ class TraceQueryNormalizedField(BaseModel): class TraceQueryMetadataField(BaseModel): - model_config = ConfigDict(frozen=True) + model_config = _HELP_CONFIG path: tuple[str | int, ...] - types: tuple[str, ...] + types: tuple[MetadataValueType, ...] expression: str class TraceQueryMetadata(BaseModel): - model_config = ConfigDict(frozen=True) - table: str + model_config = _HELP_CONFIG + table: TraceTableName column: str fields: tuple[TraceQueryMetadataField, ...] sampled_rows: int @@ -63,15 +66,15 @@ class TraceQueryMetadata(BaseModel): class TraceQueryAttributeField(BaseModel): - model_config = ConfigDict(frozen=True) + model_config = _HELP_CONFIG key: str - type: str + type: Literal["String"] expression: str class TraceQueryAttributes(BaseModel): - model_config = ConfigDict(frozen=True) - table: str + model_config = _HELP_CONFIG + table: TraceTableName column: str fields: tuple[TraceQueryAttributeField, ...] truncated: bool @@ -81,7 +84,7 @@ class TraceQueryAttributes(BaseModel): class TraceQueryRelationship(BaseModel): - model_config = ConfigDict(frozen=True) + model_config = _HELP_CONFIG left: str right: str additional_predicates: str @@ -89,13 +92,13 @@ class TraceQueryRelationship(BaseModel): class TraceQueryExample(BaseModel): - model_config = ConfigDict(frozen=True) + model_config = _HELP_CONFIG name: str sql: str class TraceQueryHelp(BaseModel): - model_config = ConfigDict(frozen=True) + model_config = _HELP_CONFIG dialect: str access: str response: str diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index 2ed2df4e55e..23778983d75 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -2,7 +2,7 @@ from collections.abc import Awaitable, Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal, Protocol, TypedDict, TypeVar, runtime_checkable -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly from litellm.rust_bridge.loader import get_native_bridge @@ -111,7 +111,7 @@ class NativeStore(Protocol): def query_sql(self, sql: str, scope: QueryScope, secret: str) -> Awaitable[str]: ... - def query_help(self, scope: QueryScope, secret: str) -> Awaitable[str]: ... + def query_help(self, scope: QueryScope, secret: str) -> Awaitable[JsonValue]: ... def query( self, name: ReadQueryName, parameters: Mapping[str, str | int | float | Sequence[str]] @@ -189,6 +189,13 @@ def _decode_query_response(adapter: TypeAdapter[_ResponseT], body: str) -> _Resp raise RuntimeError("Native trace query returned an invalid response") from error +def _validate_query_response(adapter: TypeAdapter[_ResponseT], value: JsonValue) -> _ResponseT: + try: + return adapter.validate_python(value) + except ValidationError as error: + raise RuntimeError("Native trace query returned an invalid response") from error + + class ClickHouseStorage: def __init__(self, config: TraceStorageConfig) -> None: native: Final = _native() @@ -216,7 +223,7 @@ class ClickHouseStorage: async def query_help(self, scope: QueryScope, secret: str) -> TraceQueryHelp: result: Final = await self._native.query_help(scope, secret) - return _decode_query_response(_HELP_RESPONSE, result) + return _validate_query_response(_HELP_RESPONSE, result) async def lens_sample(self, parameters: LensSampleParams) -> tuple[ExecutionRow, ...]: return await self.query(LENS_SAMPLE, parameters) diff --git a/tests/unit/proxy/test_tracing_endpoints.py b/tests/unit/proxy/test_tracing_endpoints.py index 7c69796ddfa..fd05d54dcb6 100644 --- a/tests/unit/proxy/test_tracing_endpoints.py +++ b/tests/unit/proxy/test_tracing_endpoints.py @@ -2,8 +2,9 @@ Tests for the agent tracing endpoints (litellm/proxy/tracing_endpoints.py). """ -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager +from types import ModuleType from typing import Final from unittest.mock import AsyncMock, MagicMock @@ -15,9 +16,10 @@ from litellm.proxy import tracing_endpoints from litellm.proxy._types import LitellmUserRoles, ProxyLifespanState, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.tracing_runtime import manage_tracing, provide_storage +from litellm.rust_bridge import loader from litellm.rust_bridge.trace_queries import SPAN_DETAIL, SpanDetailParams from litellm.rust_bridge.trace_query_responses import TraceQueryHelp, TraceSQLResponse -from litellm.rust_bridge.traces import ClickHouseStorage +from litellm.rust_bridge.traces import AdminQueryScope, ClickHouseStorage, TraceStorageConfig from litellm.tracing import TraceReceiver, TracingPayloadTooLargeError from litellm.tracing.store import TraceStore from litellm.tracing.types import TraceScope @@ -29,14 +31,14 @@ SQL_ENVELOPE: Final = { "statistics": {"elapsed": 0.01, "rows_read": 1, "bytes_read": 8}, "rows_before_limit_at_least": 1, } -QUERY_HELP: Final = { +QUERY_HELP: Final[Mapping[str, object]] = { "dialect": "test SQL", "access": "authenticated scope", "response": "JSON envelope", - "tables": [{"name": "traces", "columns": [{"name": "value", "type": "String", "comment": "label"}]}], + "tables": [{"name": "otel_traces", "columns": [{"name": "value", "type": "String", "comment": "label"}]}], "normalized_fields": [], "metadata": { - "table": "traces", + "table": "spend_logs", "column": "metadata", "fields": [], "sampled_rows": 0, @@ -668,3 +670,60 @@ def test_queries_require_a_proxy_secret( receiver.store.storage.query_sql.assert_awaited_once_with( "SELECT 1", {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, secret ) + + +class _NativeConfig: + def __init__(self, database: str, url: str, retention_days: int) -> None: + pass + + +class _NativeReturningHelp(ModuleType): + def __init__(self, help_payload: Mapping[str, object]) -> None: + super().__init__("native_traces") + + class Storage: + def __init__(self, config: _NativeConfig) -> None: + pass + + async def query_help(self, scope: AdminQueryScope, secret: str) -> Mapping[str, object]: + return help_payload + + self.NativeTraceConfig: Final = _NativeConfig + self.NativeTraceStorage: Final = Storage + self.trace_decode_otlp: Final = list + self.trace_encode_error: Final = bytes + self.trace_normalized_field_definitions: Final = list + + +async def test_storage_validates_the_native_query_help_value(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(loader, "_cached_bridge", _NativeReturningHelp(QUERY_HELP)) + storage: Final = ClickHouseStorage(TraceStorageConfig("http://clickhouse:8123")) + assert await storage.query_help({"kind": "admin"}, "secret") == TraceQueryHelp.model_validate(QUERY_HELP) + + +@pytest.mark.parametrize( + "drift", + ( + { + "metadata": { + "table": "spend_logs", + "column": "metadata", + "fields": [{"path": ["a"], "types": ["boolen"], "expression": "a"}], + "sampled_rows": 1, + "invalid_json_rows": 0, + "truncated": False, + "sample_sql": "SELECT metadata FROM spend_logs", + "scope": "bounded sample", + } + }, + {"tables": [{"name": "traces", "columns": [{"name": "value", "type": "String"}]}]}, + {"unexpected": True}, + ), +) +async def test_storage_rejects_native_query_help_that_drifts_from_the_contract( + monkeypatch: pytest.MonkeyPatch, drift: Mapping[str, object] +) -> None: + monkeypatch.setattr(loader, "_cached_bridge", _NativeReturningHelp({**QUERY_HELP, **drift})) + storage: Final = ClickHouseStorage(TraceStorageConfig("http://clickhouse:8123")) + with pytest.raises(RuntimeError, match="invalid response"): + await storage.query_help({"kind": "admin"}, "secret") From b877a38e5fc41d797f67fd973d8ccad53b23fbac Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 2 Oct 2026 18:12:56 -0700 Subject: [PATCH 05/42] fix(proxy): stop queued registry read-throughs spending the resync budget (#44277) * fix(proxy): stop queued registry read-throughs spending the resync budget RegistryReadThrough.attempt serializes misses behind one lock, but every request that was queued behind the first one still spent a unit of the 20-per-5s resync budget and re-ran the DB resync, even though the first request had already loaded the object. A burst of more than 20 requests for a model created on another worker therefore exhausted the budget and the rest got 400 "Invalid model name". attempt now checks whether the key is already loaded once it holds the lock and returns early without touching the budget. Models check the router's model names and deployment ids, guardrails and agents reuse their existing registry lookups. * test(proxy): gate the queued read-through test on events and cover each registry's loaded check The queued-requests test now holds the first resync on an asyncio.Event instead of a timed sleep and records calls in a recorder with tuple and frozenset state. New tests show the model, guardrail and agent read-throughs each answer an object that is already loaded without reading the database, so rewiring any registry's loaded check now fails a test * test(proxy): keep the queued read-through recorder inside its test and type the agent registry fixture --- .../common_utils/registry_read_through.py | 28 ++- .../test_registry_read_through.py | 166 ++++++++++++++++-- tests/unit/proxy/conftest.py | 4 +- 3 files changed, 184 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py index 8a82e253c5c..32beb0b7447 100644 --- a/litellm/proxy/common_utils/registry_read_through.py +++ b/litellm/proxy/common_utils/registry_read_through.py @@ -36,6 +36,7 @@ READ_THROUGH_MAX_RESYNCS_PER_WINDOW: Final = 20 class RegistryReadThrough: __slots__ = ( + "_is_loaded", "_lock", "_max_resyncs_per_window", "_miss_ttl_seconds", @@ -49,11 +50,13 @@ class RegistryReadThrough: def __init__( self, resync: Callable[[str], Awaitable[bool]], + is_loaded: Callable[[str], bool], miss_ttl_seconds: float = READ_THROUGH_MISS_TTL_SECONDS, max_resyncs_per_window: int = READ_THROUGH_MAX_RESYNCS_PER_WINDOW, resync_window_seconds: float = READ_THROUGH_RESYNC_WINDOW_SECONDS, ) -> None: self._resync = resync + self._is_loaded = is_loaded self._miss_ttl_seconds = miss_ttl_seconds self._max_resyncs_per_window = max_resyncs_per_window self._resync_window_seconds = resync_window_seconds @@ -78,6 +81,8 @@ class RegistryReadThrough: async with self._lock: if self._recent_misses.get_cache(key) is not None: return False + if self._is_loaded(key): + return True if not self._consume_resync_budget(): verbose_proxy_logger.warning( "registry read-through for %r skipped: resync budget of %s per %ss exhausted", @@ -190,9 +195,26 @@ async def _resync_agents(agent_id_or_name: str) -> bool: return True -model_registry_read_through: Final = RegistryReadThrough(resync=_resync_model_deployments) -guardrail_registry_read_through: Final = RegistryReadThrough(resync=_resync_guardrails) -agent_registry_read_through: Final = RegistryReadThrough(resync=_resync_agents) +def _model_is_loaded(model_name_or_id: str) -> bool: + from litellm.proxy import proxy_server + + router: Final = proxy_server.llm_router + if router is None: + return False + return model_name_or_id in router.model_names or router.has_model_id(model_name_or_id) + + +def _guardrail_is_loaded(guardrail_name: str) -> bool: + return _initialized_guardrail(guardrail_name) is not None + + +def _agent_is_loaded(agent_id_or_name: str) -> bool: + return _agent_from_registry(agent_id_or_name) is not None + + +model_registry_read_through: Final = RegistryReadThrough(resync=_resync_model_deployments, is_loaded=_model_is_loaded) +guardrail_registry_read_through: Final = RegistryReadThrough(resync=_resync_guardrails, is_loaded=_guardrail_is_loaded) +agent_registry_read_through: Final = RegistryReadThrough(resync=_resync_agents, is_loaded=_agent_is_loaded) def _agent_from_registry(agent_id_or_name: str) -> "AgentResponse | None": diff --git a/tests/unit/proxy/common_utils/test_registry_read_through.py b/tests/unit/proxy/common_utils/test_registry_read_through.py index 9e20386bf3d..5713f7dfaa7 100644 --- a/tests/unit/proxy/common_utils/test_registry_read_through.py +++ b/tests/unit/proxy/common_utils/test_registry_read_through.py @@ -1,10 +1,17 @@ import asyncio -from typing import Final +from typing import TYPE_CHECKING, Final import pytest from litellm.proxy.common_utils.registry_read_through import RegistryReadThrough +if TYPE_CHECKING: + from litellm.proxy.agent_endpoints.agent_registry import AgentRegistry + + +def nothing_loaded(_key: str) -> bool: + return False + class ResyncSpy: def __init__(self, found: bool = True, error: Exception | None = None) -> None: @@ -22,7 +29,7 @@ class ResyncSpy: @pytest.mark.asyncio async def test_attempt_returns_true_when_resync_finds_object(): spy: Final = ResyncSpy(found=True) - read_through: Final = RegistryReadThrough(resync=spy) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded) assert await read_through.attempt("new-model") is True assert spy.calls == ["new-model"] @@ -31,7 +38,7 @@ async def test_attempt_returns_true_when_resync_finds_object(): @pytest.mark.asyncio async def test_attempt_found_key_is_not_negative_cached(): spy: Final = ResyncSpy(found=True) - read_through: Final = RegistryReadThrough(resync=spy) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded) assert await read_through.attempt("new-model") is True assert await read_through.attempt("new-model") is True @@ -41,7 +48,7 @@ async def test_attempt_found_key_is_not_negative_cached(): @pytest.mark.asyncio async def test_missing_key_is_negative_cached_within_ttl(): spy: Final = ResyncSpy(found=False) - read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded, miss_ttl_seconds=60.0) assert await read_through.attempt("ghost-model") is False assert await read_through.attempt("ghost-model") is False @@ -51,7 +58,7 @@ async def test_missing_key_is_negative_cached_within_ttl(): @pytest.mark.asyncio async def test_negative_cache_expires_and_resync_runs_again(): spy: Final = ResyncSpy(found=False) - read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=0.05) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded, miss_ttl_seconds=0.05) assert await read_through.attempt("ghost-model") is False await asyncio.sleep(0.1) @@ -62,7 +69,7 @@ async def test_negative_cache_expires_and_resync_runs_again(): @pytest.mark.asyncio async def test_resync_exception_returns_false_without_negative_caching(): spy: Final = ResyncSpy(error=RuntimeError("db down")) - read_through: Final = RegistryReadThrough(resync=spy) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded) assert await read_through.attempt("new-model") is False assert await read_through.attempt("new-model") is False @@ -77,7 +84,7 @@ async def test_concurrent_attempts_for_missing_key_resync_once(): return await super().__call__(key) spy: Final = SlowResyncSpy(found=False) - read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded, miss_ttl_seconds=60.0) results: Final = await asyncio.gather(*(read_through.attempt("ghost-model") for _ in range(5))) assert results == [False] * 5 @@ -87,7 +94,7 @@ async def test_concurrent_attempts_for_missing_key_resync_once(): @pytest.mark.asyncio async def test_distinct_keys_do_not_share_negative_cache(): spy: Final = ResyncSpy(found=False) - read_through: Final = RegistryReadThrough(resync=spy, miss_ttl_seconds=60.0) + read_through: Final = RegistryReadThrough(resync=spy, is_loaded=nothing_loaded, miss_ttl_seconds=60.0) assert await read_through.attempt("ghost-a") is False assert await read_through.attempt("ghost-b") is False @@ -98,7 +105,11 @@ async def test_distinct_keys_do_not_share_negative_cache(): async def test_resync_budget_exhausted_blocks_resync_without_negative_caching(): spy: Final = ResyncSpy(found=False) read_through: Final = RegistryReadThrough( - resync=spy, miss_ttl_seconds=60.0, max_resyncs_per_window=2, resync_window_seconds=60.0 + resync=spy, + is_loaded=nothing_loaded, + miss_ttl_seconds=60.0, + max_resyncs_per_window=2, + resync_window_seconds=60.0, ) assert await read_through.attempt("ghost-a") is False @@ -108,10 +119,48 @@ async def test_resync_budget_exhausted_blocks_resync_without_negative_caching(): assert read_through._recent_misses.get_cache("ghost-c") is None +@pytest.mark.asyncio +async def test_requests_queued_behind_a_successful_resync_spend_no_budget(): + from unittest.mock import AsyncMock, call + + entered: Final = asyncio.Event() + release: Final = asyncio.Event() + new_model_loaded: Final = asyncio.Event() + + async def gated_load(key: str) -> bool: + entered.set() + await release.wait() + if key == "new-model": + new_model_loaded.set() + return True + + def is_loaded(key: str) -> bool: + return key == "new-model" and new_model_loaded.is_set() + + resync: Final = AsyncMock(side_effect=gated_load) + read_through: Final = RegistryReadThrough( + resync=resync, + is_loaded=is_loaded, + max_resyncs_per_window=2, + resync_window_seconds=60.0, + ) + + burst: Final = asyncio.gather(*(read_through.attempt("new-model") for _ in range(25))) + await entered.wait() + release.set() + + assert await burst == [True] * 25 + assert resync.await_args_list == [call("new-model")] + assert await read_through.attempt("other-model") is True + assert resync.await_args_list == [call("new-model"), call("other-model")] + + @pytest.mark.asyncio async def test_resync_budget_replenishes_after_window(): spy: Final = ResyncSpy(found=True) - read_through: Final = RegistryReadThrough(resync=spy, max_resyncs_per_window=1, resync_window_seconds=0.05) + read_through: Final = RegistryReadThrough( + resync=spy, is_loaded=nothing_loaded, max_resyncs_per_window=1, resync_window_seconds=0.05 + ) assert await read_through.attempt("model-a") is True assert await read_through.attempt("model-b") is False @@ -551,3 +600,100 @@ async def test_agent_read_through_hydrates_identity_binding(lookup, clean_agent_ assert agent.identity is not None assert agent.identity.model_dump(include=set(binding)) == binding assert clean_agent_registry.get_agent_by_id(agent_id="agent-id").identity == agent.identity + + +def test_model_is_loaded_matches_router_model_names_and_deployment_ids(monkeypatch: pytest.MonkeyPatch): + import litellm.proxy.proxy_server as proxy_server + from litellm import Router + from litellm.proxy.common_utils.registry_read_through import _model_is_loaded + + router: Final = Router( + model_list=[ + { + "model_name": "loaded-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": {"id": "loaded-deployment-id"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert _model_is_loaded("loaded-model") is True + assert _model_is_loaded("loaded-deployment-id") is True + assert _model_is_loaded("model-created-on-a-sibling") is False + + monkeypatch.setattr(proxy_server, "llm_router", None) + assert _model_is_loaded("loaded-model") is False + + +@pytest.mark.asyncio +async def test_model_read_through_answers_a_loaded_model_without_reading_the_db(monkeypatch: pytest.MonkeyPatch): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm import Router + from litellm.proxy.common_utils.registry_read_through import model_registry_read_through + + prisma_client: Final = MagicMock() + prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(side_effect=AssertionError("db read")) + router: Final = Router( + model_list=[ + { + "model_name": "wired-loaded-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", router) + + assert await model_registry_read_through.attempt("wired-loaded-model") is True + prisma_client.db.litellm_proxymodeltable.find_many.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_guardrail_read_through_answers_a_loaded_guardrail_without_reading_the_db( + monkeypatch: pytest.MonkeyPatch, +): + from unittest.mock import AsyncMock, MagicMock + + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import guardrail_registry_read_through + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER + from litellm.types.guardrails import Guardrail + + guardrail_id: Final = "wired-loaded-guardrail-id" + guardrail_name: Final = "wired-loaded-guardrail" + prisma_client: Final = MagicMock() + prisma_client.db.litellm_guardrailstable.find_first = AsyncMock(side_effect=AssertionError("db read")) + monkeypatch.setattr(proxy_server, "prisma_client", prisma_client) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + + IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( + guardrail=Guardrail(**dict(FakeGuardrailRow(guardrail_id, guardrail_name))) + ) + try: + assert await guardrail_registry_read_through.attempt(guardrail_name) is True + prisma_client.db.litellm_guardrailstable.find_first.assert_not_awaited() + finally: + IN_MEMORY_GUARDRAIL_HANDLER.delete_in_memory_guardrail(guardrail_id) + + +@pytest.mark.asyncio +async def test_agent_read_through_answers_a_loaded_agent_without_reading_the_db( + clean_agent_registry: "AgentRegistry", monkeypatch: pytest.MonkeyPatch +): + import litellm.proxy.proxy_server as proxy_server + from litellm.proxy.common_utils.registry_read_through import agent_registry_read_through + from litellm.types.agents import AgentResponse + + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + clean_agent_registry.register_agent( + agent_config=AgentResponse.model_validate( + FakeAgentRow("wired-loaded-agent-id", "wired-loaded-agent").model_dump() + ) + ) + + assert await agent_registry_read_through.attempt("wired-loaded-agent-id") is True + assert await agent_registry_read_through.attempt("wired-loaded-agent") is True diff --git a/tests/unit/proxy/conftest.py b/tests/unit/proxy/conftest.py index 50c89387d80..6dec588b763 100644 --- a/tests/unit/proxy/conftest.py +++ b/tests/unit/proxy/conftest.py @@ -411,6 +411,8 @@ def create_proxy_test_client( def fresh_agent_read_through(monkeypatch): from litellm.proxy.common_utils import registry_read_through - read_through = registry_read_through.RegistryReadThrough(resync=registry_read_through._resync_agents) + read_through = registry_read_through.RegistryReadThrough( + resync=registry_read_through._resync_agents, is_loaded=registry_read_through._agent_is_loaded + ) monkeypatch.setattr(registry_read_through, "agent_registry_read_through", read_through) return read_through From b28ce93d2bdc8467b404234f40fe3d17aa7d0d80 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:28:22 -0700 Subject: [PATCH 06/42] feat(lens): show investigation progress as one staged bar with time left (#44301) * feat(lens): compute overall investigation progress, speed and time left * test(lens): cover overall progress, speed and time left estimates * feat(lens): show investigation progress as one staged bar with time left * feat(lens): track per-stage counts and durations for the progress readout * test(lens): cover stage durations and short time left labels * feat(lens): restyle investigation progress as a terminal-style readout --- .../lens/_components/LensProgress.tsx | 155 ++++++++++++------ .../lens/_components/lensData.test.ts | 94 +++++++++++ .../(dashboard)/lens/_components/lensData.ts | 84 +++++++++- 3 files changed, 285 insertions(+), 48 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx index 8249f284ada..3e01671a832 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx @@ -1,11 +1,31 @@ "use client"; import { useEffect, useState } from "react"; -import { Check, Loader2 } from "lucide-react"; import { Button } from "@/components/ui/button"; -import { analysisElapsed, analysisProgress, nextCheckStatus, type Lens, type Job } from "./lensData"; +import { + analysisElapsed, + analysisFraction, + analysisPace, + analysisProgress, + analysisStages, + durationText, + nextCheckStatus, + remainingLabel, + stageDurations, + type Lens, + type Job, + type ProgressSample, +} from "./lensData"; -const steps = ["Review runs", "Find patterns", "Check evidence"]; +const steps = ["review runs", "find patterns", "check evidence"]; +const markers = { done: "✓", active: "▸", todo: "·" }; +const rowText = { done: "text-foreground", active: "font-medium text-foreground", todo: "text-muted-foreground" }; +const blocks = 32; + +function stageState(index: number, current: number): keyof typeof markers { + if (index < current) return "done"; + return index === current ? "active" : "todo"; +} export function LensProgress({ job, onCancel }: { job: Job; onCancel?: () => void }) { const [now, setNow] = useState(Date.now); @@ -14,57 +34,98 @@ export function LensProgress({ job, onCancel }: { job: Job; onCancel?: () => voi return () => window.clearInterval(timer); }, []); const progress = analysisProgress(job); - const percent = progress.total ? Math.min(100, (progress.done / progress.total) * 100) : undefined; + const fraction = analysisFraction(progress); + const [samples, setSamples] = useState([]); + const latest = samples.at(-1); + if (!latest || latest.step !== progress.step || latest.done !== progress.done) { + setSamples([...samples, { at: now, step: progress.step, done: progress.done, fraction }].slice(-120)); + } + const pace = analysisPace(samples, now); + const percent = Math.round(fraction * 100); + const queued = progress.step < 0; + const counts = analysisStages(job); + const durations = stageDurations(samples, job.created_at, now); + const filled = Math.round(fraction * blocks); + const stats = [ + ["eta", remainingLabel(pace.secondsLeft)], + ["rate", pace.perMinute === null ? "–" : `${Math.round(pace.perMinute)}/min`], + ["elapsed", analysisElapsed(job.created_at, now)], + ]; return ( -
-
-
-
- - {analysisElapsed(job.created_at, now)} elapsed +
+
+ + {progress.title} + {!queued && · {progress.detail}} -
-
    - {steps.map((label, index) => ( -
  1. -
    - - {index < progress.step && } - {label} - -
    2. - ))} -
-
-

{progress.detail}

-
-
-
-
-
- {job.status === "running" && You can leave this page while the investigation runs.} {onCancel && ( - )}
+
+
+ +
+ {Array.from({ length: blocks }, (_, index) => ( + + ))} +
+ + {percent}% +
+
    + {steps.map((label, index) => { + const state = stageState(index, progress.step); + const { done, total } = counts[index]; + const seconds = durations[index]; + return ( +
  1. + + {label} + + {state === "todo" || !total ? "–" : `${Math.min(done, total)}/${total}`} + + {seconds === null ? "" : durationText(seconds)} +
    2. + ); + })} +
+
+ {queued ? ( + {progress.detail} + ) : ( + stats.map(([key, value]) => ( + + {key} {value} + + )) + )} +
+
); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts index 3be6164fce2..25276fe5a44 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts @@ -4,6 +4,10 @@ import { workerConnected, type LensList, analysisProgress, + analysisFraction, + analysisPace, + remainingLabel, + stageDurations, normalizeFilters, sortedFindings, type Finding, @@ -136,6 +140,96 @@ describe("Analysis progress", () => { }); }); +describe("Analysis pace", () => { + it("fills the bar left to right across stages without jumping backwards at a stage boundary", () => { + const endOfReview = analysisFraction({ step: 0, done: 20, total: 20 }); + const startOfGrouping = analysisFraction({ step: 1, done: 0, total: 4 }); + expect(analysisFraction({ step: -1, done: 0, total: 0 })).toBe(0); + expect(analysisFraction({ step: 0, done: 10, total: 20 })).toBeLessThan(endOfReview); + expect(startOfGrouping).toBeCloseTo(endOfReview); + expect(analysisFraction({ step: 2, done: 5, total: 5 })).toBeCloseTo(1); + }); + + it("measures speed within the current stage and projects time left from overall progress", () => { + const start = Date.parse("2026-09-30T12:00:00Z"); + const pace = analysisPace( + [ + { at: start, step: 0, done: 0, fraction: 0 }, + { at: start + 30000, step: 0, done: 30, fraction: 0.25 }, + ], + start + 30000, + ); + expect(pace.perMinute).toBe(60); + expect(pace.secondsLeft).toBe(90); + }); + + it("waits for enough samples instead of showing a wild first estimate", () => { + const start = Date.parse("2026-09-30T12:00:00Z"); + expect(analysisPace([{ at: start, step: 0, done: 1, fraction: 0.01 }], start + 2000)).toEqual({ + perMinute: null, + secondsLeft: null, + }); + expect(remainingLabel(null)).toBe("estimating"); + }); + + it("lengthens the estimate while progress stalls", () => { + const start = Date.parse("2026-09-30T12:00:00Z"); + const samples = [ + { at: start, step: 0, done: 0, fraction: 0 }, + { at: start + 10000, step: 0, done: 10, fraction: 0.1 }, + ]; + const moving = analysisPace(samples, start + 10000).secondsLeft ?? 0; + const stalled = analysisPace(samples, start + 40000).secondsLeft ?? 0; + expect(stalled).toBeGreaterThan(moving); + }); + + it("still estimates when the worker reports progress less than once a minute", () => { + const start = Date.parse("2026-09-30T12:00:00Z"); + const pace = analysisPace( + [ + { at: start, step: 2, done: 0, fraction: 0.8 }, + { at: start + 90000, step: 2, done: 1, fraction: 0.85 }, + ], + start + 90000, + ); + expect(pace.secondsLeft).toBeCloseTo(270); + expect(pace.perMinute).toBeCloseTo(2 / 3); + }); + + it("measures from the last minute rather than the whole run once updates are frequent", () => { + const start = Date.parse("2026-09-30T12:00:00Z"); + const samples = [ + { at: start, step: 0, done: 0, fraction: 0 }, + { at: start + 120000, step: 0, done: 12, fraction: 0.06 }, + { at: start + 180000, step: 0, done: 72, fraction: 0.36 }, + ]; + expect(analysisPace(samples, start + 180000).perMinute).toBe(60); + }); + + it("rounds remaining time up so the label never promises less than the estimate", () => { + expect(remainingLabel(61)).toBe("~2m"); + expect(remainingLabel(30)).toBe("<1m"); + }); +}); + +describe("Stage durations", () => { + const start = Date.parse("2026-09-30T12:00:00Z"); + const createdAt = "2026-09-30T12:00:00Z"; + + it("times finished stages from the transitions it saw and the active stage up to now", () => { + const samples = [ + { at: start + 5000, step: 0, done: 10, fraction: 0.1 }, + { at: start + 124000, step: 1, done: 0, fraction: 0.6 }, + ]; + expect(stageDurations(samples, createdAt, start + 145000)).toEqual([124, 21, null]); + }); + + it("does not guess when a stage started before the page was opened", () => { + const samples = [{ at: start + 90000, step: 1, done: 2, fraction: 0.7 }]; + expect(stageDurations(samples, createdAt, start + 100000)).toEqual([null, null, null]); + }); +}); + describe("Lens selection and findings", () => { it("preserves literal equals signs in a metadata value", () => { expect(normalizeFilters([{ key: " swarm ", value: " research=v2 " }])).toEqual([ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts index 1e30dc75a38..7aa2df9edbe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts @@ -126,8 +126,90 @@ export function analysisProgress(job: Job) { }; } +export function analysisStages(job: Job): { done: number; total: number }[] { + const { + screened = 0, + selected = 0, + grouped_batches = 0, + grouping_batches = 0, + investigated = 0, + candidates = 0, + } = job.coverage ?? {}; + return [ + { done: screened, total: selected }, + { done: grouped_batches, total: grouping_batches }, + { done: investigated, total: candidates }, + ]; +} + +export interface ProgressSample { + at: number; + step: number; + done: number; + fraction: number; +} + +export const stageWeights = [0.6, 0.2, 0.2]; + +export function analysisFraction({ + step, + done, + total, +}: Pick, "step" | "done" | "total">): number { + if (step < 0) return 0; + const before = stageWeights.slice(0, step).reduce((sum, weight) => sum + weight, 0); + return before + stageWeights[step] * (total ? Math.min(1, done / total) : 0); +} + +function windowStart(samples: readonly ProgressSample[], now: number): ProgressSample | undefined { + return samples.findLast((sample) => now - sample.at >= 60000) ?? samples[0]; +} + +export function analysisPace(samples: readonly ProgressSample[], now: number) { + const latest = samples.at(-1); + if (!latest) return { perMinute: null, secondsLeft: null }; + const first = windowStart( + samples.filter((sample) => sample.step === latest.step), + now, + ); + const anchor = windowStart(samples, now); + if (!first || !anchor) return { perMinute: null, secondsLeft: null }; + const stepMinutes = (now - first.at) / 60000; + const perMinute = stepMinutes >= 1 / 6 ? (latest.done - first.done) / stepMinutes : null; + const spanSeconds = (now - anchor.at) / 1000; + const gained = latest.fraction - anchor.fraction; + const secondsLeft = spanSeconds >= 10 && gained > 0 ? ((1 - latest.fraction) * spanSeconds) / gained : null; + return { perMinute, secondsLeft }; +} + +export function stageDurations(samples: readonly ProgressSample[], createdAt: string, now: number): (number | null)[] { + const current = samples.at(-1)?.step ?? -1; + const starts = [0, 1, 2].map((stage) => { + if (stage === 0) return Date.parse(createdAt); + const entered = samples.findIndex( + (sample, index) => index > 0 && sample.step >= stage && samples[index - 1].step < stage, + ); + return entered < 0 ? null : samples[entered].at; + }); + return starts.map((start, stage) => { + if (start === null || stage > current) return null; + const end = stage === current ? now : starts[stage + 1]; + return end === null ? null : Math.max(0, Math.floor((end - start) / 1000)); + }); +} + +export function remainingLabel(seconds: number | null): string { + if (seconds === null) return "estimating"; + if (seconds < 60) return "<1m"; + if (seconds < 3600) return `~${Math.ceil(seconds / 60)}m`; + return `~${Math.floor(seconds / 3600)}h ${Math.ceil((seconds % 3600) / 60)}m`; +} + export function analysisElapsed(createdAt: string, now: number): string { - const seconds = Math.max(0, Math.floor((now - Date.parse(createdAt)) / 1000)); + return durationText(Math.max(0, Math.floor((now - Date.parse(createdAt)) / 1000))); +} + +export function durationText(seconds: number): string { if (!Number.isFinite(seconds)) return "0s"; if (seconds < 60) return `${seconds}s`; if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`; From 4b9f9903f33310a416a5ae6ff79a4cdbd66a06e7 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:35:34 +0000 Subject: [PATCH 07/42] refactor(ui): compose dashboard pages with shared layouts (#44306) * refactor(ui): compose logs tabs directly in the route * feat(ui): share composable dashboard page layouts * refactor(ui): compose page header and logs toolbar from parts PageHeader drops its icon/title/subtitle/primaryAction/tabs/utilities props and the leadingControls render prop in favor of PageHeaderTitle, PageHeaderDescription and PageHeaderControls that each wrap one element and forward native props. LogsTableToolbar's 15 props collapse into one LogsTimeRange value plus composable LogsToolbar, LogsTimeRangePicker and LogsToolbarSwitch parts assembled in the panel. Co-Authored-By: Claude Opus 5.5 * refactor(ui): express DataTable layout classes as cva variants Replaces the hand-rolled class-pair constants with boolean cva variants, which also brings DataTable back under the complexity budget. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Yujong Lee Co-authored-by: Claude Opus 5.5 --- tests/e2e/ui/tests/logs/logs.spec.ts | 69 +++++++++ .../_components/AccessGroupsPage.tsx | 81 ++++++----- .../(dashboard)/api-keys/ApiKeysDashboard.tsx | 5 +- .../budgets/_components/budget_panel.tsx | 61 ++++---- .../_components/CostOptimizationView.tsx | 56 ++++---- .../_components/GuardrailsMonitorView.tsx | 5 +- .../_components/GuardrailsOverview.tsx | 30 ++-- .../src/app/(dashboard)/layout.tsx | 2 +- .../logs/page.integration.test.tsx} | 18 +-- .../(dashboard)/logs/page.test.tsx} | 28 ++-- .../src/app/(dashboard)/logs/page.tsx | 75 ++++++++-- .../_components/ModelInsightsView.tsx | 21 +-- .../projects/_components/ProjectsPage.tsx | 77 +++++----- .../_components/ROICalculatorView.tsx | 53 +++---- .../DeletedKeysPage/DeletedKeysPage.tsx | 7 +- .../DeletedKeysTable/DeletedKeysTable.tsx | 1 + .../DeletedTeamsPage/DeletedTeamsPage.tsx | 7 +- .../DeletedTeamsTable/DeletedTeamsTable.tsx | 1 + .../src/components/Teams.test.tsx | 10 -- ui/litellm-dashboard/src/components/Teams.tsx | 63 ++++---- .../VirtualKeysPage/VirtualKeysTable.tsx | 21 +-- .../shared/DataTable/DataTable.test.tsx | 22 --- .../components/shared/DataTable/DataTable.tsx | 94 +++++++----- .../src/components/shared/Page.tsx | 40 ++++++ .../src/components/shared/PageHeader.test.tsx | 99 +++++-------- .../src/components/shared/PageHeader.tsx | 78 ++++------ .../components/view_logs/AuditLogsPanel.tsx | 6 +- .../components/view_logs/AuditLogsTable.tsx | 1 + .../components/view_logs/LogsTableToolbar.tsx | 135 +++++++----------- .../view_logs/RequestLogsPanel.test.tsx | 8 +- .../components/view_logs/RequestLogsPanel.tsx | 68 ++++----- .../src/components/view_logs/index.tsx | 105 -------------- 32 files changed, 661 insertions(+), 686 deletions(-) rename ui/litellm-dashboard/src/{components/view_logs/index.integration.test.tsx => app/(dashboard)/logs/page.integration.test.tsx} (90%) rename ui/litellm-dashboard/src/{components/view_logs/index.test.tsx => app/(dashboard)/logs/page.test.tsx} (88%) create mode 100644 ui/litellm-dashboard/src/components/shared/Page.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/index.tsx diff --git a/tests/e2e/ui/tests/logs/logs.spec.ts b/tests/e2e/ui/tests/logs/logs.spec.ts index 60b547ccda0..688bfe8e3b4 100644 --- a/tests/e2e/ui/tests/logs/logs.spec.ts +++ b/tests/e2e/ui/tests/logs/logs.spec.ts @@ -54,6 +54,75 @@ test.describe("Logs page", () => { permissions: ["clipboard-read", "clipboard-write"], }); + test("log tables fill the available height and empty requests stay centered after resizing", async ({ + page, + }) => { + await navigateToPage(page, Page.Logs); + await dismissFeedbackPopup(page); + await visibleTestId(page, "datatable-search").fill( + `missing-request-${uniqueSuffix()}`, + ); + const emptyTitle = page.getByText("No matching requests", { exact: true }); + await expect(emptyTitle).toBeVisible(); + + for (const viewport of [ + { width: 1440, height: 900 }, + { width: 1024, height: 720 }, + ]) { + await page.setViewportSize(viewport); + await expect + .poll(async () => { + const frame = await visibleTestId( + page, + "data-table-frame", + ).boundingBox(); + return frame + ? Math.abs(viewport.height - frame.y - frame.height - 24) + : Infinity; + }) + .toBeLessThanOrEqual(2); + await expect + .poll(async () => { + const body = await page + .locator("table") + .filter({ visible: true }) + .first() + .locator("tbody") + .boundingBox(); + const scroller = await visibleTestId( + page, + "data-table-scroller", + ).boundingBox(); + const message = await emptyTitle.locator("..").boundingBox(); + if (!body || !message || !scroller) return Infinity; + return Math.max( + Math.abs( + message.x + message.width / 2 - scroller.x - scroller.width / 2, + ), + Math.abs(message.y + message.height / 2 - body.y - body.height / 2), + ); + }) + .toBeLessThanOrEqual(4); + for (const tab of ["Deleted Keys", "Deleted Teams"]) { + await page.getByRole("tab", { name: tab, exact: true }).click(); + await expect + .poll(async () => { + const frame = await visibleTestId( + page, + "data-table-frame", + ).boundingBox(); + return frame + ? Math.abs(viewport.height - frame.y - frame.height - 24) + : Infinity; + }) + .toBeLessThanOrEqual(2); + } + await page + .getByRole("tab", { name: "Request Logs", exact: true }) + .click(); + } + }); + test("a chat sent from the Playground lands in Logs with its content", async ({ page, request }) => { const prompt = `logs-playground-prompt-${uniqueSuffix()}`; await openPlayground(page); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx index 2e82fe3c418..ac40a28a258 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/AccessGroupsPage.tsx @@ -1,9 +1,10 @@ +import { Page, PageContent } from "@/components/shared/Page"; import { AccessGroupResponse, useAccessGroups } from "@/app/(dashboard)/hooks/accessGroups/useAccessGroups"; import { useDeleteAccessGroup } from "@/app/(dashboard)/hooks/accessGroups/useDeleteAccessGroup"; import { Boxes, Plus, SearchIcon, X } from "lucide-react"; import { useMemo, useState } from "react"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { AccessGroupDetail } from "./AccessGroupsDetailsPage"; @@ -59,49 +60,53 @@ export function AccessGroupsPage() { } return ( -
- } - title="Access Groups" - subtitle="Manage resource permissions for your organization" - primaryAction={ - canModify ? ( + + + + + Access Groups + + Manage resource permissions for your organization + {canModify && ( + - ) : undefined - } - /> + + )} + -
- - - - - setSearchText(e.target.value)} - /> - {searchText && ( - - setSearchText("")}> - - + +
+ + + - )} - -
+ setSearchText(e.target.value)} + /> + {searchText && ( + + setSearchText("")}> + + + + )} + +
- 0} - canModify={canModify} - onGroupClick={setSelectedGroupId} - onDeleteClick={setGroupToDelete} - /> + 0} + canModify={canModify} + onGroupClick={setSelectedGroupId} + onDeleteClick={setGroupToDelete} + /> + @@ -126,6 +131,6 @@ export function AccessGroupsPage() { }} confirmLoading={deleteMutation.isPending} /> -
+ ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/api-keys/ApiKeysDashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/api-keys/ApiKeysDashboard.tsx index 376fee72b88..915df2f5ded 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/api-keys/ApiKeysDashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/api-keys/ApiKeysDashboard.tsx @@ -1,5 +1,6 @@ "use client"; +import { Page } from "@/components/shared/Page"; import { teamListCall as v2TeamListCall } from "@/app/(dashboard)/hooks/teams/useTeams"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { KeyResponse, Team } from "@/components/key_team_helpers/key_list"; @@ -71,7 +72,7 @@ export default function ApiKeysDashboard() { }, [accessToken, userID, userRole]); return ( -
+ -
+ ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx index 7455c252e26..630d90e91b9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx @@ -3,13 +3,15 @@ * */ +import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; import { Plus, Wallet } from "lucide-react"; import React, { useCallback, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { prism } from "react-syntax-highlighter/dist/esm/styles/prism"; import { useSyntaxTheme } from "@/hooks/useSyntaxTheme"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; +import { ToolbarSeparator } from "@/components/shared/ToolbarSeparator"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; @@ -78,35 +80,30 @@ const BudgetPanel: React.FC = ({ accessToken }) => { }; return ( -
- - } - title="Budgets" - subtitle="Spend, TPM and RPM limits you can assign to customers." - primaryAction={ - canModify ? ( - - ) : undefined - } - tabs={({ leadingControls }) => ( - - {leadingControls} - - Budgets - - - Examples - - - )} - /> + + + + + + Budgets + + Spend, TPM and RPM limits you can assign to customers. + + + {canModify && ( + <> + + + + )} + Budgets + Examples + + +
@@ -174,8 +171,8 @@ const BudgetPanel: React.FC = ({ accessToken }) => {
- -
+ + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx index 0aa4f88495a..6a4c49963df 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/CostOptimizationView.tsx @@ -1,12 +1,13 @@ "use client"; +import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; import React from "react"; import { Info, PiggyBank } from "lucide-react"; import useCan from "@/app/(dashboard)/hooks/useCan"; import { Alert, AlertDescription } from "@/components/shared/Alert"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { TabsContent } from "@/components/ui/tabs"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import UsageTab from "./UsageTab"; import PromptCompressionTab from "./PromptCompressionTab"; import PromptCachingTab from "./PromptCachingTab"; @@ -33,37 +34,30 @@ const CostOptimizationView: React.FC = ({ accessToken }; return ( -
- - } - title="Cost Optimization" - subtitle="Track and configure the mechanisms that save you money: prompt compression and prompt caching. Auto routers live under Models + Endpoints, on the Auto-Routers tab" - tabs={({ leadingControls }) => ( - - {leadingControls} - - Overall - + + + + + + Cost Optimization + + + Track and configure the mechanisms that save you money: prompt compression and prompt caching. Auto routers + live under Models + Endpoints, on the Auto-Routers tab + + + + Overall {canViewProxyWideCostData && ( <> - - Prompt Compression - - - Prompt Caching - - - Auto-Router - + Prompt Compression + Prompt Caching + Auto-Router )} - - )} - /> + + +
= ({ accessToken )} - -
+ + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx index f90a46e19e4..1dd6686d7fe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsMonitorView.tsx @@ -1,3 +1,4 @@ +import { Page } from "@/components/shared/Page"; import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; import { parseAsString, useQueryState } from "nuqs"; import React, { useCallback, useMemo, useState } from "react"; @@ -48,7 +49,7 @@ export default function GuardrailsMonitorView({ accessToken = null }: Guardrails ); return ( -
+ {!selectedGuardrailId ? ( )} -
+ ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx index 468e6967d81..5627e7fc3cb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailsOverview.tsx @@ -18,7 +18,7 @@ import { type UsageUnits, } from "@/components/GuardrailsMonitor/usageUnits"; import { Button } from "@/components/ui/button"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; import { EvaluationSettingsModal } from "./EvaluationSettingsModal"; import { MetricCard } from "@/components/GuardrailsMonitor/MetricCard"; @@ -282,20 +282,20 @@ export function GuardrailsOverview({ return (
- } - title="Guardrails Monitor" - subtitle="Monitor guardrail performance across all requests" - utilities={ - <> - {dateRangeControl} - - - } - /> + + + + Guardrails Monitor + + Monitor guardrail performance across all requests + + {dateRangeControl} + + +
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx index 72f26919060..9d8718cb84a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/layout.tsx @@ -149,7 +149,7 @@ function DashboardShell({ children }: { children: React.ReactNode }) { -
{children}
+
{children}
{!isPlayground && }
diff --git a/ui/litellm-dashboard/src/components/view_logs/index.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.integration.test.tsx similarity index 90% rename from ui/litellm-dashboard/src/components/view_logs/index.integration.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/logs/page.integration.test.tsx index f2d70b74c96..dc8a57e50ba 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.integration.test.tsx @@ -1,8 +1,8 @@ import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import SpendLogsTable from "./index"; -import { renderWithProviders, testQueryClient } from "../../../tests/test-utils"; +import LogsPage from "./page"; +import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; const { useAuthorizedMock, useOrganizationsMock } = vi.hoisted(() => ({ useAuthorizedMock: vi.fn(), @@ -17,7 +17,7 @@ vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ useOrganizations: useOrganizationsMock, })); -vi.mock("./RequestLogsPanel", () => ({ +vi.mock("@/components/view_logs/RequestLogsPanel", () => ({ default: function RequestLogsPanelMock() { return
; }, @@ -47,17 +47,13 @@ const defaultProps = { const ORG_ADMIN_MEMBERSHIPS = [{ organization_id: "org-1", members: [{ user_id: "user-1", user_role: "org_admin" }] }]; const renderAs = (sessionRole: string, organizations: unknown[] = []) => { - useAuthorizedMock.mockReturnValue({ - accessToken: "sk-test", - userId: "user-1", - userRole: sessionRole, - premiumUser: true, - }); + const session = { ...defaultProps, userId: defaultProps.userID, userRole: sessionRole }; + useAuthorizedMock.mockReturnValue(session); useOrganizationsMock.mockReturnValue({ data: organizations }); - return renderWithProviders(); + return renderWithProviders(); }; -describe("SpendLogsTable network access by role", () => { +describe("LogsPage network access by role", () => { beforeEach(() => { testQueryClient.clear(); vi.clearAllMocks(); diff --git a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.test.tsx similarity index 88% rename from ui/litellm-dashboard/src/components/view_logs/index.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/logs/page.test.tsx index 7c8c14771bd..c7e0a82a363 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.test.tsx @@ -1,8 +1,8 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import SpendLogsTable from "./index"; -import { renderWithProviders } from "../../../tests/test-utils"; +import LogsPage from "./page"; +import { renderWithProviders } from "../../../../tests/test-utils"; const { useAuthorizedMock, useOrganizationsMock } = vi.hoisted(() => ({ useAuthorizedMock: vi.fn(), @@ -17,25 +17,25 @@ vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ useOrganizations: useOrganizationsMock, })); -vi.mock("./RequestLogsPanel", () => ({ +vi.mock("@/components/view_logs/RequestLogsPanel", () => ({ default: function RequestLogsPanelMock({ isActive }: { isActive: boolean }) { return
{isActive ? "active" : "inactive"}
; }, })); -vi.mock("./AuditLogsPanel", () => ({ +vi.mock("@/components/view_logs/AuditLogsPanel", () => ({ default: function AuditLogsPanelMock({ isActive }: { isActive: boolean }) { return
{isActive ? "active" : "inactive"}
; }, })); -vi.mock("../DeletedKeysPage/DeletedKeysPage", () => ({ +vi.mock("@/components/DeletedKeysPage/DeletedKeysPage", () => ({ default: function DeletedKeysPageMock() { return
; }, })); -vi.mock("../DeletedTeamsPage/DeletedTeamsPage", () => ({ +vi.mock("@/components/DeletedTeamsPage/DeletedTeamsPage", () => ({ default: function DeletedTeamsPageMock() { return
; }, @@ -52,16 +52,16 @@ const defaultProps = { const ORG_ADMIN_MEMBERSHIPS = [{ organization_id: "org-1", members: [{ user_id: "user-1", user_role: "org_admin" }] }]; const renderAs = (sessionRole: string, organizations: unknown[] = []) => { - useAuthorizedMock.mockReturnValue({ userId: "user-1", userRole: sessionRole }); + useAuthorizedMock.mockReturnValue({ ...defaultProps, userId: defaultProps.userID, userRole: sessionRole }); useOrganizationsMock.mockReturnValue({ data: organizations }); - return renderWithProviders(); + return renderWithProviders(); }; const tabNames = () => screen.getAllByRole("tab").map((tab) => tab.textContent); -describe("SpendLogsTable", () => { +describe("LogsPage", () => { beforeEach(() => { - useAuthorizedMock.mockReturnValue({ userId: "user-1", userRole: "Admin" }); + useAuthorizedMock.mockReturnValue({ ...defaultProps, userId: defaultProps.userID }); useOrganizationsMock.mockReturnValue({ data: [] }); }); @@ -182,17 +182,17 @@ describe("SpendLogsTable", () => { describe("auth-not-ready guard", () => { it("shows a loading spinner when credentials are not yet resolved", () => { - useAuthorizedMock.mockReturnValue({ userRole: "Admin" }); - renderWithProviders(); + useAuthorizedMock.mockReturnValue({ ...defaultProps, userId: defaultProps.userID, accessToken: null }); + renderWithProviders(); - expect(document.querySelector('[aria-busy="true"]')).toBeInTheDocument(); + expect(screen.getByRole("status", { name: "Loading" })).toHaveAttribute("aria-busy", "true"); expect(screen.queryByRole("tab", { name: "Request Logs" })).not.toBeInTheDocument(); }); it("renders the tabs (no spinner) once all credentials are present", () => { renderAs("Admin"); - expect(document.querySelector('[aria-busy="true"]')).not.toBeInTheDocument(); + expect(screen.queryByRole("status", { name: "Loading" })).not.toBeInTheDocument(); expect(screen.getByRole("tab", { name: "Request Logs" })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx index 88909e3b87f..51603dec29e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/logs/page.tsx @@ -1,17 +1,74 @@ "use client"; -import SpendLogsTable from "@/components/view_logs"; +import { useState } from "react"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import useCan from "@/app/(dashboard)/hooks/useCan"; +import DeletedKeysPage from "@/components/DeletedKeysPage/DeletedKeysPage"; +import DeletedTeamsPage from "@/components/DeletedTeamsPage/DeletedTeamsPage"; +import AuditLogsPanel from "@/components/view_logs/AuditLogsPanel"; +import RequestLogsPanel from "@/components/view_logs/RequestLogsPanel"; +import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; -export default function Logs() { +type LogsTab = "request logs" | "audit logs" | "deleted keys" | "deleted teams"; + +export default function LogsPage() { const { accessToken, userRole, userId, token, premiumUser } = useAuthorized(); + const [activeTab, setActiveTab] = useState("request logs"); + const canViewAuditLogs = useCan("viewAuditLogs"); + const canViewDeletedTeams = useCan("viewDeletedTeams"); + + const credentialsPending = !accessToken || !token; + const identityPending = !userRole || !userId; + + if (credentialsPending || identityPending) { + return ( +
+ +
+ ); + } + return ( - + + setActiveTab(value)}> + + Request Logs + {canViewAuditLogs && Audit Logs} + Deleted Keys + {canViewDeletedTeams && Deleted Teams} + + + + + + {canViewAuditLogs && ( + + + + )} + + + + {canViewDeletedTeams && ( + + + + )} + + ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/ModelInsightsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/ModelInsightsView.tsx index c946e8aabe0..ab406597589 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/ModelInsightsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/model-insights/_components/ModelInsightsView.tsx @@ -1,5 +1,6 @@ "use client"; +import { Page } from "@/components/shared/Page"; import React from "react"; import { Bar, BarChart, CartesianGrid, Treemap, XAxis, YAxis } from "recharts"; import { ArrowDownRight, ArrowUpRight, BarChart3, Layers, Minus } from "lucide-react"; @@ -7,7 +8,7 @@ import { ArrowDownRight, ArrowUpRight, BarChart3, Layers, Minus } from "lucide-r import { apiClient } from "@/components/networking"; import { extractErrorMessage } from "@/utils/errorUtils"; import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageHeader, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { ChartConfig, ChartContainer, ChartTooltip, ChartTooltipContent } from "@/components/ui/chart"; @@ -210,12 +211,16 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string ) satisfies ChartConfig; return ( -
- } - title="Model Leaderboard" - subtitle={`See which models your gateway used from ${data.start_date} through ${data.end_date}`} - /> + + + + + Model Leaderboard + + + See which models your gateway used from {data.start_date} through {data.end_date} + + @@ -375,6 +380,6 @@ export default function ModelInsightsView({ accessToken }: { accessToken: string

-
+ ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx index 2d3c1acf75e..cb595916041 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/projects/_components/ProjectsPage.tsx @@ -1,9 +1,10 @@ +import { Page, PageContent } from "@/components/shared/Page"; import { useProjects } from "@/app/(dashboard)/hooks/projects/useProjects"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { Folder, Plus, SearchIcon, X } from "lucide-react"; import { parseAsString, useQueryState } from "nuqs"; import { useMemo, useState } from "react"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { CreateProjectModal } from "./ProjectModals/CreateProjectModal"; @@ -56,49 +57,53 @@ export function ProjectsPage() { } return ( -
- } - title="Projects" - subtitle="Manage projects within your teams" - primaryAction={ + + + + + Projects + + Manage projects within your teams + - } - /> + + -
- - - - - setSearchText(e.target.value)} - /> - {searchText && ( - - setSearchText("")}> - - + +
+ + + - )} - -
+ setSearchText(e.target.value)} + /> + {searchText && ( + + setSearchText("")}> + + + + )} + +
- 0} - onProjectClick={(id) => void setSelectedProjectId(id)} - teamAliasMap={teamAliasMap} - isTeamsLoading={isTeamsLoading} - /> + 0} + onProjectClick={(id) => void setSelectedProjectId(id)} + teamAliasMap={teamAliasMap} + isTeamsLoading={isTeamsLoading} + /> + setIsCreateModalVisible(false)} /> -
+ ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx index 40c1104354d..2b17d2019ee 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx @@ -1,11 +1,12 @@ "use client"; +import { Page } from "@/components/shared/Page"; import React from "react"; import { Calculator, RefreshCw } from "lucide-react"; import { apiClient } from "@/components/networking"; -import { PageHeader } from "@/components/shared/PageHeader"; import { DemoNotice } from "@/components/shared/DemoNotice"; +import { PageHeader, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; @@ -219,33 +220,33 @@ export default function ROICalculatorView({ const syncedAt = syncIsUpToDate ? summary?.synced_at : null; return ( -
- } - title="ROI Calculator" - titleAction={ - !liveSummary && showLiveStatus ? ( + + +
+ + + ROI Calculator + + {!liveSummary && showLiveStatus && ( - ) : undefined - } - subtitle={ - <> - {summary - ? `${summary.start} through ${summary.end} · UTC` - : "Compare gateway spend with estimated engineering effort for merged pull requests"} - {syncedAt && ( - - Last synced {formatSyncedAt(syncedAt)} - {!status.running && status.phase === "complete" && status.reused > 0 - ? ` · ${status.reused} of ${status.total} estimates reused` - : ""} - - )} - - } - /> + )} +
+ + {summary + ? `${summary.start} through ${summary.end} · UTC` + : "Compare gateway spend with estimated engineering effort for merged pull requests"} + {syncedAt && ( + + Last synced {formatSyncedAt(syncedAt)} + {!status.running && status.phase === "complete" && status.reused > 0 + ? ` · ${status.reused} of ${status.total} estimates reused` + : ""} + + )} + + {sampleSummary && setSampleSummary(null)} />} {liveSummary && showLiveStatus && (

@@ -365,6 +366,6 @@ export default function ROICalculatorView({ onSave={updateIdentity} /> )} -

+ ); } diff --git a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.tsx b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.tsx index 0af668b5e9c..2686806c95b 100644 --- a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.tsx +++ b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysPage.tsx @@ -1,4 +1,5 @@ "use client"; +import { PageContent } from "@/components/shared/Page"; import { useState } from "react"; import { PaginationState } from "@tanstack/react-table"; import { Info } from "lucide-react"; @@ -14,9 +15,9 @@ export default function DeletedKeysPage() { const { data: keysData, isLoading } = useDeletedKeys(pagination.pageIndex + 1, pagination.pageSize); return ( -
+ {!premiumUser && ( - + Coming soon to Enterprise @@ -31,6 +32,6 @@ export default function DeletedKeysPage() { pagination={pagination} onPaginationChange={setPagination} /> -
+ ); } diff --git a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx index bc6941a7860..afe507366eb 100644 --- a/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/DeletedKeysPage/DeletedKeysTable/DeletedKeysTable.tsx @@ -44,6 +44,7 @@ export function DeletedKeysTable({ return ( key.token || String(index)} diff --git a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.tsx b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.tsx index 8c3aac2cac7..a491fe138f6 100644 --- a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.tsx +++ b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsPage.tsx @@ -1,6 +1,7 @@ "use client"; import { PaginationState } from "@tanstack/react-table"; import { Info } from "lucide-react"; +import { PageContent } from "@/components/shared/Page"; import { useState } from "react"; import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { DEFAULT_PAGE_SIZE_OPTIONS } from "@/components/shared/DataTable"; @@ -17,9 +18,9 @@ export default function DeletedTeamsPage() { const { data: teamsData, isLoading } = useDeletedTeams(pagination.pageIndex + 1, pagination.pageSize); return ( -
+ {!premiumUser && ( - + Coming soon to Enterprise @@ -34,6 +35,6 @@ export default function DeletedTeamsPage() { onPaginationChange={setPagination} rowCount={teamsData?.total ?? 0} /> -
+ ); } diff --git a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx index c7e759754b8..236b8b7cf18 100644 --- a/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx +++ b/ui/litellm-dashboard/src/components/DeletedTeamsPage/DeletedTeamsTable/DeletedTeamsTable.tsx @@ -44,6 +44,7 @@ export function DeletedTeamsTable({ return ( team.team_id || String(index)} diff --git a/ui/litellm-dashboard/src/components/Teams.test.tsx b/ui/litellm-dashboard/src/components/Teams.test.tsx index 5ff95d2af0c..3b89ebdd318 100644 --- a/ui/litellm-dashboard/src/components/Teams.test.tsx +++ b/ui/litellm-dashboard/src/components/Teams.test.tsx @@ -529,15 +529,6 @@ describe("Teams - team detail deep link (?team=)", () => { expect(onUrlUpdate.mock.calls.at(-1)![0].searchParams.has("team")).toBe(false); await waitFor(() => expect(screen.queryByTestId("team-info-view")).not.toBeInTheDocument()); }); - - it("should preserve the legacy inset for the team detail view", async () => { - renderWithQueryClient(, { - searchParams: "?team=team-from-url", - }); - - await waitFor(() => expect(mockTeamInfoView).toHaveBeenCalled()); - expect(screen.getByRole("main")).toHaveClass("px-12", "py-6"); - }); }); describe("Teams - Create Team CTA is grouped with the tabs on the left", () => { @@ -553,7 +544,6 @@ describe("Teams - Create Team CTA is grouped with the tabs on the left", () => { const createButton = within(tabNav).getByTestId("create-team-button"); const firstTab = within(tabNav).getByRole("tab", { name: "Your Teams" }); - expect(screen.getByRole("main")).toHaveClass("p-8"); expect(within(tabNav).getByRole("separator")).toBeInTheDocument(); expect(createButton.compareDocumentPosition(firstTab) & Node.DOCUMENT_POSITION_FOLLOWING).toBeTruthy(); }); diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index c2a23cef83a..dfc0a59f0a0 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -1,3 +1,4 @@ +import { Page, PageTabs, PageTabsList, PageTabsTrigger } from "@/components/shared/Page"; import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import useCan from "@/app/(dashboard)/hooks/useCan"; import AvailableTeamsPanel from "@/components/team/AvailableTeamsPanel"; @@ -15,12 +16,13 @@ import { SearchSelect } from "@/components/shared/SearchSelect"; import { labelWithDocsHint, labelWithHint } from "@/components/shared/form/LabelWithHint"; import { useZodForm } from "@/lib/forms/useZodForm"; import { TagsInput } from "@/app/(dashboard)/guardrails/_components/content_filter/TagsInput"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { TabsContent } from "@/components/ui/tabs"; import { ChevronDown, Plus, Users } from "lucide-react"; import React, { useEffect, useMemo, useState } from "react"; import { z } from "zod/v4"; import { useQuery, useQueryClient } from "@tanstack/react-query"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; +import { ToolbarSeparator } from "@/components/shared/ToolbarSeparator"; import { Button as UIButton } from "@/components/ui/button"; import { teamsTableKeys } from "@/app/(dashboard)/hooks/teams/useTeams"; import { parseAsString, useQueryState } from "nuqs"; @@ -652,7 +654,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser ]; return ( -
+ {selectedTeamId ? ( = ({ accessToken, userID, userRole, premiumUser premiumUser={premiumUser} /> ) : ( - - } - title="Teams" - subtitle="Manage teams, members, and their access to models and budgets" - primaryAction={ - canCreateOrManageTeams(userRole, userID, organizations) ? ( - - - Create Team - - ) : undefined - } - tabs={({ leadingControls }) => ( - - {leadingControls} + + + + + Teams + + Manage teams, members, and their access to models and budgets + + + {canCreateOrManageTeams(userRole, userID, organizations) && ( + <> + + + Create Team + + + + )} {tabItems.map((item) => ( - + {item.label} - + ))} - - )} - /> + + + {tabItems.map((item) => ( {item.children} ))} - + )} {canCreateOrManageTeams(userRole, userID, organizations) && ( @@ -1312,7 +1309,7 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser )} -
+ ); }; diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index ec405ee9cad..cc25d22e546 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -16,7 +16,8 @@ import { type UrlTableStateOptions, } from "@/components/shared/DataTable"; import { SearchSelect } from "@/components/shared/SearchSelect"; -import { PageHeader } from "@/components/shared/PageHeader"; +import { PageContent } from "@/components/shared/Page"; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "@/components/shared/PageHeader"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { useDebouncedValue } from "@tanstack/react-pacer/debouncer"; @@ -229,13 +230,15 @@ export function VirtualKeysTable({ headerActions }: VirtualKeysTableProps) { } return ( -
- } - title="Virtual Keys" - subtitle="Every key that authenticates requests to the gateway." - primaryAction={headerActions} - /> + + + + + Virtual Keys + + Every key that authenticates requests to the gateway. + {headerActions != null && {headerActions}} + )} /> -
+ ); } diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index df88287b369..05ba7e04c39 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -899,28 +899,6 @@ describe("DataTable layout", () => { expect(screen.getByTestId("data-table-head")).toHaveClass("sticky", "bg-background"); }); - it("caps fillHeight at the parent's height instead of stretching to it, so a short table stays short", () => { - render(); - const outer = screen.getByTestId("data-table-root"); - const frame = screen.getByTestId("data-table-frame"); - const scroller = screen.getByTestId("data-table-scroller"); - - // A ceiling, not a stretch: flex-1 here would hold the footer at the bottom on a two-row table. - expect(outer).toHaveClass("max-h-full", "flex-col"); - expect(outer).not.toHaveClass("flex-1"); - expect(frame).toHaveClass("flex-col"); - expect(frame).not.toHaveClass("flex-1"); - expect(scroller).not.toHaveClass("flex-1"); - - expect(scroller).toHaveClass("min-h-0", "overflow-auto"); - expect(scroller).toHaveStyle({ maxHeight: "" }); - // Without this the Table primitive's own overflow container captures the sticky header. - expect(scroller).toHaveClass("[&_[data-slot=table-container]]:overflow-visible"); - - // Rows pass under the header, so the semi-transparent row tint alone would let them show through. - expect(screen.getByTestId("data-table-head")).toHaveClass("sticky", "bg-background"); - }); - it("leaves the default layout untouched when neither height mode is set", () => { render(); const scroller = screen.getByTestId("data-table-scroller"); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx index feb1615b4e1..a94c4e34611 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -40,7 +40,7 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; -import { cn } from "@/lib/cva.config"; +import { cn, cva } from "@/lib/cva.config"; import "./columnMeta"; import { DataTablePagination, DEFAULT_PAGE_SIZE_OPTIONS } from "./DataTablePagination"; @@ -58,25 +58,40 @@ const INTERACTIVE_SELECTOR = "button, a, input, select, textarea, [role=checkbox const noop = () => {}; -/** - * Height-filling mode. The table still sizes to its rows; the parent's height is only a ceiling, so - * a short table keeps its footer under the last row and a long one scrolls its rows instead of the - * page. - */ -const FILL_CLASSES = { - outer: "flex max-h-full min-h-0 flex-col", - frame: "flex min-h-0 flex-col", - body: "min-h-0", -} as const; +const dataTableRoot = cva("w-full", { + variants: { fill: { true: "flex h-full min-h-0 flex-1 flex-col", false: null } }, +}); -const NO_FILL_CLASSES = { outer: "", frame: "", body: "" } as const; +const dataTableFrame = cva("overflow-hidden rounded-lg border border-border", { + variants: { fill: { true: "flex min-h-0 flex-1 flex-col", false: null } }, +}); -const STICKY_CLASSES = { - body: "[&_[data-slot=table-container]]:overflow-visible", - header: "bg-background", -} as const; +const dataTableScroller = cva("", { + variants: { + sticky: { true: "overflow-auto [&_[data-slot=table-container]]:overflow-visible", false: "overflow-x-auto" }, + fill: { true: "min-h-0 flex-1", false: null }, + stretchEmpty: { true: "[container-type:inline-size] [&_[data-slot=table-container]]:h-full", false: null }, + }, +}); -const NO_STICKY_CLASSES = { body: "", header: "" } as const; +const dataTableTable = cva("", { + variants: { + resizable: { true: "table-fixed", false: null }, + stretchEmpty: { true: "h-full", false: null }, + }, +}); + +const dataTableHeader = cva("", { + variants: { sticky: { true: "sticky top-0 z-sticky bg-background", false: null } }, +}); + +const dataTableBody = cva("", { + variants: { stretchEmpty: { true: "h-full", false: null } }, +}); + +const messageCell = cva("h-24 text-center align-middle text-sm whitespace-normal text-muted-foreground", { + variants: { stretch: { true: "p-0", false: null } }, +}); function columnDefId(column: ColumnDef): string | undefined { if ("id" in column && typeof column.id === "string") { @@ -314,14 +329,23 @@ function DataTableBodyRow({ ); } -function MessageRow({ colSpan, children }: { colSpan: number; children: React.ReactNode }) { +function MessageRow({ + colSpan, + children, + stretch = false, +}: { + colSpan: number; + children: React.ReactNode; + stretch?: boolean; +}) { return ( - - {children} + + {stretch ? ( +
{children}
+ ) : ( + children + )} ); @@ -598,8 +622,7 @@ export function DataTable(props: DataTableProps { @@ -636,7 +659,11 @@ export function DataTable(props: DataTableProps{noDataMessage ?? }; + return ( + + {noDataMessage ?? } + + ); } return rows.map((row) => ( (props: DataTableProps -
+
+
{toolbar !== undefined &&
{toolbar(table)}
}
- - + + {table.getHeaderGroups().map((headerGroup) => ( {headerGroup.headers.map((header) => ( @@ -682,7 +706,7 @@ export function DataTable(props: DataTableProps ))} - {renderBody()} + {renderBody()} {footer !== undefined && {footer(table)}}
diff --git a/ui/litellm-dashboard/src/components/shared/Page.tsx b/ui/litellm-dashboard/src/components/shared/Page.tsx new file mode 100644 index 00000000000..2799ab43911 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/Page.tsx @@ -0,0 +1,40 @@ +"use client"; + +import type { ComponentProps } from "react"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { cn } from "@/lib/cva.config"; + +export function Page({ className, ...props }: ComponentProps<"div">) { + return
; +} + +export function PageContent({ className, ...props }: ComponentProps<"div">) { + return
; +} + +export function PageTabs({ className, ...props }: ComponentProps) { + return ; +} + +export function PageTabsList({ className, ...props }: ComponentProps) { + return ( + + ); +} + +export function PageTabsTrigger({ className, ...props }: ComponentProps) { + return ( + + ); +} + +export function PageTabsContent({ className, ...props }: ComponentProps) { + return ; +} diff --git a/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx b/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx index 3741542abad..b3283611997 100644 --- a/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/PageHeader.test.tsx @@ -1,77 +1,54 @@ import { renderWithProviders, screen, within } from "@/../tests/test-utils"; +import { Users } from "lucide-react"; import { describe, expect, it } from "vitest"; -import { PageHeader } from "./PageHeader"; - -const identity = { - icon: Teams icon, - title: "Teams", - subtitle: "Manage teams, members, and their access to models and budgets", -}; +import { PageHeader, PageHeaderControls, PageHeaderDescription, PageHeaderTitle } from "./PageHeader"; describe("PageHeader", () => { - it("should render the page identity", () => { - renderWithProviders(); - - expect(screen.getByRole("heading", { name: "Teams" })).toBeInTheDocument(); - expect(screen.getByText("Teams icon").parentElement).toHaveAttribute("aria-hidden", "true"); - expect(screen.getByText(identity.subtitle)).toBeInTheDocument(); - }); - - it("should apply the standard title and subtext typography", () => { - renderWithProviders(); - - const icon = screen.getByText("Teams icon").parentElement; - expect(screen.getByRole("heading", { name: "Teams" })).toHaveClass("text-2xl", "font-semibold", "tracking-tight"); - expect(screen.getByText(identity.subtitle)).toHaveClass("mt-1.5", "text-sm", "text-muted-foreground"); - expect(icon).toHaveClass("size-5", "[&_svg]:size-5", "[&_svg]:stroke-[1.75]"); - expect(icon?.parentElement).toHaveClass("gap-2.5"); - }); - - it("should render the primary action, divider, tabs, and utilities in the standard control row", () => { + it("should name the page heading by its text alone when it carries an icon", () => { renderWithProviders( - Create Team} - tabs={ -
- -
- } - utilities={} - />, + + + + Teams + + Manage teams + , + ); + + expect(screen.getByRole("heading", { level: 1, name: "Teams" })).toBeInTheDocument(); + expect(screen.getByText("Manage teams")).toBeInTheDocument(); + }); + + it("should group the composed controls under one accessible label", () => { + renderWithProviders( + + Teams + + + + + , ); const controls = screen.getByRole("group", { name: "Page controls" }); - expect(controls).toHaveClass("mt-5", "h-9"); - expect(within(controls).getByRole("separator")).toHaveClass("mx-4", "h-6"); - expect(controls).toHaveTextContent("Create TeamYour TeamsRefresh"); + expect( + within(controls) + .getAllByRole("button") + .map((button) => button.textContent), + ).toEqual(["Create Team", "Refresh"]); }); - it("should omit the divider when tabs are absent", () => { - renderWithProviders(Create Team} />); - - expect(screen.queryByRole("separator")).not.toBeInTheDocument(); - }); - - it("should provide standard controls to an embedded tab shell", () => { + it("should forward native attributes to each part", () => { renderWithProviders( - Create Team} - tabs={({ leadingControls, utilities }) => ( -
- {leadingControls} - - {utilities} -
- )} - utilities={} - />, + + Teams + + , ); - const tabs = screen.getByRole("tablist"); - expect(within(tabs).getByRole("separator")).toBeInTheDocument(); - expect(tabs).toHaveTextContent("Create TeamYour TeamsRefresh"); + expect(screen.getByTestId("header")).toContainElement(screen.getByRole("heading", { name: "Teams" })); + expect(screen.getByRole("heading", { name: "Teams" })).toHaveAttribute("id", "page-title"); + expect(screen.getByRole("group", { name: "Team controls" })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx index 6fc5b0f3238..28a22f0a69f 100644 --- a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx +++ b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx @@ -1,62 +1,36 @@ "use client"; -import * as React from "react"; +import type { ComponentProps } from "react"; -import { ToolbarSeparator } from "./ToolbarSeparator"; +import { cn } from "@/lib/cva.config"; -interface EmbeddedTabsSlots { - leadingControls: React.ReactNode; - utilities: React.ReactNode; +export function PageHeader({ className, ...props }: ComponentProps<"div">) { + return
; } -interface PageHeaderProps { - title: React.ReactNode; - subtitle: React.ReactNode; - icon: React.ReactNode; - primaryAction?: React.ReactNode; - titleAction?: React.ReactNode; - tabs?: React.ReactNode | ((slots: EmbeddedTabsSlots) => React.ReactNode); - utilities?: React.ReactNode; -} - -export function PageHeader({ title, subtitle, icon, primaryAction, titleAction, tabs, utilities }: PageHeaderProps) { - const leadingControls = - primaryAction == null ? null : ( -
- {primaryAction} - {tabs != null && } -
- ); - const utilityControls = utilities == null ? null :
{utilities}
; - const hasControlRow = primaryAction != null || tabs != null || utilities != null; - +export function PageHeaderTitle({ className, ...props }: ComponentProps<"h1">) { return ( -
-
-
- -

{title}

-
- {titleAction} -
-

{subtitle}

- - {typeof tabs === "function" ? ( -
{tabs({ leadingControls, utilities: utilityControls })}
- ) : ( - hasControlRow && ( -
- {leadingControls} - {tabs} - {utilityControls != null &&
{utilityControls}
} -
- ) +

+ {...props} + /> + ); +} + +export function PageHeaderDescription({ className, ...props }: ComponentProps<"p">) { + return

; +} + +export function PageHeaderControls({ className, ...props }: ComponentProps<"div">) { + return ( +

); } diff --git a/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx index abc00b75884..6480cb59682 100644 --- a/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/AuditLogsPanel.tsx @@ -100,7 +100,7 @@ export default function AuditLogsPanel({ if (!premiumUser) { return ( -
+

✨ Enterprise Feature.

This is a LiteLLM Enterprise feature, and requires a valid key to use. @@ -128,10 +128,6 @@ export default function AuditLogsPanel({ return ( <> -

-

Audit Logs

-
- row.id} diff --git a/ui/litellm-dashboard/src/components/view_logs/LogsTableToolbar.tsx b/ui/litellm-dashboard/src/components/view_logs/LogsTableToolbar.tsx index 60657872ccc..3496b709d9b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/LogsTableToolbar.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/LogsTableToolbar.tsx @@ -2,72 +2,59 @@ import moment from "moment"; import { CalendarDays } from "lucide-react"; -import { useState } from "react"; +import { type ComponentProps, useState } from "react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Switch } from "@/components/ui/switch"; +import { cn } from "@/lib/cva.config"; import { QUICK_SELECT_OPTIONS } from "./constants"; import { getTimeRangeDisplay } from "./logs_utils"; -interface LogsTableToolbarProps { +const DATETIME_LOCAL_FORMAT = "YYYY-MM-DDTHH:mm"; + +export interface LogsTimeRange { startTime: string; - onStartTimeChange: (value: string) => void; endTime: string; - onEndTimeChange: (value: string) => void; isCustomDate: boolean; - onIsCustomDateChange: (value: boolean) => void; - selectedTimeInterval: { value: number; unit: string }; - onSelectedTimeIntervalChange: (value: { value: number; unit: string }) => void; - isLiveTail: boolean; - onIsLiveTailChange: (value: boolean) => void; - excludeInternalHealthChecks: boolean; - onExcludeInternalHealthChecksChange: (value: boolean) => void; - onResetToFirstPage: () => void; - onResetFilters: () => void; + interval: { value: number; unit: string }; } -export function LogsTableToolbar({ - startTime, - onStartTimeChange, - endTime, - onEndTimeChange, - isCustomDate, - onIsCustomDateChange, - selectedTimeInterval, - onSelectedTimeIntervalChange, - isLiveTail, - onIsLiveTailChange, - excludeInternalHealthChecks, - onExcludeInternalHealthChecksChange, - onResetToFirstPage, - onResetFilters, -}: LogsTableToolbarProps) { - const [quickSelectOpen, setQuickSelectOpen] = useState(false); +const relativeTimeRange = (interval: LogsTimeRange["interval"]): LogsTimeRange => ({ + startTime: moment() + .subtract(interval.value, interval.unit as moment.unitOfTime.DurationConstructor) + .format(DATETIME_LOCAL_FORMAT), + endTime: moment().format(DATETIME_LOCAL_FORMAT), + isCustomDate: false, + interval, +}); - const applyQuickSelect = (option: { label: string; value: number; unit: string }) => { - onResetToFirstPage(); - onEndTimeChange(moment().format("YYYY-MM-DDTHH:mm")); - onStartTimeChange( - moment() - .subtract(option.value, option.unit as moment.unitOfTime.DurationConstructor) - .format("YYYY-MM-DDTHH:mm"), - ); - onSelectedTimeIntervalChange({ value: option.value, unit: option.unit }); - onIsCustomDateChange(false); - setQuickSelectOpen(false); - }; +export const defaultLogsTimeRange = (): LogsTimeRange => relativeTimeRange({ value: 24, unit: "hours" }); + +export function LogsToolbar({ className, ...props }: ComponentProps<"div">) { + return
; +} + +interface LogsTimeRangePickerProps { + value: LogsTimeRange; + onValueChange: (value: LogsTimeRange) => void; +} + +export function LogsTimeRangePicker({ value, onValueChange }: LogsTimeRangePickerProps) { + const [open, setOpen] = useState(false); const selectedOption = QUICK_SELECT_OPTIONS.find( - (option) => option.value === selectedTimeInterval.value && option.unit === selectedTimeInterval.unit, + (option) => option.value === value.interval.value && option.unit === value.interval.unit, ); - const displayLabel = isCustomDate ? getTimeRangeDisplay(isCustomDate, startTime, endTime) : selectedOption?.label; + const displayLabel = value.isCustomDate + ? getTimeRangeDisplay(true, value.startTime, value.endTime) + : selectedOption?.label; return ( -
- + <> + @@ -83,7 +70,10 @@ export function LogsTableToolbar({ key={option.label} variant="ghost" className="w-full justify-start font-normal" - onClick={() => applyQuickSelect(option)} + onClick={() => { + onValueChange(relativeTimeRange({ value: option.value, unit: option.unit })); + setOpen(false); + }} > {option.label} @@ -92,10 +82,7 @@ export function LogsTableToolbar({ @@ -103,58 +90,32 @@ export function LogsTableToolbar({ - {isCustomDate && ( + {value.isCustomDate && (
{ - onStartTimeChange(event.target.value); - onResetToFirstPage(); - }} + value={value.startTime} + onChange={(event) => onValueChange({ ...value, startTime: event.target.value })} /> to { - onEndTimeChange(event.target.value); - onResetToFirstPage(); - }} + value={value.endTime} + onChange={(event) => onValueChange({ ...value, endTime: event.target.value })} />
)} - -
- Live Tail - -
- -
- Hide Health Checks - -
- - -
+ ); } -export function LiveTailBanner({ onStop }: { onStop: () => void }) { +export function LogsToolbarSwitch({ label, ...props }: ComponentProps & { label: string }) { return ( -
- Auto-refreshing every 15 seconds - +
+ {label} +
); } diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx index 8a0e529e2e8..aefbf68d662 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.test.tsx @@ -792,15 +792,15 @@ describe("RequestLogsPanel", () => { }); describe("live tail", () => { - it("shows the auto-refresh banner on the first page and hides it once stopped", async () => { + it("shows the refresh status on the first page and hides it when live tail is disabled", async () => { const user = userEvent.setup(); renderPanel(); - expect(await screen.findByText("Auto-refreshing every 15 seconds")).toBeInTheDocument(); + expect(await screen.findByText("Refreshing every 15s")).toBeInTheDocument(); - await user.click(screen.getByRole("button", { name: "Stop" })); + await user.click(screen.getByRole("switch", { name: "Live Tail" })); - expect(screen.queryByText("Auto-refreshing every 15 seconds")).not.toBeInTheDocument(); + expect(screen.queryByText("Refreshing every 15s")).not.toBeInTheDocument(); }); }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index 0b5e5ff9616..ff26e30baca 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -7,6 +7,7 @@ import moment from "moment"; import { useCallback, useEffect, useMemo, useState } from "react"; import { DEFAULT_PAGE_SIZE_OPTIONS } from "@/components/shared/DataTable"; +import { Button } from "@/components/ui/button"; import { AutoRouterModelGroupsProvider } from "@/components/shared/table_cells"; import { DEBOUNCE_WAIT_MS } from "@/utils/debounceConstants"; import type { KeyResponse } from "../key_team_helpers/key_list"; @@ -23,11 +24,16 @@ import { } from "./log_filter_logic"; import { useLogDetailRouting } from "./logDetailRouting"; import { LogDetailsDrawer } from "./LogDetailsDrawer"; -import { LiveTailBanner, LogsTableToolbar } from "./LogsTableToolbar"; +import { + defaultLogsTimeRange, + type LogsTimeRange, + LogsTimeRangePicker, + LogsToolbar, + LogsToolbarSwitch, +} from "./LogsTableToolbar"; import { RequestLogsTable } from "./RequestLogsTable"; const PAGE_SIZE = DEFAULT_PAGE_SIZE_OPTIONS[0]; -const DEFAULT_INTERVAL = { value: 24, unit: "hours" }; const matchesLogId = (log: LogEntry, logId: string) => log.request_id === logId || log.litellm_call_id === logId; const findLogById = (logs: readonly LogEntry[], logId: string): LogEntry | null => logs.find((log) => log.request_id === logId) ?? logs.find((log) => log.litellm_call_id === logId) ?? null; @@ -46,10 +52,8 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, const [columnFilters, setColumnFilters] = useState([]); const [sessionCursors, setSessionCursors] = useState>({}); - const [startTime, setStartTime] = useState(moment().subtract(24, "hours").format("YYYY-MM-DDTHH:mm")); - const [endTime, setEndTime] = useState(moment().format("YYYY-MM-DDTHH:mm")); - const [isCustomDate, setIsCustomDate] = useState(false); - const [selectedTimeInterval, setSelectedTimeInterval] = useState<{ value: number; unit: string }>(DEFAULT_INTERVAL); + const [timeRange, setTimeRange] = useState(defaultLogsTimeRange); + const { startTime, endTime, isCustomDate } = timeRange; const [selectedKeyIdInfoView, setSelectedKeyIdInfoView] = useState(null); const [selectedLog, setSelectedLog] = useState(null); @@ -232,12 +236,17 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, [resetToFirstPage], ); + const handleTimeRangeChange = useCallback( + (value: LogsTimeRange) => { + setTimeRange(value); + resetToFirstPage(); + }, + [resetToFirstPage], + ); + const handleResetFilters = useCallback(() => { setColumnFilters([]); - setStartTime(moment().subtract(24, "hours").format("YYYY-MM-DDTHH:mm")); - setEndTime(moment().format("YYYY-MM-DDTHH:mm")); - setIsCustomDate(false); - setSelectedTimeInterval(DEFAULT_INTERVAL); + setTimeRange(defaultLogsTimeRange()); resetToFirstPage(); }, [resetToFirstPage]); @@ -288,12 +297,6 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, return ( -
-

Request Logs

-
- - {isLiveTail && pagination.pageIndex === 0 && setIsLiveTail(false)} />} - + + + + {isLiveTail && pagination.pageIndex === 0 && ( + + Refreshing every 15s + + )} + + + } /> diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx deleted file mode 100644 index fe3b99a1882..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ /dev/null @@ -1,105 +0,0 @@ -import { useState } from "react"; -import useCan from "@/app/(dashboard)/hooks/useCan"; -import DeletedKeysPage from "../DeletedKeysPage/DeletedKeysPage"; -import DeletedTeamsPage from "../DeletedTeamsPage/DeletedTeamsPage"; -import AuditLogsPanel from "./AuditLogsPanel"; -import RequestLogsPanel from "./RequestLogsPanel"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; - -interface SpendLogsTableProps { - accessToken: string | null; - token: string | null; - userRole: string | null; - userID: string | null; - premiumUser: boolean; -} - -type LogsTabId = "request logs" | "audit logs" | "deleted keys" | "deleted teams"; - -interface LogsTab { - id: LogsTabId; - label: string; -} - -const REQUEST_LOGS_TAB: LogsTab = { id: "request logs", label: "Request Logs" }; -const AUDIT_LOGS_TAB: LogsTab = { id: "audit logs", label: "Audit Logs" }; -const DELETED_KEYS_TAB: LogsTab = { id: "deleted keys", label: "Deleted Keys" }; -const DELETED_TEAMS_TAB: LogsTab = { id: "deleted teams", label: "Deleted Teams" }; - -const tabContentClassName = (tabId: LogsTabId): string => - tabId === REQUEST_LOGS_TAB.id ? "flex min-h-0 flex-1 flex-col" : "min-h-0 flex-1 overflow-y-auto"; - -export default function SpendLogsTable({ accessToken, token, userRole, userID, premiumUser }: SpendLogsTableProps) { - const [activeTab, setActiveTab] = useState(REQUEST_LOGS_TAB.id); - const canViewAuditLogs = useCan("viewAuditLogs"); - const canViewDeletedTeams = useCan("viewDeletedTeams"); - - if (!accessToken || !token || !userRole || !userID) { - return ( -
- -
- ); - } - - const tabs: LogsTab[] = [ - REQUEST_LOGS_TAB, - ...(canViewAuditLogs ? [AUDIT_LOGS_TAB] : []), - DELETED_KEYS_TAB, - ...(canViewDeletedTeams ? [DELETED_TEAMS_TAB] : []), - ]; - - const renderPanel = (tabId: LogsTabId) => { - switch (tabId) { - case "request logs": - return ( - - ); - case "audit logs": - return ( - - ); - case "deleted keys": - return ; - case "deleted teams": - return ; - } - }; - - return ( -
- setActiveTab(value as LogsTabId)} - className="min-h-0 flex-1 gap-0" - > - - {tabs.map((tab) => ( - - {tab.label} - - ))} - - {tabs.map((tab) => ( - - {renderPanel(tab.id)} - - ))} - -
- ); -} From 677205b3f5fe918e537a60dbd01c91c9b209c30e Mon Sep 17 00:00:00 2001 From: yujonglee Date: Fri, 2 Oct 2026 18:38:54 -0700 Subject: [PATCH 08/42] fix(proxy): share ownership permissions for spend logs and traces (#44239) * refactor(proxy): extract shared spend log read policy * test(proxy): use named bindings for spend scope regression * test(proxy): reuse existing spend log query harness * test(proxy): cover spend log permission lookup adoption * chore(proxy): relocate existing spend query baseline * refactor(proxy): make scope query returns explicit * refactor(proxy): inject deferred log permission lookup * test(proxy): cover teamless management compatibility lookup * refactor(proxy): compose user and team log grants * refactor(proxy): share generic authorization composition * refactor(proxy): compose trace read permissions * refactor(proxy): centralize spend and trace authorization * refactor(proxy): strengthen spend and trace scope types * refactor(proxy): flatten log read scope into owned logs Replace the AnyOf grant tree with a flat OwnedLogs(user_id, team_ids) scope, and OwnedTraces(logs, api_key_hash) for traces, since every consumer flattened the tree back into that shape. A caller with no user id now gets an empty scope instead of matching ownerless rows through Prisma's IS NULL. The dead request_id guard in ui_view_spend_logs is removed, and the management facets inject the log team lookup and reuse read_scope_sql instead of the list shim. Co-Authored-By: Claude Opus 5.5 * test(proxy): run spend scope tests through one SQLite emulator Replace the string-matching payload emulator and the hand-rolled Prisma where interpreter with one SQLite helper that runs the real scope SQL. Session scope tests now go through the endpoint, including the no-user caller that must not match ownerless rows. Drop duplicated lookup-failure and trace mapping cases. load_permitted_log_team_ids returns no teams without a database instead of relying on the resolver's broad except. Co-Authored-By: Claude Opus 5.5 * fix(proxy): unify log and trace ownership permissions * test(tracing): align fixtures with ownership read scopes * refactor(tracing): align query scopes with row ownership * refactor(spend): make ownership SQL predicates explicit * test(spend): validate ownership SQL against PostgreSQL * docs(traces): drop key-row visibility from query help guide Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(spend): reach the empty-memberships branch in team lookup test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(ui): regenerate dashboard API types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../traces-clickhouse/query/list_traces.sql | 3 +- .../traces-clickhouse/query/span_detail.sql | 6 +- .../traces-clickhouse/query/span_error.sql | 3 +- .../query/spend_by_response_ids.sql | 3 +- .../query/trace_identity.sql | 3 +- .../traces-clickhouse/query/trace_spans.sql | 3 +- .../crates/traces-clickhouse/src/query.rs | 2 +- .../traces-clickhouse/src/query/named.rs | 6 +- .../traces-clickhouse/src/query_access.rs | 56 +--- .../crates/traces-clickhouse/src/sql.rs | 2 +- .../templates/query_help.jinja | 2 +- .../traces-clickhouse/tests/admin_sql.rs | 2 +- .../traces-clickhouse/tests/migrations.rs | 60 ++-- .../crates/traces-clickhouse/tests/queries.rs | 43 +-- .../tests/queries/read_access.json | 6 +- .../traces-clickhouse/tests/query_access.rs | 45 +-- litellm-rust/crates/traces/src/query/named.rs | 1 - .../crates/traces/src/query_access.rs | 25 +- .../crates/traces/tests/query/named.rs | 16 +- .../crates/traces/tests/query_access.rs | 40 ++- litellm/proxy/auth/authorization.py | 77 +++++ .../proxy/auth/authorization_dependencies.py | 57 ++++ .../management_v1/spend_logs.py | 44 +-- .../proxy/spend_tracking/log_visibility.py | 44 --- .../spend_management_endpoints.py | 247 +++++---------- litellm/proxy/tracing_endpoints.py | 69 ++--- litellm/rust_bridge/trace_queries.py | 6 - litellm/rust_bridge/traces.py | 22 +- litellm/tracing/types.py | 1 - .../unbounded_in_baseline.txt | 5 +- .../spend/test_spend_log_read_scope.py | 226 ++++++++++++++ tests/test_litellm/tracing/test_receiver.py | 2 +- tests/test_litellm/tracing/test_store.py | 22 +- tests/test_litellm_rust/test_traces.py | 41 ++- tests/unit/proxy/auth/test_authorization.py | 29 ++ .../common_utils/test_http_parsing_utils.py | 2 +- .../management_v1/test_spend_logs.py | 62 ++-- .../spend_tracking/test_log_visibility.py | 47 --- .../test_spend_management_endpoints.py | 287 ++++++++---------- .../test_spend_query_optimization.py | 28 +- tests/unit/proxy/test_tracing_endpoints.py | 211 ++++++++++--- tests/unit/rust_bridge/test_trace_queries.py | 2 - ui/litellm-dashboard/src/lib/http/schema.d.ts | 37 ++- 43 files changed, 1051 insertions(+), 844 deletions(-) create mode 100644 litellm/proxy/auth/authorization.py create mode 100644 litellm/proxy/auth/authorization_dependencies.py delete mode 100644 litellm/proxy/spend_tracking/log_visibility.py create mode 100644 tests/integration/spend/test_spend_log_read_scope.py create mode 100644 tests/unit/proxy/auth/test_authorization.py delete mode 100644 tests/unit/proxy/spend_tracking/test_log_visibility.py diff --git a/litellm-rust/crates/traces-clickhouse/query/list_traces.sql b/litellm-rust/crates/traces-clickhouse/query/list_traces.sql index 1598f04aba2..c52adf7ef49 100644 --- a/litellm-rust/crates/traces-clickhouse/query/list_traces.sql +++ b/litellm-rust/crates/traces-clickhouse/query/list_traces.sql @@ -18,8 +18,7 @@ SELECT TraceId AS trace_id, FROM agent_traces_by_key WHERE ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND UserIds = [{user_id:String}]) - OR has({team_ids:Array(String)}, TeamId) - OR ({api_key_hash:String} != '' AND ApiKeyHash = {api_key_hash:String})) + OR has({team_ids:Array(String)}, TeamId)) GROUP BY TeamId, ApiKeyHash, TraceId HAVING min(StartTs) >= fromUnixTimestamp64Milli({start_ms:Int64}) AND min(StartTs) < fromUnixTimestamp64Milli({end_ms:Int64}) diff --git a/litellm-rust/crates/traces-clickhouse/query/span_detail.sql b/litellm-rust/crates/traces-clickhouse/query/span_detail.sql index 4da48e00d4b..7db742ea3ee 100644 --- a/litellm-rust/crates/traces-clickhouse/query/span_detail.sql +++ b/litellm-rust/crates/traces-clickhouse/query/span_detail.sql @@ -9,8 +9,7 @@ LEFT JOIN ( AND ObservationType = 'llm' AND Output != '' AND ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND UserId = {user_id:String}) - OR has({team_ids:Array(String)}, TeamId) - OR ({api_key_hash:String} != '' AND ApiKeyHash = {api_key_hash:String})) + OR has({team_ids:Array(String)}, TeamId)) AND ({trace_ref:String} = '' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) = {trace_ref:String}) GROUP BY TeamId, ApiKeyHash, ParentSpanId @@ -19,8 +18,7 @@ LEFT JOIN ( WHERE o.TraceId = {trace_id:String} AND o.SpanId = {span_id:String} AND ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND o.UserId = {user_id:String}) - OR has({team_ids:Array(String)}, o.TeamId) - OR ({api_key_hash:String} != '' AND o.ApiKeyHash = {api_key_hash:String})) + OR has({team_ids:Array(String)}, o.TeamId)) AND ({trace_ref:String} = '' OR hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) = {trace_ref:String}) LIMIT 1 diff --git a/litellm-rust/crates/traces-clickhouse/query/span_error.sql b/litellm-rust/crates/traces-clickhouse/query/span_error.sql index 087962227a8..e1226c4d23c 100644 --- a/litellm-rust/crates/traces-clickhouse/query/span_error.sql +++ b/litellm-rust/crates/traces-clickhouse/query/span_error.sql @@ -6,8 +6,7 @@ FROM otel_traces WHERE TraceId = {trace_id:String} AND SpanId = {span_id:String} AND ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND UserId = {user_id:String}) - OR has({team_ids:Array(String)}, TeamId) - OR ({api_key_hash:String} != '' AND ApiKeyHash = {api_key_hash:String})) + OR has({team_ids:Array(String)}, TeamId)) AND ({trace_ref:String} = '' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) = {trace_ref:String}) AND ({error_version:String} = '' OR hex(SHA256(StatusMessage)) = {error_version:String}) diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql index eb132099ac5..963c832232b 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql @@ -6,6 +6,5 @@ WHERE response_id IN {response_ids:Array(String)} AND start_time < fromUnixTimestamp64Milli({end_ms:Int64}) AND ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND user = {user_id:String}) - OR has({team_ids:Array(String)}, team_id) - OR ({api_key_hash:String} != '' AND api_key = {api_key_hash:String})) + OR has({team_ids:Array(String)}, team_id)) ORDER BY start_time DESC diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_identity.sql b/litellm-rust/crates/traces-clickhouse/query/trace_identity.sql index 6b860065468..e3881b150b7 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_identity.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_identity.sql @@ -3,7 +3,6 @@ FROM otel_traces WHERE TraceId = {trace_id:String} AND ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND UserId = {user_id:String}) - OR has({team_ids:Array(String)}, TeamId) - OR ({api_key_hash:String} != '' AND ApiKeyHash = {api_key_hash:String})) + OR has({team_ids:Array(String)}, TeamId)) GROUP BY TeamId, ApiKeyHash, TraceId LIMIT 2 diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql index ade1fdf0d86..f00329d8136 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql @@ -12,8 +12,7 @@ FROM otel_traces AS o WHERE o.TraceId = {trace_id:String} AND ({all_teams:UInt8} = 1 OR ({user_id:String} != '' AND o.UserId = {user_id:String}) - OR has({team_ids:Array(String)}, o.TeamId) - OR ({api_key_hash:String} != '' AND o.ApiKeyHash = {api_key_hash:String})) + OR has({team_ids:Array(String)}, o.TeamId)) AND ({trace_ref:String} = '' OR hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) = {trace_ref:String}) ORDER BY o.Timestamp, o.EngineReceivedMs, o.StatusMessage diff --git a/litellm-rust/crates/traces-clickhouse/src/query.rs b/litellm-rust/crates/traces-clickhouse/src/query.rs index 459f0f95fe7..9f9c5463b7e 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query.rs @@ -425,7 +425,7 @@ pub async fn query_help(client: &Client, connection: &Connection) -> Result( - json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "api_key_hash": "key", "start_ms": -1, "end_ms": 10, "cursor_ms": 0, "cursor_trace_id": "", "limit": u32::MAX}), + json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "start_ms": -1, "end_ms": 10, "cursor_ms": 0, "cursor_trace_id": "", "limit": u32::MAX}), quoted, ); round_trip::( - json!({"all_teams": 0, "user_id": "", "team_ids": [], "api_key_hash": "key", "trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}), + json!({"all_teams": 0, "user_id": "", "team_ids": [], "trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}), quoted, ); round_trip::( - json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "api_key_hash": "", "response_ids": ["response"], "start_ms": -1, "end_ms": 10}), + json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "start_ms": -1, "end_ms": 10}), quoted, ); } diff --git a/litellm-rust/crates/traces-clickhouse/src/query_access.rs b/litellm-rust/crates/traces-clickhouse/src/query_access.rs index 5d82a5e2e93..e6ca322098e 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query_access.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query_access.rs @@ -176,17 +176,13 @@ impl QueryReaders { } fn predicate(scope: &QueryScope, table: TraceTable) -> String { - let (team, key) = match table { - TraceTable::OtelTraces | TraceTable::AgentTracesByKey => ("TeamId", "ApiKeyHash"), - TraceTable::SpendLogs => ("team_id", "api_key"), + let team = match table { + TraceTable::OtelTraces | TraceTable::AgentTracesByKey => "TeamId", + TraceTable::SpendLogs => "team_id", }; match scope { - QueryScope::Admin => "1".to_owned(), - QueryScope::Logs { - user_id, - team_ids, - api_key_hash, - } => { + QueryScope::All => "1".to_owned(), + QueryScope::Owned { user_id, team_ids } => { let owner = literal(user_id); let user_clause = match table { TraceTable::OtelTraces => format!("UserId = {owner}"), @@ -203,21 +199,8 @@ fn predicate(scope: &QueryScope, table: TraceTable) -> String { } else { format!("{team} IN ({teams})") }; - format!( - "({owner} != '' AND {user_clause}) OR ({team_clause}) OR ({hash} != '' AND {key} = {hash})", - owner = owner, - hash = literal(api_key_hash), - ) + format!("({owner} != '' AND {user_clause}) OR ({team_clause})") } - QueryScope::Team { team_id } => format!("{team} = {}", literal(team_id)), - QueryScope::Key { - team_id, - api_key_hash, - } => format!( - "{team} = {} AND {key} = {}", - literal(team_id), - literal(api_key_hash) - ), } } @@ -239,33 +222,24 @@ mod tests { use rstest::rstest; #[rstest] - #[case::otel(TraceTable::OtelTraces, "TeamId", "ApiKeyHash")] - #[case::agent(TraceTable::AgentTracesByKey, "TeamId", "ApiKeyHash")] - #[case::spend(TraceTable::SpendLogs, "team_id", "api_key")] + #[case::otel(TraceTable::OtelTraces, "TeamId", "UserId = ''")] + #[case::agent(TraceTable::AgentTracesByKey, "TeamId", "UserIds = ['']")] + #[case::spend(TraceTable::SpendLogs, "team_id", "user = ''")] fn predicates_preserve_scope_and_escape_values( #[case] table: TraceTable, #[case] team: &str, - #[case] key: &str, + #[case] user: &str, ) { - assert_eq!(predicate(&QueryScope::Admin, table), "1"); + assert_eq!(predicate(&QueryScope::All, table), "1"); assert_eq!( predicate( - &QueryScope::Team { - team_id: "team'\\".into() + &QueryScope::Owned { + user_id: String::new(), + team_ids: vec!["team'\\".into()] }, table ), - format!("{team} = 'team\\'\\\\'") - ); - assert_eq!( - predicate( - &QueryScope::Key { - team_id: "".into(), - api_key_hash: "key'\\".into() - }, - table - ), - format!("{team} = '' AND {key} = 'key\\'\\\\'") + format!("('' != '' AND {user}) OR ({team} IN ('team\\'\\\\'))") ); } } diff --git a/litellm-rust/crates/traces-clickhouse/src/sql.rs b/litellm-rust/crates/traces-clickhouse/src/sql.rs index 0f739314418..16a610ca13a 100644 --- a/litellm-rust/crates/traces-clickhouse/src/sql.rs +++ b/litellm-rust/crates/traces-clickhouse/src/sql.rs @@ -64,7 +64,7 @@ mod tests { #[case] specific: serde_json::Value, ) { let common = serde_json::json!({ - "all_teams": 1, "user_id": "", "team_ids": [], "api_key_hash": "", "trace_id": "trace", "trace_ref": "" + "all_teams": 1, "user_id": "", "team_ids": [], "trace_id": "trace", "trace_ref": "" }); let parameters: BTreeMap = common .as_object() diff --git a/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja b/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja index 6e746221022..3bea0ff4efe 100644 --- a/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja +++ b/litellm-rust/crates/traces-clickhouse/templates/query_help.jinja @@ -46,7 +46,7 @@ Gotchas {% block reader_limits %}The reader enforces {{ limits.result_rows }} result rows, {{ limits.result_mib() }} MiB response bytes, {{ limits.memory_mib() }} MiB memory and a {{ limits.execution_seconds }} second query limit; exceeding limits fails instead of returning partial results{% endblock %} -{% block reader_profile %}LiteLLM provisions SELECT-only readers from the configured ClickHouse connection and enforces request-log visibility through row policies. Callers see their own user rows and permitted teams, or their own key rows when no user identity is available. Provisioning requires CREATE USER, ALTER USER, CREATE ROW POLICY, and GRANT SELECT permissions{% endblock %} +{% block reader_profile %}LiteLLM provisions SELECT-only readers from the configured ClickHouse connection and enforces request-log visibility through row policies. Callers see their own user rows and permitted teams. Provisioning requires CREATE USER, ALTER USER, CREATE ROW POLICY, and GRANT SELECT permissions{% endblock %} {% block output_format %}Do not add FORMAT clauses; the endpoint requires ClickHouse JSON output{% endblock %} diff --git a/litellm-rust/crates/traces-clickhouse/tests/admin_sql.rs b/litellm-rust/crates/traces-clickhouse/tests/admin_sql.rs index 614dad7a35a..49499ebfa39 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/admin_sql.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/admin_sql.rs @@ -41,7 +41,7 @@ async fn database() -> Result> { } let readers = QueryReaders::new(Connection::writer(&admin_url)?, "litellm".into()); let connection = readers - .connection(&client, &QueryScope::Admin, "test-secret") + .connection(&client, &QueryScope::All, "test-secret") .await?; let url = connection.url().to_string(); Ok(Database { diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index 9fdeb450c70..2f9e88af3e7 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -104,7 +104,6 @@ async fn schema_supports_span_rollups_and_spend_joins( all_teams: 0, user_id: String::new(), team_ids: vec!["team-1".into()], - api_key_hash: String::new(), }, trace_id: "trace-1".into(), trace_ref: String::new(), @@ -119,7 +118,6 @@ async fn schema_supports_span_rollups_and_spend_joins( ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), ("team_ids".into(), Parameter::Strings(vec!["team-1".into()])), - ("api_key_hash".into(), Parameter::Text(String::new())), ( "start_ms".into(), Parameter::Integer(timestamp / 1_000_000 - 1000), @@ -153,7 +151,6 @@ async fn schema_supports_span_rollups_and_spend_joins( ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), ("team_ids".into(), Parameter::Strings(vec!["team-1".into()])), - ("api_key_hash".into(), Parameter::Text(String::new())), ( "start_ms".into(), Parameter::Integer(timestamp / 1_000_000 - 1000), @@ -421,6 +418,7 @@ async fn listed_agent_names_preserve_scope_and_cursor( vec![serde_json::from_value(serde_json::json!({ "Timestamp": timestamp, "TraceId": trace, "SpanId": span, "ParentSpanId": parent, "ServiceName": "shared-app", "SpanName": span, "AgentName": agent, + "UserId": if key == "one" { "owner" } else { "other" }, "Framework": framework, "ObservationType": "agent", "ResourceAttributes": {"litellm.team_id": team, "litellm.api_key_hash": key} }))?], @@ -442,9 +440,8 @@ async fn listed_agent_names_preserve_scope_and_cursor( let connection = Connection::configured(&database.url, "trace_test", "default", "")?; let parameters = BTreeMap::from([ ("all_teams".into(), Parameter::Integer(0)), - ("user_id".into(), Parameter::Text(String::new())), + ("user_id".into(), Parameter::Text("owner".into())), ("team_ids".into(), Parameter::Strings(vec![])), - ("api_key_hash".into(), Parameter::Text("one".into())), ( "start_ms".into(), Parameter::Integer(timestamp / 1_000_000 - 1000), @@ -596,7 +593,6 @@ async fn rollup_merges_spans_across_days_without_losing_root_fields( ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), ("team_ids".into(), Parameter::Strings(vec!["team-1".into()])), - ("api_key_hash".into(), Parameter::Text(String::new())), ( "start_ms".into(), Parameter::Integer(day_start / 1_000_000 - 2000), @@ -795,7 +791,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( for (key, text) in [("one", "timeout"), ("two", "success")] { insert_rows(&database, "otel_traces", vec![serde_json::from_value(serde_json::json!({ "Timestamp": timestamp, "TraceId": "shared", "SpanId": "root", "ParentSpanId": "", - "ServiceName": "review", "SpanName": "release", "Input": text, + "ServiceName": "review", "SpanName": "release", "Input": text, "UserId": key, "ResourceAttributes": {"litellm.team_id": "team", "litellm.api_key_hash": key, "swarm": "release"} }))?]).await?; } @@ -849,7 +845,6 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), ("team_ids".into(), Parameter::Strings(vec!["team".into()])), - ("api_key_hash".into(), Parameter::Text(String::new())), ]); let identities: serde_json::Value = serde_json::from_str( &execute_named_read( @@ -861,11 +856,11 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( .await?, )?; assert_eq!(identities["data"].as_array().map(Vec::len), Some(2)); - let key_params = identity_params + let user_params = identity_params .into_iter() .chain([ ("team_ids".into(), Parameter::Strings(vec![])), - ("api_key_hash".into(), Parameter::Text("one".into())), + ("user_id".into(), Parameter::Text("one".into())), ]) .collect(); let identity: serde_json::Value = serde_json::from_str( @@ -873,7 +868,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( &database.client, &connection, ReadQuery::TraceIdentity, - &key_params, + &user_params, ) .await?, )?; @@ -1170,7 +1165,6 @@ async fn trace_error_previews_preserve_paginated_diagnostics( ("all_teams".into(), Parameter::Integer(1)), ("user_id".into(), Parameter::Text(String::new())), ("team_ids".into(), Parameter::Strings(vec![])), - ("api_key_hash".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(String::new())), ]); let body = execute_named_read( @@ -1217,10 +1211,7 @@ async fn trace_error_previews_preserve_paginated_diagnostics( } assert_eq!(recovered, message); parameters.insert("all_teams".into(), Parameter::Integer(0)); - parameters.insert( - "api_key_hash".into(), - Parameter::Text("unrelated-key".into()), - ); + parameters.insert("user_id".into(), Parameter::Text("unrelated-user".into())); let denied = execute_named_read(&database.client, &reader, ReadQuery::SpanError, ¶meters).await?; assert_eq!( @@ -1265,7 +1256,6 @@ async fn duplicate_span_preview_matches_diagnostic( ("all_teams".into(), Parameter::Integer(1)), ("user_id".into(), Parameter::Text(String::new())), ("team_ids".into(), Parameter::Strings(vec![])), - ("api_key_hash".into(), Parameter::Text(String::new())), ("trace_ref".into(), Parameter::Text(String::new())), ("error_version".into(), Parameter::Text(String::new())), ("error_offset".into(), Parameter::Integer(0)), @@ -1716,16 +1706,16 @@ fn field_definitions_match_serialized_normalized_span() { } #[rstest] -#[case::own_user("owner", vec![], "", vec!["own"])] -#[case::own_user_and_permitted_team("owner", vec!["permitted"], "", vec!["own", "team"])] -#[case::key_only("", vec![], "request-key", vec!["own"])] -#[case::no_identity("", vec![], "", vec![])] +#[case::own_user("owner", vec![], None, vec!["own"])] +#[case::own_user_and_permitted_team("owner", vec!["permitted"], None, vec!["own", "team"])] +#[case::no_identity("", vec![], None, vec![])] +#[case::legacy_key_without_identity("", vec![], Some("request-key"), vec![])] #[tokio::test] async fn named_and_sql_readers_share_request_log_visibility( #[future(awt)] database: TestResult, #[case] user: &str, #[case] teams: Vec<&str>, - #[case] key: &str, + #[case] legacy_key: Option<&str>, #[case] expected: Vec<&str>, ) -> TestResult { use litellm_traces_clickhouse::query::named::{ @@ -1748,12 +1738,10 @@ async fn named_and_sql_readers_share_request_log_visibility( let reader = Connection::reader(&database.url, "trace_test")?; let params = SpendByResponseIdsParams::from(litellm_traces::query::named::SpendByResponseIdsParams { - access: ReadAccessParams { - all_teams: 0, - user_id: user.into(), - team_ids: teams.iter().map(|team| (*team).into()).collect(), - api_key_hash: key.into(), - }, + access: serde_json::from_value::(serde_json::json!({ + "all_teams": 0, "user_id": user, "team_ids": teams, + "api_key_hash": legacy_key.unwrap_or_default(), + }))?, response_ids: vec!["shared-response".into()], start_ms: timestamp / 1_000_000 - 1, end_ms: timestamp / 1_000_000 + 1, @@ -1765,10 +1753,9 @@ async fn named_and_sql_readers_share_request_log_visibility( spend.iter().map(|row| row.0.request_id.as_str()).collect(); let expected: std::collections::BTreeSet<_> = expected.into_iter().collect(); assert_eq!(actual, expected); - let scope = QueryScope::Logs { + let scope = QueryScope::Owned { user_id: user.into(), team_ids: teams.into_iter().map(str::to_owned).collect(), - api_key_hash: key.into(), }; if user.is_empty() && scope.validate().is_err() { assert!( @@ -1829,7 +1816,6 @@ async fn rollup_cost_completeness_preserves_missing_ids_and_fails_closed_for_his all_teams: 0, user_id: "".into(), team_ids: vec!["team".into()], - api_key_hash: "".into(), }, start_ms: timestamp / 1_000_000 - 1, end_ms: timestamp / 1_000_000 + 1, @@ -1850,7 +1836,6 @@ async fn rollup_cost_completeness_preserves_missing_ids_and_fails_closed_for_his user_id: "owner".into(), team_ids: vec![], all_teams: 0, - api_key_hash: String::new(), }, ..params.0 }, @@ -1882,18 +1867,16 @@ async fn rollup_cost_completeness_preserves_missing_ids_and_fails_closed_for_his } #[rstest] -#[case::admin(1, "", vec![], "", "own answer")] -#[case::user(0, "owner", vec![], "", "own answer")] -#[case::team(0, "", vec!["alpha"], "", "own answer")] -#[case::key(0, "", vec![], "one", "own answer")] -#[case::no_identity(0, "", vec![], "", "")] +#[case::admin(1, "", vec![], "own answer")] +#[case::user(0, "owner", vec![], "own answer")] +#[case::team(0, "", vec!["alpha"], "own answer")] +#[case::no_identity(0, "", vec![], "")] #[tokio::test] async fn agent_final_answer_preserves_visibility_and_trace_ownership( #[future(awt)] database: TestResult, #[case] all_teams: u8, #[case] user: &str, #[case] teams: Vec<&str>, - #[case] key: &str, #[case] expected: &str, ) -> TestResult { use litellm_traces_clickhouse::query::named::{ReadAccessParams, SpanDetail, SpanDetailParams}; @@ -1952,7 +1935,6 @@ async fn agent_final_answer_preserves_visibility_and_trace_ownership( all_teams, user_id: user.into(), team_ids: teams.into_iter().map(str::to_owned).collect(), - api_key_hash: key.into(), }, trace_id: "shared".into(), trace_ref: String::new(), diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries.rs b/litellm-rust/crates/traces-clickhouse/tests/queries.rs index 3b9f108af94..c72255d5e0d 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/queries.rs @@ -23,23 +23,20 @@ use support::TestResult; enum ScopeCase { Admin, Team, - Key, OtherTeam, } impl ScopeCase { fn scope(self) -> QueryScope { match self { - Self::Admin => QueryScope::Admin, - Self::Team => QueryScope::Team { - team_id: "team-a".into(), + Self::Admin => QueryScope::All, + Self::Team => QueryScope::Owned { + user_id: String::new(), + team_ids: vec!["team-a".into()], }, - Self::Key => QueryScope::Key { - team_id: "team-a".into(), - api_key_hash: "key-a".into(), - }, - Self::OtherTeam => QueryScope::Team { - team_id: "team-b".into(), + Self::OtherTeam => QueryScope::Owned { + user_id: String::new(), + team_ids: vec!["team-b".into()], }, } } @@ -60,13 +57,7 @@ async fn curated_queries_return_expected_rows( #[future(awt)] seeded_database: TestResult, #[case] sql: &str, #[case] expected_json: &str, - #[values( - ScopeCase::Admin, - ScopeCase::Team, - ScopeCase::Key, - ScopeCase::OtherTeam - )] - scope: ScopeCase, + #[values(ScopeCase::Admin, ScopeCase::Team, ScopeCase::OtherTeam)] scope: ScopeCase, ) -> TestResult { let fixture = seeded_database?; let reader = fixture @@ -103,11 +94,7 @@ async fn typed_queries_read_normalized_spans_and_keep_trace_identities_separate( let fixture = seeded_database?; let reader = fixture .readers - .connection( - &fixture.database.client, - &QueryScope::Admin, - "fixture-secret", - ) + .connection(&fixture.database.client, &QueryScope::All, "fixture-secret") .await?; let params = ListTracesParams::from(contracts::ListTracesParams { access: admin_access?, @@ -176,11 +163,7 @@ async fn typed_trace_cursor_returns_the_next_fixture_trace( let fixture = seeded_database?; let reader = fixture .readers - .connection( - &fixture.database.client, - &QueryScope::Admin, - "fixture-secret", - ) + .connection(&fixture.database.client, &QueryScope::All, "fixture-secret") .await?; let params = ListTracesParams::from(contracts::ListTracesParams { access: admin_access?, @@ -218,11 +201,7 @@ async fn captured_deeplite_exports_round_trip_through_clickhouse( let decoded = insert_export(&fixture, export, "team-a", "key-a").await?; let reader = fixture .readers - .connection( - &fixture.database.client, - &QueryScope::Admin, - "fixture-secret", - ) + .connection(&fixture.database.client, &QueryScope::All, "fixture-secret") .await?; let params = TraceSpansParams { access: admin_access?, diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries/read_access.json b/litellm-rust/crates/traces-clickhouse/tests/queries/read_access.json index a743b382c20..f0af446092e 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries/read_access.json +++ b/litellm-rust/crates/traces-clickhouse/tests/queries/read_access.json @@ -1,6 +1,8 @@ { "all_teams": 1, "user_id": "", - "team_ids": ["team-a", "team-b"], - "api_key_hash": "" + "team_ids": [ + "team-a", + "team-b" + ] } diff --git a/litellm-rust/crates/traces-clickhouse/tests/query_access.rs b/litellm-rust/crates/traces-clickhouse/tests/query_access.rs index be749351a72..ef5b76c2097 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/query_access.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/query_access.rs @@ -25,8 +25,8 @@ async fn database() -> Result> { let writer = Connection::parse(&url)?; ensure_schema(&client, &writer, "trace_test", 7).await?; for sql in [ - "INSERT INTO trace_test.otel_traces (TeamId, ApiKeyHash, TraceId, SpanId, Timestamp, SpanAttributes, UserId) VALUES ('team-a', 'key-a1', 'shared-trace', 'a1', now(), map('visible', 'a'), 'owner'), ('team-a', 'key-a2', 'shared-trace', 'a2', now(), map('visible', 'a'), 'other'), ('team-b', 'key-b', 'shared-trace', 'b', now(), map('secret-b', 'b'), 'owner'), ('', 'key-teamless', 'shared-trace', 'teamless', now(), map('visible', 'teamless'), ''), ('', 'key-other', 'shared-trace', 'other-teamless', now(), map('visible', 'other'), '')", - "INSERT INTO trace_test.spend_logs (team_id, api_key, request_id, start_time, end_time, metadata, user) VALUES ('team-a', 'key-a1', 'a1', now(), now(), '{\"visible\":1}', 'owner'), ('team-a', 'key-a2', 'a2', now(), now(), '{\"visible\":1}', 'other'), ('team-b', 'key-b', 'b', now(), now(), '{\"secret_b\":1}', 'owner'), ('', 'key-teamless', 'teamless', now(), now(), '{}', ''), ('', 'key-other', 'other-teamless', now(), now(), '{}', '')", + "INSERT INTO trace_test.otel_traces (TeamId, ApiKeyHash, TraceId, SpanId, Timestamp, SpanAttributes, UserId) VALUES ('team-a', 'key-a1', 'shared-trace', 'a1', now(), map('visible', 'a'), 'owner'), ('team-a', 'key-a2', 'shared-trace', 'a2', now(), map('visible', 'a'), 'other'), ('team-b', 'key-b', 'shared-trace', 'b', now(), map('secret-b', 'b'), 'owner'), ('team-c', 'key-a1', 'shared-trace', 'same-key-foreign', now(), map('visible', 'foreign'), 'other'), ('', 'key-teamless', 'shared-trace', 'teamless', now(), map('visible', 'teamless'), ''), ('', 'key-other', 'shared-trace', 'other-teamless', now(), map('visible', 'other'), '')", + "INSERT INTO trace_test.spend_logs (team_id, api_key, request_id, start_time, end_time, metadata, user) VALUES ('team-a', 'key-a1', 'a1', now(), now(), '{\"visible\":1}', 'owner'), ('team-a', 'key-a2', 'a2', now(), now(), '{\"visible\":1}', 'other'), ('team-b', 'key-b', 'b', now(), now(), '{\"secret_b\":1}', 'owner'), ('team-c', 'key-a1', 'same-key-foreign', now(), now(), '{}', 'other'), ('', 'key-teamless', 'teamless', now(), now(), '{}', ''), ('', 'key-other', 'other-teamless', now(), now(), '{}', '')", "CREATE TABLE trace_test.private_data (secret String) ENGINE = Memory", "INSERT INTO trace_test.private_data VALUES ('hidden')", ] { @@ -43,15 +43,12 @@ async fn database() -> Result> { } #[rstest] -#[case::own_user(QueryScope::Logs { user_id: "owner".into(), team_ids: vec![], api_key_hash: "".into() }, vec!["a1", "b"])] -#[case::own_user_and_permitted_team(QueryScope::Logs { user_id: "owner".into(), team_ids: vec!["team-a".into()], api_key_hash: "".into() }, vec!["a1", "a2", "b"])] -#[case::key_only_logs(QueryScope::Logs { user_id: "".into(), team_ids: vec![], api_key_hash: "key-teamless".into() }, vec!["teamless"])] -#[case::quoted_user(QueryScope::Logs { user_id: "owner' OR 1=1 --".into(), team_ids: vec![], api_key_hash: "".into() }, vec![])] -#[case::team(QueryScope::Team { team_id: "team-a".to_owned() }, vec!["a1", "a2"])] -#[case::project_key(QueryScope::Key { team_id: "team-a".to_owned(), api_key_hash: "key-a1".to_owned() }, vec!["a1"])] -#[case::teamless_key(QueryScope::Key { team_id: "".to_owned(), api_key_hash: "key-teamless".to_owned() }, vec!["teamless"])] -#[case::admin(QueryScope::Admin, vec!["a1", "a2", "b", "other-teamless", "teamless"])] -#[case::quoted_team(QueryScope::Team { team_id: "team-a' OR 1=1 --\\".to_owned() }, vec![])] +#[case::own_user(QueryScope::Owned { user_id: "owner".into(), team_ids: vec![] }, vec!["a1", "b"])] +#[case::own_user_and_permitted_team(QueryScope::Owned { user_id: "owner".into(), team_ids: vec!["team-a".into()] }, vec!["a1", "a2", "b"])] +#[case::quoted_user(QueryScope::Owned { user_id: "owner' OR 1=1 --".into(), team_ids: vec![] }, vec![])] +#[case::team(QueryScope::Owned { user_id: String::new(), team_ids: vec!["team-a".to_owned() ] }, vec!["a1", "a2"])] +#[case::admin(QueryScope::All, vec!["a1", "a2", "b", "other-teamless", "same-key-foreign", "teamless"])] +#[case::quoted_team(QueryScope::Owned { user_id: String::new(), team_ids: vec!["team-a' OR 1=1 --\\".to_owned() ] }, vec![])] #[tokio::test] async fn queries_and_help_are_scoped_by_the_database( #[future(awt)] database: Result>, @@ -111,8 +108,9 @@ async fn rotating_master_secret_revokes_previous_reader_credentials( #[future(awt)] database: Result>, ) -> Result<(), Box> { let database = database?; - let scope = QueryScope::Team { - team_id: "team-a".to_owned(), + let scope = QueryScope::Owned { + user_id: String::new(), + team_ids: vec!["team-a".to_owned()], }; let old_reader = database .readers @@ -159,8 +157,9 @@ async fn managed_reader_rejects_privilege_and_scope_bypasses( #[future(awt)] database: Result>, ) -> Result<(), Box> { let database = database?; - let scope = QueryScope::Team { - team_id: "team-a".to_owned(), + let scope = QueryScope::Owned { + user_id: String::new(), + team_ids: vec!["team-a".to_owned()], }; let reader = database .readers @@ -213,14 +212,15 @@ async fn provisioning_failure_never_returns_a_writer_connection( let database = database?; let reader = database .readers - .connection(&database.client, &QueryScope::Admin, "test-master-secret") + .connection(&database.client, &QueryScope::All, "test-master-secret") .await?; let no_provision_privileges = QueryReaders::new(reader, "trace_test".to_owned()); let result = no_provision_privileges .connection( &database.client, - &QueryScope::Team { - team_id: "team-a".to_owned(), + &QueryScope::Owned { + user_id: String::new(), + team_ids: vec!["team-a".to_owned()], }, "other-secret", ) @@ -232,7 +232,7 @@ async fn provisioning_failure_never_returns_a_writer_connection( assert!(matches!( database .readers - .connection(&database.client, &QueryScope::Admin, "") + .connection(&database.client, &QueryScope::All, "") .await, Err(Error::MissingSecret) )); @@ -241,8 +241,9 @@ async fn provisioning_failure_never_returns_a_writer_connection( .readers .connection( &database.client, - &QueryScope::Team { - team_id: String::new() + &QueryScope::Owned { + user_id: String::new(), + team_ids: vec![String::new()] }, "test-master-secret" ) @@ -263,6 +264,6 @@ async fn provisioning_failure_never_returns_a_writer_connection( ) .await?; let rows: Value = serde_json::from_str(&rows)?; - assert_eq!(rows["data"][0]["count"], 5); + assert_eq!(rows["data"][0]["count"], 6); Ok(()) } diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index b39c44d49cb..b45c076c3ba 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -6,7 +6,6 @@ pub struct ReadAccessParams { pub all_teams: u8, pub user_id: String, pub team_ids: Vec, - pub api_key_hash: String, } #[derive(Debug, Deserialize, Serialize)] diff --git a/litellm-rust/crates/traces/src/query_access.rs b/litellm-rust/crates/traces/src/query_access.rs index 51bb5c6a097..2f57bd4c0ce 100644 --- a/litellm-rust/crates/traces/src/query_access.rs +++ b/litellm-rust/crates/traces/src/query_access.rs @@ -5,33 +5,20 @@ use crate::InvalidScope; #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryScope { - Admin, - Team { - team_id: String, - }, - Logs { + All, + Owned { user_id: String, team_ids: Vec, - api_key_hash: String, - }, - Key { - team_id: String, - api_key_hash: String, }, } impl QueryScope { pub fn validate(&self) -> Result<(), InvalidScope> { match self { - Self::Admin => Ok(()), - Self::Team { team_id } if !team_id.is_empty() => Ok(()), - Self::Key { api_key_hash, .. } if !api_key_hash.is_empty() => Ok(()), - Self::Logs { - user_id, - team_ids, - api_key_hash, - } if (!user_id.is_empty() || !team_ids.is_empty() || !api_key_hash.is_empty()) - && team_ids.iter().all(|team| !team.is_empty()) => + Self::All => Ok(()), + Self::Owned { user_id, team_ids } + if (!user_id.is_empty() || !team_ids.is_empty()) + && team_ids.iter().all(|team| !team.is_empty()) => { Ok(()) } diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index 7db77df8a29..bfe50a8684d 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -9,12 +9,16 @@ fn round_trip(wire: Value) { } #[rstest] -#[case::admin(vec![], "")] -#[case::multiple_teams(vec!["team-a", "team-b"], "")] -#[case::key(vec!["team-a"], "key")] -#[case::teamless_key(vec![], "key")] -fn named_requests_preserve_all_access_cases(#[case] teams: Vec<&str>, #[case] key: &str) { - let access = json!({"all_teams": u8::from(teams.is_empty() && key.is_empty()), "user_id": "", "team_ids": teams, "api_key_hash": key}); +#[case::admin(1, "", vec![])] +#[case::own_user(0, "user", vec![])] +#[case::multiple_teams(0, "user", vec!["team-a", "team-b"])] +#[case::no_identity(0, "", vec![])] +fn named_requests_preserve_all_access_cases( + #[case] all_teams: u8, + #[case] user: &str, + #[case] teams: Vec<&str>, +) { + let access = json!({"all_teams": all_teams, "user_id": user, "team_ids": teams}); round_trip::(access.clone()); let request = |specific: Value| { Value::Object( diff --git a/litellm-rust/crates/traces/tests/query_access.rs b/litellm-rust/crates/traces/tests/query_access.rs index e21abc6f427..79155065605 100644 --- a/litellm-rust/crates/traces/tests/query_access.rs +++ b/litellm-rust/crates/traces/tests/query_access.rs @@ -3,18 +3,12 @@ use rstest::rstest; use serde_json::{Value, json}; #[rstest] -#[case::admin(json!({"kind": "admin"}), true)] -#[case::team(json!({"kind": "team", "team_id": "team"}), true)] -#[case::empty_team(json!({"kind": "team", "team_id": ""}), false)] -#[case::key(json!({"kind": "key", "team_id": "team", "api_key_hash": "key"}), true)] -#[case::teamless_key(json!({"kind": "key", "team_id": "", "api_key_hash": "key"}), true)] -#[case::empty_key(json!({"kind": "key", "team_id": "team", "api_key_hash": ""}), false)] -#[case::user_logs(json!({"kind": "logs", "user_id": "user", "team_ids": [], "api_key_hash": ""}), true)] -#[case::permitted_teams(json!({"kind": "logs", "user_id": "", "team_ids": ["team"], "api_key_hash": ""}), true)] -#[case::key_logs(json!({"kind": "logs", "user_id": "", "team_ids": [], "api_key_hash": "key"}), true)] -#[case::anonymous_logs(json!({"kind": "logs", "user_id": "", "team_ids": [], "api_key_hash": ""}), false)] -#[case::empty_permitted_team(json!({"kind": "logs", "user_id": "user", "team_ids": [""], "api_key_hash": ""}), false)] -#[case::empty_teamless_key(json!({"kind": "key", "team_id": "", "api_key_hash": ""}), false)] +#[case::all(json!({"kind": "all"}), true)] +#[case::own_user(json!({"kind": "owned", "user_id": "user", "team_ids": []}), true)] +#[case::permitted_teams(json!({"kind": "owned", "user_id": "", "team_ids": ["team"]}), true)] +#[case::own_user_and_permitted_teams(json!({"kind": "owned", "user_id": "user", "team_ids": ["team"]}), true)] +#[case::no_identity(json!({"kind": "owned", "user_id": "", "team_ids": []}), false)] +#[case::empty_permitted_team(json!({"kind": "owned", "user_id": "user", "team_ids": [""]}), false)] fn scope_validation_preserves_authorization_and_wire_shape( #[case] wire: Value, #[case] valid: bool, @@ -28,22 +22,22 @@ fn scope_validation_preserves_authorization_and_wire_shape( } #[rstest] -#[case::unknown_kind(json!({"kind": "all"}))] -#[case::unknown_field(json!({"kind": "team", "team_id": "team", "extra": true}))] -#[case::missing_team(json!({"kind": "key", "api_key_hash": "key"}))] -#[case::missing_key(json!({"kind": "key", "team_id": "team"}))] +#[case::unknown_kind(json!({"kind": "unknown"}))] +#[case::unknown_field(json!({"kind": "owned", "user_id": "user", "team_ids": [], "extra": true}))] +#[case::legacy_admin(json!({"kind": "admin"}))] +#[case::legacy_logs(json!({"kind": "logs", "user_id": "user", "team_ids": []}))] +#[case::legacy_team(json!({"kind": "team", "team_id": "team"}))] +#[case::key_scope(json!({"kind": "key", "team_id": "team", "api_key_hash": "key"}))] +#[case::key_grant(json!({"kind": "owned", "user_id": "user", "team_ids": [], "api_key_hash": "key"}))] fn scope_rejects_invalid_wire_shape(#[case] wire: Value) { assert!(serde_json::from_value::(wire).is_err()); } #[rstest] -fn admin_preserves_existing_extra_field_handling() { +fn all_preserves_existing_extra_field_handling() { let scope: QueryScope = - serde_json::from_value(json!({"kind": "admin", "team_id": "ignored"})).unwrap(); - assert!(matches!(scope, QueryScope::Admin)); + serde_json::from_value(json!({"kind": "all", "team_id": "ignored"})).unwrap(); + assert!(matches!(scope, QueryScope::All)); assert!(scope.validate().is_ok()); - assert_eq!( - serde_json::to_value(scope).unwrap(), - json!({"kind": "admin"}) - ); + assert_eq!(serde_json::to_value(scope).unwrap(), json!({"kind": "all"})); } diff --git a/litellm/proxy/auth/authorization.py b/litellm/proxy/auth/authorization.py new file mode 100644 index 00000000000..91549f19d2e --- /dev/null +++ b/litellm/proxy/auth/authorization.py @@ -0,0 +1,77 @@ +from collections.abc import Awaitable, Callable, Iterable, Sequence +from dataclasses import dataclass +from typing import Final, TypeAlias + +from litellm.proxy._types import KeyManagementRoutes, LiteLLM_TeamTable, LitellmUserRoles, UserAPIKeyAuth + + +@dataclass(frozen=True, slots=True) +class AllRows: + """Unrestricted reads, granted by the consuming endpoint's role checks.""" + + +@dataclass(frozen=True, slots=True) +class OwnedRows: + """Rows owned by ``user_id`` or by any of ``team_ids``; a ``None`` user grants no own-user rows.""" + + user_id: str | None + team_ids: tuple[str, ...] = () + + +ReadScope: TypeAlias = AllRows | OwnedRows + + +async def resolve_owned_read_scope( + user_id: str | None, + permitted_team_lookup: Callable[[], Awaitable[Sequence[str]]], +) -> OwnedRows: + """Resolve own-user and permitted-team reads, falling back to own-user on lookup failure.""" + if user_id is None: + return OwnedRows(None) + try: + team_ids: Final = tuple(await permitted_team_lookup()) + except Exception: # noqa: BLE001 # preserve spend-log own-user fallback for every permission lookup failure + return OwnedRows(user_id) + return OwnedRows(user_id, team_ids) + + +def can_read_team_logs(auth: UserAPIKeyAuth, team: LiteLLM_TeamTable) -> bool: + from litellm.proxy.management.teams.access import is_team_admin + from litellm.proxy.management_endpoints.common_utils import ( + _team_member_has_permission, # pyright: ignore[reportPrivateUsage] # reuse existing team permission policy + ) + + return is_team_admin(user_api_key_dict=auth, team_obj=team) or _team_member_has_permission( + user_api_key_dict=auth, + team_obj=team, + permission=KeyManagementRoutes.SPEND_LOGS.value, + ) + + +def permitted_log_team_ids(auth: UserAPIKeyAuth, teams: Iterable[LiteLLM_TeamTable]) -> tuple[str, ...]: + return tuple(team.team_id for team in teams if can_read_team_logs(auth, team)) + + +async def can_read_log_owner( + user_id: str | None, + owner_user: str | None, + owner_team_id: str | None, + team_permission_lookup: Callable[[str], Awaitable[bool]], +) -> bool: + """Authorize stored ownership without swallowing direct team-lookup failures.""" + if owner_user is not None and owner_user == user_id: + return True + if owner_team_id: + return await team_permission_lookup(owner_team_id) + return False + + +async def resolve_trace_read_scope( + auth: UserAPIKeyAuth, + permitted_team_lookup: Callable[[], Awaitable[Sequence[str]]], +) -> ReadScope | None: + if auth.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + return AllRows() + if not auth.user_id: + return None + return await resolve_owned_read_scope(auth.user_id, permitted_team_lookup) diff --git a/litellm/proxy/auth/authorization_dependencies.py b/litellm/proxy/auth/authorization_dependencies.py new file mode 100644 index 00000000000..3e7ae75dc86 --- /dev/null +++ b/litellm/proxy/auth/authorization_dependencies.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from functools import partial +from typing import TYPE_CHECKING, Annotated, Final, TypeAlias + +from fastapi import Depends + +from litellm.proxy._types import LiteLLM_TeamTable, UserAPIKeyAuth +from litellm.proxy.auth.authorization import permitted_log_team_ids + +if TYPE_CHECKING: + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import PrismaClient, ProxyLogging + + +LogTeamLookup: TypeAlias = Callable[[UserAPIKeyAuth], Awaitable[tuple[str, ...]]] + + +async def load_permitted_log_team_ids( + auth: UserAPIKeyAuth, + *, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging, +) -> tuple[str, ...]: + from litellm.proxy.auth.auth_checks import get_user_object + from litellm.repositories.team_repository import TeamRepository + + if prisma_client is None: + return () + user_obj: Final = await get_user_object( + user_id=auth.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=proxy_logging_obj, + ) + if user_obj is None or not user_obj.teams: + return () + team_rows: Final = await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": user_obj.teams}}) + return permitted_log_team_ids(auth, (LiteLLM_TeamTable.model_validate(row.model_dump()) for row in team_rows)) + + +async def get_log_team_lookup() -> LogTeamLookup: + """Bind infrastructure without performing permission I/O before the handler's checks.""" + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache + + return partial( + load_permitted_log_team_ids, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +LogTeamLookupDependency: TypeAlias = Annotated[LogTeamLookup, Depends(get_log_team_lookup)] diff --git a/litellm/proxy/management_endpoints/management_v1/spend_logs.py b/litellm/proxy/management_endpoints/management_v1/spend_logs.py index 1cbc454ca5e..e9e6e05ce15 100644 --- a/litellm/proxy/management_endpoints/management_v1/spend_logs.py +++ b/litellm/proxy/management_endpoints/management_v1/spend_logs.py @@ -1,12 +1,15 @@ """`/management/v1/spend_logs` facets.""" from datetime import datetime, timezone +from functools import partial from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, Query, Request from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth +from litellm.proxy.auth.authorization import resolve_owned_read_scope +from litellm.proxy.auth.authorization_dependencies import LogTeamLookup, LogTeamLookupDependency from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, @@ -16,7 +19,6 @@ from litellm.proxy.list_api.common import ( reject_unknown_query_params, ) from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX -from litellm.proxy.utils import PrismaClient from litellm.types.proxy.management_endpoints.management_v1 import ( FacetListResponse, PageMeta, @@ -37,49 +39,29 @@ def _as_utc(value: datetime) -> datetime: async def _spend_log_scope_clause( user_api_key_dict: UserAPIKeyAuth, - prisma_client: PrismaClient, + log_team_lookup: LogTeamLookup, next_param_index: int, -) -> tuple[str | None, tuple[str | list[str], ...]]: +) -> tuple[str | None, tuple[object, ...]]: """SQL predicate restricting the facet to spend logs this caller may read. Returns ``(None, ())`` for a proxy admin. Mirrors the scoping ``/spend/logs/ui`` applies, so a dropdown can never offer a value from a row the caller could not open. """ - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _get_permitted_team_ids_for_spend_logs, - _is_admin_view_safe, - ) + from litellm.proxy.spend_tracking.spend_management_endpoints import _is_admin_view_safe, read_scope_sql if _is_admin_view_safe(user_api_key_dict=user_api_key_dict): return None, () - - try: - permitted_team_ids = await _get_permitted_team_ids_for_spend_logs( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - ) - except Exception: - permitted_team_ids = [] - - caller_user_id: Final = user_api_key_dict.user_id - # = ANY(::text[]) rather than an expanded IN list, matching the clause - # ui_view_spend_logs builds: one parameter whatever the team count. - templates: Final = (('"user" = ${}',) if caller_user_id is not None else ()) + ( - ("team_id = ANY(${}::text[])",) if permitted_team_ids else () + scope: Final = await resolve_owned_read_scope( + user_api_key_dict.user_id, partial(log_team_lookup, user_api_key_dict) ) - params: Final = ((caller_user_id,) if caller_user_id is not None else ()) + ( - (permitted_team_ids,) if permitted_team_ids else () - ) - if not templates: - return "FALSE", () - clauses: Final = tuple(template.format(next_param_index + offset) for offset, template in enumerate(templates)) - return f"({' OR '.join(clauses)})", params + return read_scope_sql(scope, next_param_index) async def _list_spend_log_facet( request: Request, user_api_key_dict: UserAPIKeyAuth, + log_team_lookup: LogTeamLookup, start_time: datetime, end_time: datetime, q: str | None, @@ -107,7 +89,7 @@ async def _list_spend_log_facet( scope_clause, scope_params = await _spend_log_scope_clause( user_api_key_dict=user_api_key_dict, - prisma_client=prisma_client, + log_team_lookup=log_team_lookup, next_param_index=len(window_params) + len(search_params) + 1, ) @@ -178,6 +160,7 @@ async def _list_spend_log_facet( async def list_spend_log_end_users( request: Request, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + log_team_lookup: LogTeamLookupDependency, start_time: Annotated[ datetime, Query(alias="filter[startTime][gte]", description="Window start (UTC when no offset is given)"), @@ -211,6 +194,7 @@ async def list_spend_log_end_users( return await _list_spend_log_facet( request=request, user_api_key_dict=user_api_key_dict, + log_team_lookup=log_team_lookup, start_time=start_time, end_time=end_time, q=q, @@ -229,6 +213,7 @@ async def list_spend_log_end_users( async def list_spend_log_users( request: Request, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + log_team_lookup: LogTeamLookupDependency, start_time: Annotated[ datetime, Query(alias="filter[startTime][gte]", description="Window start (UTC when no offset is given)"), @@ -245,6 +230,7 @@ async def list_spend_log_users( return await _list_spend_log_facet( request=request, user_api_key_dict=user_api_key_dict, + log_team_lookup=log_team_lookup, start_time=start_time, end_time=end_time, q=q, diff --git a/litellm/proxy/spend_tracking/log_visibility.py b/litellm/proxy/spend_tracking/log_visibility.py deleted file mode 100644 index 83f236d3028..00000000000 --- a/litellm/proxy/spend_tracking/log_visibility.py +++ /dev/null @@ -1,44 +0,0 @@ -from collections.abc import Awaitable, Callable -from dataclasses import dataclass -from typing import Final - -from fastapi import HTTPException - -from litellm.proxy._types import UserAPIKeyAuth - - -@dataclass(frozen=True, slots=True) -class LogVisibility: - all_teams: bool = False - user_id: str = "" - team_ids: tuple[str, ...] = () - api_key_hash: str = "" - - -async def permitted_log_teams(auth: UserAPIKeyAuth) -> tuple[str, ...]: - from litellm.proxy.proxy_server import prisma_client - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _get_permitted_team_ids_for_spend_logs_or_empty, # pyright: ignore[reportPrivateUsage] # Reuse request-log policy - ) - - if prisma_client is None: - return () - return await _get_permitted_team_ids_for_spend_logs_or_empty(prisma_client=prisma_client, user_api_key_dict=auth) - - -async def log_visibility( - auth: UserAPIKeyAuth, - team_lookup: Callable[[UserAPIKeyAuth], Awaitable[tuple[str, ...]]] = permitted_log_teams, -) -> LogVisibility: - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _is_admin_view_safe, # pyright: ignore[reportPrivateUsage] # Reuse request-log policy - ) - - if _is_admin_view_safe(user_api_key_dict=auth): - return LogVisibility(all_teams=True) - if auth.user_id: - team_ids: Final = await team_lookup(auth) - return LogVisibility(user_id=auth.user_id, team_ids=team_ids, api_key_hash=auth.token or "") - if auth.token: - return LogVisibility(api_key_hash=auth.token) - raise HTTPException(status_code=403, detail="Not allowed to view logs") diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 3102fc63cf4..48cc684549f 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -3,8 +3,8 @@ import collections import json import os from collections.abc import Mapping, Sequence -from dataclasses import dataclass from datetime import date, datetime, timedelta, timezone +from functools import partial from itertools import groupby from types import MappingProxyType from typing import ( @@ -37,6 +37,18 @@ from litellm.constants import ( from litellm.litellm_core_utils.classifier_logging import classifier_audit_fields, classifier_input_snapshot from litellm.proxy._types import * from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject +from litellm.proxy.auth.authorization import ( + AllRows, + OwnedRows, + ReadScope, + can_read_log_owner, + can_read_team_logs, + resolve_owned_read_scope, +) +from litellm.proxy.auth.authorization_dependencies import ( + LogTeamLookup, + LogTeamLookupDependency, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.spend_tracking.spend_capture_rate import ( @@ -403,11 +415,6 @@ async def _find_team_row(prisma_client: PrismaClient, team_id: str) -> _Supports return await _team_table(prisma_client).find_unique(where={"team_id": team_id}) -async def _find_team_rows(prisma_client: PrismaClient, team_ids: Sequence[str]) -> Sequence[_SupportsModelDump]: - """Read team rows as Prisma model instances.""" - return await _team_table(prisma_client).find_many(where={"team_id": {"in": team_ids}}) - - @router.get( "/spend/keys", tags=["Budget & Spend Tracking"], @@ -2474,6 +2481,7 @@ def _build_spend_log_search_condition( ) async def ui_view_spend_logs( request: Request, + log_team_lookup: LogTeamLookupDependency, api_key: str | None = fastapi.Query( default=None, description="Get spend logs based on api key", @@ -2775,16 +2783,8 @@ async def ui_view_spend_logs( and team_id is None and (is_request_id_lookup or _can_user_view_spend_log(user_api_key_dict=user_api_key_dict)) ) - permitted_team_ids: Final = ( - await _get_permitted_team_ids_for_spend_logs_or_empty( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - ) - if user_scope_applies - else () - ) - explicit_user_requires_caller_scope: Final = ( - user_scope_applies and not permitted_team_ids and user_id is not None + read_scope: Final = ( + await _spend_log_read_scope(user_api_key_dict, log_team_lookup) if user_scope_applies else AllRows() ) if not is_admin_view: if team_id is not None: @@ -2799,22 +2799,6 @@ async def ui_view_spend_logs( detail={"error": f"Not authorized to view team spend for team_id={team_id}"}, ) where_conditions["team_id"] = team_id - elif user_scope_applies: - if permitted_team_ids: - if user_id is None: - where_conditions.pop("user", None) - where_conditions["OR"] = [ - {"user": user_api_key_dict.user_id}, - {"team_id": {"in": permitted_team_ids}}, - ] - else: - if user_id is None: - where_conditions["user"] = user_api_key_dict.user_id - else: - where_conditions["AND"] = where_conditions.get("AND", []) + [ - {"user": user_api_key_dict.user_id} - ] - where_conditions.pop("team_id", None) # Calculate skip value for pagination skip: Final = (page - 1) * page_size @@ -2874,17 +2858,11 @@ async def ui_view_spend_logs( sql_params.append(request_id_filter) p += 1 - # Multi-team OR filter: (user = $X OR team_id = ANY($Y)) - if permitted_team_ids: - or_clause: Final = f'("user" = ${p} OR team_id = ANY(${p + 1}::text[]))' - sql_params.append(user_api_key_dict.user_id) - sql_params.append(permitted_team_ids) - p += 2 - sql_conditions.append(or_clause) - elif explicit_user_requires_caller_scope: - sql_conditions.append(f'"user" = ${p}') - sql_params.append(user_api_key_dict.user_id) - p += 1 + scope_clause, scope_params = read_scope_sql(read_scope, p) + if scope_clause: + sql_conditions.append(scope_clause) + sql_params.extend(scope_params) + p += len(scope_params) if session_id is not None and isinstance(session_id, str): like_escaped_session_id: Final = session_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") @@ -3390,6 +3368,7 @@ async def _resolve_request_response_payload( ) async def ui_view_request_response_for_request_id( request_id: str, + log_team_lookup: LogTeamLookupDependency, start_date: str | None = fastapi.Query( default=None, description="Time from which to start viewing key spend", @@ -3442,6 +3421,7 @@ async def ui_view_request_response_for_request_id( user_api_key_dict=user_api_key_dict, request_id=request_id, caller_is_admin=caller_is_admin, + log_team_lookup=log_team_lookup, ) ) stored_request_id: Final = _stored_request_id(spend_log_row, request_id) @@ -4512,6 +4492,7 @@ async def ui_get_spend_by_tags( }, ) async def ui_view_session_spend_logs( + log_team_lookup: LogTeamLookupDependency, session_id: str = fastapi.Query( description="Get all spend logs for a particular session", ), @@ -4549,36 +4530,16 @@ async def ui_view_session_spend_logs( detail="Database not connected", ) - if _is_admin_view_safe(user_api_key_dict=user_api_key_dict): - scope_sql = "" - scope_params = () - where_conditions = {"session_id": session_id} - else: - try: - permitted_team_ids = ( - await _get_permitted_team_ids_for_spend_logs( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - ) - if _can_user_view_spend_log(user_api_key_dict=user_api_key_dict) - else [] - ) - except Exception: # noqa: BLE001 # mirror /spend/logs/ui: failed team lookup falls back to own-logs-only scope - permitted_team_ids = [] - if permitted_team_ids: - scope_sql = ' AND ("user" = $4 OR team_id = ANY($5::text[]))' - scope_params = (user_api_key_dict.user_id, permitted_team_ids) - where_conditions = { - "session_id": session_id, - "OR": [ - {"user": user_api_key_dict.user_id}, - {"team_id": {"in": permitted_team_ids}}, - ], - } - else: - scope_sql = ' AND "user" = $4' - scope_params = (user_api_key_dict.user_id,) - where_conditions = {"session_id": session_id, "user": user_api_key_dict.user_id} + read_scope: Final = ( + AllRows() + if _is_admin_view_safe(user_api_key_dict=user_api_key_dict) + else await _spend_log_read_scope(user_api_key_dict, log_team_lookup) + if _can_user_view_spend_log(user_api_key_dict=user_api_key_dict) + else OwnedRows(user_api_key_dict.user_id) + ) + scope_clause, scope_params = read_scope_sql(read_scope, 4) + scope_sql: Final = f" AND {scope_clause}" if scope_clause else "" + where_conditions: Final = {"session_id": session_id, **_read_scope_where(read_scope)} # Calculate pagination offsets skip: Final = (page - 1) * page_size @@ -4859,22 +4820,12 @@ async def _can_team_member_view_log( Returns True if the team exists and the user is either a team admin or a team member with the ``/spend/logs`` permission. """ - from litellm.proxy.management.teams.access import is_team_admin - from litellm.proxy.management_endpoints.common_utils import _team_member_has_permission - if team_id is None: return False team_row: Final = await _find_team_row(prisma_client, team_id) if team_row is None: return False - team_obj: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump()) - if is_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj): - return True - return _team_member_has_permission( - user_api_key_dict=user_api_key_dict, - team_obj=team_obj, - permission=KeyManagementRoutes.SPEND_LOGS.value, - ) + return can_read_team_logs(user_api_key_dict, LiteLLM_TeamTable.model_validate(team_row.model_dump())) def _can_user_view_spend_log(user_api_key_dict: UserAPIKeyAuth) -> bool: @@ -4899,15 +4850,12 @@ async def _user_can_view_spend_log_owner( owner_user: str | None, owner_team_id: str | None, ) -> bool: - if owner_user is not None and owner_user == user_api_key_dict.user_id: - return True - if owner_team_id: - return await _can_team_member_view_log( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - team_id=owner_team_id, - ) - return False + return await can_read_log_owner( + user_api_key_dict.user_id, + owner_user, + owner_team_id, + partial(_can_team_member_view_log, prisma_client, user_api_key_dict), + ) def _spend_log_forbidden(request_id: str) -> HTTPException: @@ -4940,44 +4888,50 @@ async def _assert_user_can_view_request_id( raise _spend_log_forbidden(request_id) -@dataclass(frozen=True, slots=True) -class _SpendLogViewer: - user_id: str | None - team_ids: tuple[str, ...] - - -async def _spend_log_viewer(prisma_client: PrismaClient, user_api_key_dict: UserAPIKeyAuth) -> _SpendLogViewer: - return _SpendLogViewer( - user_id=user_api_key_dict.user_id, - team_ids=await _get_permitted_team_ids_for_spend_logs_or_empty( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - ), +async def _spend_log_read_scope(user_api_key_dict: UserAPIKeyAuth, log_team_lookup: LogTeamLookup) -> OwnedRows: + return await resolve_owned_read_scope( + user_api_key_dict.user_id, + partial(log_team_lookup, user_api_key_dict), ) -def _viewer_scope_clause(viewer: _SpendLogViewer | None) -> tuple[str, tuple[object, ...]]: - match viewer: - case None: - return ("", ()) - case _SpendLogViewer(user_id=user_id, team_ids=()): - return (' AND "user" = $2', (user_id,)) - case _SpendLogViewer(user_id=user_id, team_ids=team_ids): - return (' AND ("user" = $2 OR team_id = ANY($3::text[]))', (user_id, team_ids)) +def read_scope_sql(scope: ReadScope, next_param: int) -> tuple[str, tuple[object, ...]]: + if isinstance(scope, AllRows): + return "", () + if scope.user_id is not None and scope.team_ids: + return ( + f'("user" = ${next_param} OR team_id = ANY(${next_param + 1}::text[]))', + (scope.user_id, scope.team_ids), + ) + if scope.user_id is not None: + return f'"user" = ${next_param}', (scope.user_id,) + if scope.team_ids: + return f"team_id = ANY(${next_param}::text[])", (scope.team_ids,) + return "FALSE", () -def _spend_log_payload_query(request_id: str, viewer: _SpendLogViewer | None) -> tuple[str, tuple[object, ...]]: +def _read_scope_where(scope: ReadScope) -> Mapping[str, object]: + if isinstance(scope, AllRows): + return {} + user_grant: Final = ({"user": scope.user_id},) if scope.user_id is not None else () + team_grant: Final = ({"team_id": {"in": list(scope.team_ids)}},) if scope.team_ids else () + grants: Final = user_grant + team_grant + return grants[0] if len(grants) == 1 else {"OR": list(grants)} + + +def _spend_log_payload_query(request_id: str, scope: ReadScope) -> tuple[str, tuple[object, ...]]: """ Fetch the one row an id lookup resolves to, preferring the exact ``request_id`` match over rows that merely carry the id as their client-set ``litellm_call_id``. A non-admin viewer only ever gets rows they own or rows of a team they may view. """ - scope, scope_params = _viewer_scope_clause(viewer) + scope_clause, scope_params = read_scope_sql(scope, 2) + scope_sql: Final = f" AND {scope_clause}" if scope_clause else "" return ( f""" SELECT request_id, messages, response, proxy_server_request, metadata, "user", team_id FROM "LiteLLM_SpendLogs" - WHERE (request_id = $1 OR litellm_call_id = $1){scope} + WHERE (request_id = $1 OR litellm_call_id = $1){scope_sql} ORDER BY (request_id = $1) DESC LIMIT 1 """, @@ -4990,6 +4944,7 @@ async def _resolve_spend_log_payload_row( user_api_key_dict: UserAPIKeyAuth, request_id: str, caller_is_admin: bool, + log_team_lookup: LogTeamLookup, ) -> Mapping[str, object] | None: """ Resolve an id lookup to the caller's own spend-log row before any payload @@ -4998,8 +4953,8 @@ async def _resolve_spend_log_payload_row( that id is only the caller's ``litellm_call_id``; the row's stored ``request_id`` is the key that names the caller's own request. """ - viewer: Final = None if caller_is_admin else await _spend_log_viewer(prisma_client, user_api_key_dict) - sql_query, sql_params = _spend_log_payload_query(request_id, viewer) + scope: Final = AllRows() if caller_is_admin else await _spend_log_read_scope(user_api_key_dict, log_team_lookup) + sql_query, sql_params = _spend_log_payload_query(request_id, scope) rows: Final[Sequence[Mapping[str, object]] | None] = await _query_raw_or_none(prisma_client, sql_query, *sql_params) if not rows: return None @@ -5075,57 +5030,3 @@ async def _assert_user_owns_cold_storage_payload( owner_user, owner_team_id = _cold_storage_payload_owner(payload) if not await _user_can_view_spend_log_owner(prisma_client, user_api_key_dict, owner_user, owner_team_id): raise _spend_log_forbidden(request_id) - - -async def _get_permitted_team_ids_for_spend_logs( - prisma_client: PrismaClient, - user_api_key_dict: UserAPIKeyAuth, -) -> list[str]: - """ - Return team IDs where the user is either a team admin or has the - ``/spend/logs`` permission, allowing them to view team-wide spend logs. - """ - # Imported here to avoid circular import: proxy_server imports this module. - from litellm.proxy.auth.auth_checks import get_user_object - from litellm.proxy.management.teams.access import is_team_admin - from litellm.proxy.management_endpoints.common_utils import _team_member_has_permission - from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache - - user_obj: Final = await get_user_object( - user_id=user_api_key_dict.user_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - user_id_upsert=False, - proxy_logging_obj=proxy_logging_obj, - ) - if user_obj is None or not user_obj.teams: - return [] - - team_rows: Final = await _find_team_rows(prisma_client, user_obj.teams) - - permitted: Final[list[str]] = [] - for team_row in team_rows: - team_obj = LiteLLM_TeamTable.model_validate(team_row.model_dump()) - if is_team_admin(user_api_key_dict=user_api_key_dict, team_obj=team_obj) or _team_member_has_permission( - user_api_key_dict=user_api_key_dict, - team_obj=team_obj, - permission=KeyManagementRoutes.SPEND_LOGS.value, - ): - permitted.append(team_obj.team_id) - return permitted - - -async def _get_permitted_team_ids_for_spend_logs_or_empty( - prisma_client: PrismaClient, - user_api_key_dict: UserAPIKeyAuth, -) -> tuple[str, ...]: - """Resolve permitted teams once, falling back to the caller's own-user scope.""" - try: - return tuple( - await _get_permitted_team_ids_for_spend_logs( - prisma_client=prisma_client, - user_api_key_dict=user_api_key_dict, - ) - ) - except Exception: - return () diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py index d741e0b29b4..046bcc9f704 100644 --- a/litellm/proxy/tracing_endpoints.py +++ b/litellm/proxy/tracing_endpoints.py @@ -10,6 +10,7 @@ GET /v1/traces/{trace_id}/spans/{span_id} SpanDetail import time from collections.abc import Mapping from dataclasses import dataclass +from functools import partial from http.client import responses from types import MappingProxyType from typing import Annotated, Final @@ -20,12 +21,13 @@ from pydantic import BaseModel, ConfigDict from litellm._logging import verbose_proxy_logger from litellm.constants import OTLP_RETRY_AFTER_SECONDS from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_trace_read_scope +from litellm.proxy.auth.authorization_dependencies import LogTeamLookupDependency from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.http_parsing_utils import is_otlp_trace_request -from litellm.proxy.spend_tracking.log_visibility import log_visibility from litellm.proxy.tracing_runtime import provide_receiver, require_receiver from litellm.rust_bridge.trace_query_responses import TraceQueryHelp, TraceSQLResponse -from litellm.rust_bridge.traces import ClickHouseStorage, QueryScope +from litellm.rust_bridge.traces import AllQueryScope, ClickHouseStorage, OwnedQueryScope, QueryScope from litellm.tracing import ( Tenant, TraceReceiver, @@ -43,14 +45,14 @@ MS_PER_DAY: Final = 24 * 60 * 60 * 1000 @dataclass(frozen=True, slots=True) class TraceAccessContext: receiver: TraceReceiver | None - read_scope: TraceScope | None + read_scope: ReadScope | None write_tenant: Tenant | None def reader(self) -> tuple[TraceReceiver, TraceScope]: tracing: Final = require_receiver(self.receiver) if self.read_scope is None: raise HTTPException(status_code=403, detail="Not allowed to view agent traces") - return tracing, self.read_scope + return tracing, _trace_scope(self.read_scope) def writer(self) -> tuple[TraceReceiver, Tenant]: if self.write_tenant is None: @@ -61,27 +63,23 @@ class TraceAccessContext: async def provide_trace_access( auth: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], tracing: Annotated[TraceReceiver | None, Depends(provide_receiver)], + log_team_lookup: LogTeamLookupDependency, ) -> TraceAccessContext: tenant: Final = Tenant( team_id=auth.team_id or "", api_key_hash=auth.token or "", org_id=auth.org_id or "", user_id=auth.user_id or "" ) write_tenant: Final = None if auth.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY else tenant - if ( - not auth.user_id - and not auth.token - and auth.user_role not in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) - ): - return TraceAccessContext(tracing, None, write_tenant) - visibility: Final = await log_visibility(auth) - return TraceAccessContext( - tracing, - TraceScope( - all_teams=1 if visibility.all_teams else 0, - user_id=visibility.user_id, - team_ids=visibility.team_ids, - api_key_hash=visibility.api_key_hash, - ), - write_tenant, + read_scope: Final = await resolve_trace_read_scope(auth, partial(log_team_lookup, auth)) + return TraceAccessContext(tracing, read_scope, write_tenant) + + +def _trace_scope(scope: ReadScope) -> TraceScope: + if isinstance(scope, AllRows): + return TraceScope(all_teams=1, user_id="", team_ids=()) + return TraceScope( + all_teams=0, + user_id=scope.user_id or "", + team_ids=scope.team_ids, ) @@ -160,7 +158,7 @@ class TraceQueryRequest(BaseModel): @dataclass(frozen=True, slots=True) class TraceQueryAccess: storage: ClickHouseStorage - scope: QueryScope + scope: ReadScope secret: str @@ -172,24 +170,27 @@ def provide_trace_query_secret() -> str: return master_key -async def trace_query_scope(auth: UserAPIKeyAuth) -> QueryScope: - visibility: Final = await log_visibility(auth) - if visibility.all_teams: - return {"kind": "admin"} - return { - "kind": "logs", - "user_id": visibility.user_id, - "team_ids": visibility.team_ids, - "api_key_hash": visibility.api_key_hash, - } +def trace_query_scope(scope: ReadScope) -> QueryScope: + if isinstance(scope, AllRows): + return AllQueryScope(kind="all") + return OwnedQueryScope( + kind="owned", + user_id=scope.user_id or "", + team_ids=scope.team_ids, + ) async def provide_trace_query_access( auth: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], tracing: Annotated[TraceReceiver | None, Depends(provide_receiver)], secret: Annotated[str, Depends(provide_trace_query_secret)], + log_team_lookup: LogTeamLookupDependency, ) -> TraceQueryAccess: - return TraceQueryAccess(require_receiver(tracing).store.storage, await trace_query_scope(auth), secret) + storage: Final = require_receiver(tracing).store.storage + scope: Final = await resolve_trace_read_scope(auth, partial(log_team_lookup, auth)) + if scope is None: + raise HTTPException(status_code=403, detail="Not allowed to view logs") + return TraceQueryAccess(storage, scope, secret) @router.post("/v1/traces/query", response_model=TraceSQLResponse, response_model_exclude_unset=True) @@ -198,7 +199,7 @@ async def query_agent_traces( access: Annotated[TraceQueryAccess, Depends(provide_trace_query_access)], ) -> TraceSQLResponse: try: - return await access.storage.query_sql(body.sql, access.scope, access.secret) + return await access.storage.query_sql(body.sql, trace_query_scope(access.scope), access.secret) except ValueError as error: raise HTTPException(status_code=400, detail=str(error)) from error except RuntimeError as error: @@ -211,7 +212,7 @@ async def help_agent_trace_queries( access: Annotated[TraceQueryAccess, Depends(provide_trace_query_access)], ) -> TraceQueryHelp: try: - return await access.storage.query_help(access.scope, access.secret) + return await access.storage.query_help(trace_query_scope(access.scope), access.secret) except RuntimeError as error: verbose_proxy_logger.warning("Trace query help unavailable: %s", error) raise HTTPException(status_code=503, detail="Trace query help is temporarily unavailable") from error diff --git a/litellm/rust_bridge/trace_queries.py b/litellm/rust_bridge/trace_queries.py index 0a9fe186646..1add1787241 100644 --- a/litellm/rust_bridge/trace_queries.py +++ b/litellm/rust_bridge/trace_queries.py @@ -30,7 +30,6 @@ class ListTracesParams(BaseModel): all_teams: Literal[0, 1] user_id: str team_ids: tuple[str, ...] - api_key_hash: str start_ms: Int64 end_ms: Int64 cursor_ms: Int64 @@ -43,7 +42,6 @@ class TraceSpansParams(BaseModel): all_teams: Literal[0, 1] user_id: str team_ids: tuple[str, ...] - api_key_hash: str trace_id: str trace_ref: str @@ -53,7 +51,6 @@ class SpanDetailParams(BaseModel): all_teams: Literal[0, 1] user_id: str team_ids: tuple[str, ...] - api_key_hash: str trace_id: str trace_ref: str span_id: str @@ -64,7 +61,6 @@ class SpanErrorParams(BaseModel): all_teams: Literal[0, 1] user_id: str team_ids: tuple[str, ...] - api_key_hash: str trace_id: str trace_ref: str span_id: str @@ -77,7 +73,6 @@ class SpendByResponseIdsParams(BaseModel): all_teams: Literal[0, 1] user_id: str team_ids: tuple[str, ...] - api_key_hash: str response_ids: tuple[str, ...] start_ms: Int64 end_ms: Int64 @@ -143,7 +138,6 @@ class TraceIdentityParams(BaseModel): all_teams: Literal[0, 1] user_id: str team_ids: tuple[str, ...] - api_key_hash: str trace_id: str diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index 23778983d75..f70b5ec3f93 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -77,29 +77,17 @@ class DecodedSpan(TypedDict): consumed_attributes: ReadOnly[tuple[str, str]] -class AdminQueryScope(TypedDict): - kind: ReadOnly[Literal["admin"]] +class AllQueryScope(TypedDict): + kind: ReadOnly[Literal["all"]] -class TeamQueryScope(TypedDict): - kind: ReadOnly[Literal["team"]] - team_id: ReadOnly[str] - - -class KeyQueryScope(TypedDict): - kind: ReadOnly[Literal["key"]] - team_id: ReadOnly[str] - api_key_hash: ReadOnly[str] - - -class LogQueryScope(TypedDict): - kind: ReadOnly[Literal["logs"]] +class OwnedQueryScope(TypedDict): + kind: ReadOnly[Literal["owned"]] user_id: ReadOnly[str] team_ids: ReadOnly[tuple[str, ...]] - api_key_hash: ReadOnly[str] -QueryScope = AdminQueryScope | TeamQueryScope | KeyQueryScope | LogQueryScope +QueryScope = AllQueryScope | OwnedQueryScope class NativeStore(Protocol): diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index 15b28293590..e177094d464 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -108,7 +108,6 @@ class TraceScope(TypedDict): all_teams: ReadOnly[Literal[0, 1]] user_id: ReadOnly[str] team_ids: ReadOnly[tuple[str, ...]] - api_key_hash: ReadOnly[str] class SpanRow(TypedDict): diff --git a/tests/code_coverage_tests/unbounded_in_baseline.txt b/tests/code_coverage_tests/unbounded_in_baseline.txt index c42a6b0ddf5..01d8760855f 100644 --- a/tests/code_coverage_tests/unbounded_in_baseline.txt +++ b/tests/code_coverage_tests/unbounded_in_baseline.txt @@ -125,9 +125,8 @@ litellm/proxy/proxy_server.py _fetch_db_models_for_search prisma not.in `list(db litellm/proxy/proxy_server.py _gather_team_accessible_model_ids prisma model_name.in `_resolved_names` 0 litellm/proxy/proxy_server.py get_all_team_models prisma team_id.in `user_teams` 0 litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py _prune_filter prisma model.in `chunk` 0 -litellm/proxy/spend_tracking/spend_management_endpoints.py _find_team_rows prisma team_id.in `team_ids` 0 -litellm/proxy/spend_tracking/spend_management_endpoints.py ui_view_session_spend_logs prisma team_id.in `permitted_team_ids` 0 -litellm/proxy/spend_tracking/spend_management_endpoints.py ui_view_spend_logs prisma team_id.in `permitted_team_ids` 0 +litellm/proxy/auth/authorization_dependencies.py load_permitted_log_team_ids prisma team_id.in `user_obj.teams` 0 +litellm/proxy/spend_tracking/spend_management_endpoints.py _read_scope_where prisma team_id.in `list(scope.team_ids)` 0 litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py _validate_default_teams_exist prisma team_id.in `list(team_ids)` 0 litellm/proxy/utils.py PrismaClient.check_view_exists raw-sql viewname.IN `IN ( {expected_views_str} )` 0 litellm/proxy/utils.py PrismaClient.delete_data prisma team_id.in `team_id_list` 0 diff --git a/tests/integration/spend/test_spend_log_read_scope.py b/tests/integration/spend/test_spend_log_read_scope.py new file mode 100644 index 00000000000..f9034371e35 --- /dev/null +++ b/tests/integration/spend/test_spend_log_read_scope.py @@ -0,0 +1,226 @@ +import os +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +import pytest_asyncio +from integration._support.client import Gateway +from prisma import Prisma +from psycopg import sql +from psycopg.types.json import Jsonb +from pydantic import TypeAdapter + +from litellm.proxy.auth.authorization import AllRows, OwnedRows, ReadScope +from litellm.proxy.spend_tracking.spend_management_endpoints import _spend_log_payload_query, read_scope_sql + + +@dataclass(frozen=True, slots=True) +class SpendRow: + request_id: str + user: str | None + team_id: str | None + call_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class RequestId: + request_id: str + + +REQUEST_IDS: Final = TypeAdapter(tuple[RequestId, ...]) +ROWS: Final = ( + SpendRow("own", "caller", None, "foreign"), + SpendRow("team-1", "other", "first"), + SpendRow("team-2", "third", "second"), + SpendRow("foreign", "other", "outside"), + SpendRow("ownerless", None, None), + SpendRow("team-ownerless", None, "first"), +) + + +def _seed_rows( + connection: psycopg.Connection, + schema: str, + rows: tuple[SpendRow, ...], + session_id: str, + started: datetime, +) -> None: + utc_timestamp: Final = started.astimezone(timezone.utc).replace(tzinfo=None) + with connection.cursor() as cursor: + cursor.executemany( + sql.SQL( + 'INSERT INTO {} (request_id, "user", team_id, litellm_call_id, session_id, ' + '"startTime", "endTime", messages, response, call_type) ' + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'acompletion')" + ).format(sql.Identifier(schema, "LiteLLM_SpendLogs")), + tuple( + ( + row.request_id, + row.user, + row.team_id, + row.call_id, + session_id, + utc_timestamp, + utc_timestamp, + Jsonb([{"role": "user", "content": row.request_id + " payload"}]), + Jsonb({"id": row.request_id}), + ) + for row in rows + ), + ) + + +@pytest_asyncio.fixture(loop_scope="function") +async def spend_database() -> AsyncIterator[Prisma]: + schema: Final = f"integration_spend_scope_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + parsed: Final = urlsplit(url) + scoped_url: Final = urlunsplit( + parsed._replace(query=urlencode({**dict(parse_qsl(parsed.query)), "schema": schema})) + ) + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute( + sql.SQL('CREATE TABLE {} (LIKE public."LiteLLM_SpendLogs" INCLUDING ALL)').format( + sql.Identifier(schema, "LiteLLM_SpendLogs") + ) + ) + _seed_rows(setup, schema, ROWS, "scope-session", datetime(2026, 1, 1, tzinfo=timezone.utc)) + database: Final = Prisma(datasource={"url": scoped_url}) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("preceding_filters", [False, True]) +@pytest.mark.parametrize( + ("scope", "user_filter", "expected"), + [ + (AllRows(), None, ("foreign", "own", "ownerless", "team-1", "team-2", "team-ownerless")), + (OwnedRows("caller"), None, ("own",)), + (OwnedRows(None), None, ()), + (OwnedRows(None, ("first", "second")), None, ("team-1", "team-2", "team-ownerless")), + (OwnedRows(None, ("first", "second")), "other", ("team-1",)), + (OwnedRows("caller", ("first", "second")), None, ("own", "team-1", "team-2", "team-ownerless")), + (OwnedRows("caller", ("first", "second")), "other", ("team-1",)), + (OwnedRows("caller", ("first' OR TRUE --",)), None, ("own",)), + (OwnedRows("caller' OR TRUE --", ("first",)), None, ("team-1", "team-ownerless")), + ], +) +async def test_ownership_sql_selects_allowed_rows_and_intersects_filters( + spend_database: Prisma, + scope: ReadScope, + user_filter: str | None, + expected: tuple[str, ...], + preceding_filters: bool, +) -> None: + window_params: Final = ("scope-session", "2026-01-01", "2026-01-02") if preceding_filters else () + window_sql: Final = ( + 'session_id = $1 AND "startTime" >= $2::timestamp AND "startTime" < $3::timestamp AND ' + if preceding_filters + else "" + ) + clause, scope_params = read_scope_sql(scope, len(window_params) + 1) + filter_sql: Final = f' AND "user" = ${len(window_params) + len(scope_params) + 1}' if user_filter else "" + params: Final = window_params + scope_params + ((user_filter,) if user_filter else ()) + result: Final = await spend_database.query_raw( + f'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE {window_sql}{clause or "TRUE"}{filter_sql} ' + "ORDER BY request_id", + *params, + ) + assert tuple(row.request_id for row in REQUEST_IDS.validate_python(result)) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("scope", "expected"), + [(AllRows(), ("foreign",)), (OwnedRows("caller"), ("own",)), (OwnedRows(None), ())], +) +async def test_payload_sql_filters_foreign_collisions_and_prefers_exact_ids_for_admins( + spend_database: Prisma, scope: ReadScope, expected: tuple[str, ...] +) -> None: + query, params = _spend_log_payload_query("foreign", scope) + result: Final = await spend_database.query_raw(query, *params) + assert tuple(row.request_id for row in REQUEST_IDS.validate_python(result)) == expected + + +def _delete_session(session_id: str) -> None: + with psycopg.connect(os.environ["DATABASE_URL"]) as connection: + connection.execute('DELETE FROM "LiteLLM_SpendLogs" WHERE session_id = %s', (session_id,)) + + +@pytest.mark.parametrize( + ("member_role", "permissions", "team_access"), + [ + ("admin", [], True), + ("user", ["/spend/logs"], True), + ("user", ["/key/info"], False), + ("user", [], False), + ], +) +def test_spend_log_routes_preserve_user_and_permitted_team_access( + gateway: Gateway, member_role: str, permissions: list[str], team_access: bool +) -> None: + session_id: Final = f"scope-{uuid.uuid4().hex}" + started: Final = datetime.now(timezone.utc) - timedelta(hours=1) + with gateway.scenario() as scenario: + caller: Final = scenario.user(user_role="internal_user") + other: Final = scenario.user(user_role="internal_user") + team: Final = scenario.team( + members_with_roles=[{"user_id": caller, "role": member_role}], + team_member_permissions=list(permissions), + ) + outside_team: Final = scenario.team( + members_with_roles=[{"user_id": other, "role": "admin"}], + team_member_permissions=["/spend/logs"], + ) + key: Final = scenario.key(user_id=caller) + other_key: Final = scenario.key(user_id=other) + rows: Final = ( + SpendRow(session_id + "-own", caller, None, session_id + "-foreign"), + SpendRow(session_id + "-team", other, team), + SpendRow(session_id + "-foreign", other, outside_team), + SpendRow(session_id + "-ownerless", None, None), + SpendRow(session_id + "-outside", other, outside_team), + ) + scenario.cleanups.callback(_delete_session, session_id) + with psycopg.connect(os.environ["DATABASE_URL"]) as connection: + _seed_rows(connection, "public", rows, session_id, started) + expected: Final = (rows[0].request_id, rows[1].request_id) if team_access else (rows[0].request_id,) + session: Final = gateway.request("GET", "/spend/logs/session/ui", key=key, params={"session_id": session_id}) + assert session.status_code == 200, session.text + assert session.json()["total"] == len(expected), session.text + assert sorted(row["request_id"] for row in session.json()["data"]) == list(expected), session.text + filters: Final = { + "session_id": session_id, + "start_date": (started - timedelta(hours=1)).strftime("%Y-%m-%d %H:%M:%S"), + "end_date": (started + timedelta(hours=1)).strftime("%Y-%m-%d %H:%M:%S"), + } + listed: Final = gateway.request("GET", "/spend/logs/ui", key=key, params=filters) + assert listed.status_code == 200, listed.text + assert sorted(row["request_id"] for row in listed.json()["data"]) == list(expected), listed.text + narrowed: Final = gateway.request("GET", "/spend/logs/ui", key=key, params={**filters, "user_id": other}) + assert narrowed.status_code == 200, narrowed.text + assert [row["request_id"] for row in narrowed.json()["data"]] == ( + [rows[1].request_id] if team_access else [] + ), narrowed.text + refused: Final = gateway.request("GET", f"/spend/logs/ui/{rows[4].request_id}", key=key) + assert refused.status_code == 403, refused.text + for caller_key, expected_id in ((key, rows[0].request_id), (other_key, rows[2].request_id)): + payload: Final = gateway.request("GET", f"/spend/logs/ui/{rows[2].request_id}", key=caller_key) + assert payload.status_code == 200, payload.text + assert payload.json()["messages"] == [{"role": "user", "content": expected_id + " payload"}], payload.text + admin: Final = gateway.request("GET", f"/spend/logs/ui/{rows[2].request_id}") + assert admin.status_code == 200, admin.text + assert admin.json()["messages"] == [{"role": "user", "content": rows[2].request_id + " payload"}], admin.text diff --git a/tests/test_litellm/tracing/test_receiver.py b/tests/test_litellm/tracing/test_receiver.py index b8a92606417..36a10ea1e42 100644 --- a/tests/test_litellm/tracing/test_receiver.py +++ b/tests/test_litellm/tracing/test_receiver.py @@ -106,7 +106,7 @@ async def test_empty_export_writes_nothing(): async def test_reads_delegate_to_store(): store = _fake_store() tracing = TraceReceiver(store) - scope: TraceScope = {"team_ids": ("team-research",), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 0, "user_id": "", "team_ids": ("team-research",)} assert await tracing.get_trace("t1", scope) is None store.get_trace.assert_awaited_once_with("t1", scope, "") diff --git a/tests/test_litellm/tracing/test_store.py b/tests/test_litellm/tracing/test_store.py index 0c1820d7c5f..18a5865db7f 100644 --- a/tests/test_litellm/tracing/test_store.py +++ b/tests/test_litellm/tracing/test_store.py @@ -354,7 +354,7 @@ async def test_list_traces_sets_next_cursor_on_full_page(): } client.query = AsyncMock(return_value=[row, {**row, "trace_id": "t1", "trace_ref": "ref1", "start_ms": 900}]) store = TraceStore(client) - scope: TraceScope = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",)} page = await store.list_traces(scope, 0, 2000, limit=2) assert [t["trace_id"] for t in page["data"]] == ["t2", "t1"] @@ -373,7 +373,7 @@ async def test_get_span_not_found_and_found(): client = MagicMock() client.query = AsyncMock(return_value=[]) store = TraceStore(client) - scope: TraceScope = {"all_teams": 1, "user_id": "", "team_ids": (), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 1, "user_id": "", "team_ids": ()} assert await store.get_span("t", "s", scope, "ref") is None stored_input = '[{"role": "user", "content": "hi"}]' client.query = AsyncMock( @@ -425,7 +425,7 @@ async def test_trace_cost_is_scoped_and_counts_repeated_request_once(): ] client.query = AsyncMock(side_effect=[spans, tuple(SpendRow.model_validate({**row, "user": ""}) for row in spend)]) store = TraceStore(client) - scope: TraceScope = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",)} trace = await store.get_trace("trace-1", scope, "ref") @@ -473,7 +473,7 @@ async def test_run_list_uses_matching_spend_and_leaves_missing_cost_unavailable( } ] client.query = AsyncMock(side_effect=[rows, tuple(SpendRow.model_validate({**row, "user": ""}) for row in spend)]) - scope: TraceScope = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",)} page = await TraceStore(client).list_traces(scope, 0, 2000) @@ -484,7 +484,7 @@ async def test_run_list_uses_matching_spend_and_leaves_missing_cost_unavailable( @pytest.mark.asyncio async def test_ambiguous_cache_response_id_keeps_cost_unavailable(): client = MagicMock() - span = _llm_row("llm-1", "", "agent", "response-1", team_id="", api_key_hash="key-a") + span: Final = _llm_row("llm-1", "", "agent", "response-1", team_id="", user_id="user", api_key_hash="key-a") spend = [ { "request_id": request_id, @@ -496,9 +496,11 @@ async def test_ambiguous_cache_response_id_keeps_cost_unavailable(): } for request_id, cost in (("response-1", 0.25), ("response-1_cache_hit123", 0.0)) ] - client.query = AsyncMock(side_effect=[[span], tuple(SpendRow.model_validate({**row, "user": ""}) for row in spend)]) + client.query = AsyncMock( + side_effect=[[span], tuple(SpendRow.model_validate({**row, "user": "user"}) for row in spend)] + ) store = TraceStore(client) - scope: TraceScope = {"all_teams": 0, "user_id": "", "team_ids": (), "api_key_hash": "key-a"} + scope: Final[TraceScope] = {"all_teams": 0, "user_id": "user", "team_ids": ()} trace = await store.get_trace("trace-1", scope, "ref") @@ -521,7 +523,7 @@ async def test_diagnostic_continuation_preserves_content_version_scope_and_unico ] ) store = TraceStore(client) - scope = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",), "api_key_hash": "key-a"} + scope: Final[TraceScope] = {"all_teams": 0, "user_id": "", "team_ids": ("team-a",)} first = await store.get_span_error("trace-1", "span-1", scope, "scoped-run") assert first is not None and first["next_cursor"] is not None last = await store.get_span_error("trace-1", "span-1", scope, "scoped-run", first["next_cursor"]) @@ -548,7 +550,7 @@ async def test_malformed_diagnostic_cursor_never_reaches_storage(cursor): client.query = AsyncMock() with pytest.raises(ValueError, match="Invalid diagnostic cursor"): await TraceStore(client).get_span_error( - "trace", "span", {"all_teams": 1, "user_id": "", "team_ids": (), "api_key_hash": ""}, cursor=cursor + "trace", "span", {"all_teams": 1, "user_id": "", "team_ids": ()}, cursor=cursor ) client.query.assert_not_awaited() @@ -660,7 +662,7 @@ async def test_trace_id_collision_requires_a_visible_reference_before_reading_co storage: Final = MagicMock() storage.query = AsyncMock(return_value=(TraceIdentityRow(trace_ref="first"), TraceIdentityRow(trace_ref="second"))) store: Final = TraceStore(storage) - scope: Final[TraceScope] = {"all_teams": 1, "user_id": "", "team_ids": (), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 1, "user_id": "", "team_ids": ()} with pytest.raises(AmbiguousTraceError, match="provide trace_ref"): await store.get_trace("shared-id", scope) with pytest.raises(AmbiguousTraceError, match="provide trace_ref"): diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 264f489520d..e0208312f18 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -43,6 +43,7 @@ def span_row() -> dict[str, JsonValue]: "name": "root", "type": "agent", "agent": "", + "framework": "", "status": "STATUS_CODE_OK", "status_message": "", "error_truncated": 0, @@ -62,7 +63,7 @@ def span_row() -> dict[str, JsonValue]: @pytest.fixture def span_params() -> dict[str, str | int | list[str]]: - return {"trace_id": "trace-1", "trace_ref": "", "all_teams": 1, "user_id": "", "team_ids": [], "api_key_hash": ""} + return {"trace_id": "trace-1", "trace_ref": "", "all_teams": 1, "user_id": "", "team_ids": []} @pytest.mark.asyncio @@ -128,7 +129,7 @@ async def test_from_env_reads_with_clickhouse_url( recording_server.enqueue(ResponseSpec(body={"data": []})) monkeypatch.setenv("CLICKHOUSE_URL", recording_server.base_url) monkeypatch.delenv("CLICKHOUSE_READER_URL", raising=False) - scope: Final[TraceScope] = {"all_teams": 1, "user_id": "", "team_ids": (), "api_key_hash": ""} + scope: Final[TraceScope] = {"all_teams": 1, "user_id": "", "team_ids": ()} page: Final = await TraceReceiver.from_env().list_traces(scope, 0, 1) assert page == {"data": (), "next_cursor": None} assert len(recording_server.requests) == 1 @@ -295,14 +296,23 @@ async def test_insert_validates_values_without_pydantic_copy(recording_server: R assert stored["SpanAttributes"] == attributes -@pytest.mark.parametrize("role", ["proxy_admin", "proxy_admin_viewer", "internal_user"]) -def test_trace_sql_endpoint_executes_for_admin_and_preserves_clickhouse_envelope( - recording_server: RecordingServer, role: str +@pytest.mark.parametrize( + ("role", "user_id", "expected_status"), + ( + ("proxy_admin", None, 200), + ("proxy_admin_viewer", None, 200), + ("internal_user", "user", 200), + ("internal_user", None, 403), + ), +) +def test_trace_sql_endpoint_enforces_ownership_and_preserves_clickhouse_envelope( + recording_server: RecordingServer, role: str, user_id: str | None, expected_status: int ) -> None: from fastapi import FastAPI from fastapi.testclient import TestClient from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.tracing_endpoints import provide_receiver, provide_trace_query_secret, router @@ -312,19 +322,28 @@ def test_trace_sql_endpoint_executes_for_admin_and_preserves_clickhouse_envelope "rows": 1, "statistics": {"elapsed": 0.01, "rows_read": 1, "bytes_read": 1}, } - recording_server.expected_requests = 12 - for _ in range(11): - recording_server.enqueue(ResponseSpec(body="")) - recording_server.enqueue(ResponseSpec(body=envelope)) + recording_server.expected_requests = 12 if expected_status == 200 else 0 + if expected_status == 200: + for _ in range(11): + recording_server.enqueue(ResponseSpec(body="")) + recording_server.enqueue(ResponseSpec(body=envelope)) storage: Final = ClickHouseStorage(TraceStorageConfig(recording_server.base_url, "trace_test")) app: Final = FastAPI() app.include_router(router) app.dependency_overrides[provide_trace_query_secret] = lambda: "test-master-secret" - app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role, token="test") + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=role, user_id=user_id, token="test") app.dependency_overrides[provide_receiver] = lambda: TraceReceiver(TraceStore(storage)) + + async def permitted_teams(auth: UserAPIKeyAuth) -> tuple[str, ...]: + return () + + app.dependency_overrides[get_log_team_lookup] = lambda: permitted_teams with TestClient(app) as client: result: Final = client.post("/v1/traces/query", json={"sql": "SELECT 42 AS answer"}) - assert result.status_code == 200, result.text + assert result.status_code == expected_status, result.text + if expected_status == 403: + assert result.json() == {"detail": "Not allowed to view logs"} + return assert result.json() == envelope assert recording_server.requests[-1].raw_body == b"SELECT 42 AS answer" assert client.post("/v1/traces/query", json={"sql": " "}).status_code == 400 diff --git a/tests/unit/proxy/auth/test_authorization.py b/tests/unit/proxy/auth/test_authorization.py new file mode 100644 index 00000000000..7d1548dd828 --- /dev/null +++ b/tests/unit/proxy/auth/test_authorization.py @@ -0,0 +1,29 @@ +from typing import Final + +import pytest + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.authorization import OwnedRows, resolve_owned_read_scope, resolve_trace_read_scope + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token", (None, "key")) +async def test_team_membership_or_key_without_user_does_not_grant_log_access(token: str | None) -> None: + async def unexpected_lookup() -> tuple[str, ...]: + pytest.fail("Identity-less callers cannot consult team permissions") + + assert await resolve_trace_read_scope(UserAPIKeyAuth(team_id="team", token=token), unexpected_lookup) is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("token", (None, "key")) +@pytest.mark.parametrize("lookup_fails", (False, True)) +async def test_trace_reads_share_user_and_team_scope_regardless_of_key(token: str | None, lookup_fails: bool) -> None: + async def lookup() -> tuple[str, ...]: + if lookup_fails: + raise RuntimeError("team lookup failed") + return ("permitted",) + + expected: Final = OwnedRows("caller", () if lookup_fails else ("permitted",)) + assert await resolve_owned_read_scope("caller", lookup) == expected + assert await resolve_trace_read_scope(UserAPIKeyAuth(user_id="caller", token=token), lookup) == expected diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index fd747d5a6f2..f00be0f8a65 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -1354,7 +1354,7 @@ async def test_auth_body_read_and_trace_handler_leave_stream_for_receiver_limit( store: Final = MagicMock() store.insert_spans = AsyncMock() context: Final = await tracing_endpoints.provide_trace_access( - auth=UserAPIKeyAuth(token="key", team_id="team"), tracing=TraceReceiver(store) + auth=UserAPIKeyAuth(token="key", team_id="team"), tracing=TraceReceiver(store), log_team_lookup=AsyncMock() ) parsed, parse_error = await _read_request_body_deferring_parse_failure(request) diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_spend_logs.py b/tests/unit/proxy/management_endpoints/management_v1/test_spend_logs.py index b6867d338c5..7523c864985 100644 --- a/tests/unit/proxy/management_endpoints/management_v1/test_spend_logs.py +++ b/tests/unit/proxy/management_endpoints/management_v1/test_spend_logs.py @@ -1,5 +1,5 @@ from datetime import datetime, timezone -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import FastAPI, Request @@ -7,6 +7,7 @@ from fastapi.exceptions import RequestValidationError from fastapi.testclient import TestClient from litellm.proxy._types import LiteLLMRoutes, LitellmUserRoles +from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.list_api.common import ( PROBLEM_TYPE_BASE, @@ -51,7 +52,7 @@ WINDOW = "filter[startTime][gte]=2026-07-23T00:00:00Z&filter[startTime][lte]=202 @pytest.fixture def mock_prisma_client(monkeypatch): prisma_client = MagicMock() - prisma_client.db.query_raw = AsyncMock(return_value=[]) + prisma_client.db.query_raw = AsyncMock(return_value=()) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) return prisma_client @@ -71,9 +72,10 @@ def _mock_rows(mock_prisma_client, end_users: list[str]) -> AsyncMock: return query_raw -def _as_role(role: LitellmUserRoles, user_id): +def _as_role(role: LitellmUserRoles, user_id, log_team_lookup): original = app.dependency_overrides.copy() app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id=user_id, user_role=role) + app.dependency_overrides[get_log_team_lookup] = lambda: log_team_lookup return original @@ -283,13 +285,9 @@ def test_applies_no_scope_for_a_proxy_admin(mock_prisma_client, as_proxy_admin): def test_scopes_a_team_admin_to_their_own_rows_and_teams(mock_prisma_client, role): """A team admin must not see end users belonging to teams they cannot read.""" query_raw = _mock_rows(mock_prisma_client, ["cust-a"]) - original = _as_role(role, user_id="team-admin-1") + original = _as_role(role, user_id="team-admin-1", log_team_lookup=AsyncMock(return_value=("team-a", "team-b"))) try: - with patch( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - new=AsyncMock(return_value=["team-a", "team-b"]), - ): - response = _get() + response = _get() finally: app.dependency_overrides = original @@ -297,24 +295,20 @@ def test_scopes_a_team_admin_to_their_own_rows_and_teams(mock_prisma_client, rol # Same clause shape ui_view_spend_logs builds, so the two cannot diverge. assert '("user" = $3 OR team_id = ANY($4::text[]))' in query_raw.call_args.args[0] assert query_raw.call_args.args[3] == "team-admin-1" - assert query_raw.call_args.args[4] == ["team-a", "team-b"] + assert query_raw.call_args.args[4] == ("team-a", "team-b") def test_scopes_a_teamless_user_to_their_own_rows(mock_prisma_client): query_raw = _mock_rows(mock_prisma_client, []) - original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id="solo") + original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id="solo", log_team_lookup=AsyncMock(return_value=())) try: - with patch( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - new=AsyncMock(return_value=[]), - ): - response = _get() + response = _get() finally: app.dependency_overrides = original assert response.status_code == 200 sql = query_raw.call_args.args[0] - assert '("user" = $3)' in sql + assert '"user" = $3' in sql assert "team_id" not in sql assert query_raw.call_args.args[3] == "solo" @@ -322,13 +316,9 @@ def test_scopes_a_teamless_user_to_their_own_rows(mock_prisma_client): def test_returns_nothing_when_the_caller_owns_no_scope(mock_prisma_client): """Unidentifiable caller must match no rows, never fall through to unscoped.""" query_raw = _mock_rows(mock_prisma_client, []) - original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id=None) + original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id=None, log_team_lookup=AsyncMock(return_value=())) try: - with patch( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - new=AsyncMock(return_value=[]), - ): - response = _get() + response = _get() finally: app.dependency_overrides = original @@ -339,19 +329,17 @@ def test_returns_nothing_when_the_caller_owns_no_scope(mock_prisma_client): def test_scopes_when_the_permitted_team_lookup_fails(mock_prisma_client): """A failed team lookup must degrade to own-rows-only, never to unscoped.""" query_raw = _mock_rows(mock_prisma_client, []) - original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id="solo") + original = _as_role( + LitellmUserRoles.INTERNAL_USER, user_id="solo", log_team_lookup=AsyncMock(side_effect=RuntimeError("db down")) + ) try: - with patch( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - new=AsyncMock(side_effect=RuntimeError("db down")), - ): - response = _get() + response = _get() finally: app.dependency_overrides = original assert response.status_code == 200 sql = query_raw.call_args.args[0] - assert '("user" = $3)' in sql + assert '"user" = $3' in sql assert "team_id" not in sql @@ -422,24 +410,22 @@ def test_user_facet_reads_internal_users_from_spend_logs(mock_prisma_client, as_ def test_user_facet_uses_the_same_team_scope_as_request_logs(mock_prisma_client): query_raw = AsyncMock(return_value=[{"user": "member@example.com"}]) mock_prisma_client.db.query_raw = query_raw - original = _as_role(LitellmUserRoles.INTERNAL_USER, user_id="team-admin-1") + original = _as_role( + LitellmUserRoles.INTERNAL_USER, user_id="team-admin-1", log_team_lookup=AsyncMock(return_value=("team-a",)) + ) try: - with patch( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - new=AsyncMock(return_value=["team-a"]), - ): - response = _get_users() + response = _get_users() finally: app.dependency_overrides = original assert response.status_code == 200 assert '("user" = $3 OR team_id = ANY($4::text[]))' in query_raw.call_args.args[0] assert query_raw.call_args.args[3] == "team-admin-1" - assert query_raw.call_args.args[4] == ["team-a"] + assert query_raw.call_args.args[4] == ("team-a",) def test_user_facet_searches_the_internal_user_value(mock_prisma_client, as_proxy_admin): - query_raw = AsyncMock(return_value=[]) + query_raw = AsyncMock(return_value=()) mock_prisma_client.db.query_raw = query_raw _get_users(f"{WINDOW}&q=alice%40example.com") diff --git a/tests/unit/proxy/spend_tracking/test_log_visibility.py b/tests/unit/proxy/spend_tracking/test_log_visibility.py deleted file mode 100644 index 140225d2000..00000000000 --- a/tests/unit/proxy/spend_tracking/test_log_visibility.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Final - -import pytest -from fastapi import HTTPException - -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.spend_tracking.log_visibility import LogVisibility, log_visibility - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("auth", "expected"), - ( - (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), LogVisibility(all_teams=True)), - (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), LogVisibility(all_teams=True)), - ( - UserAPIKeyAuth(user_id="user", token="key", team_id="unpermitted"), - LogVisibility(user_id="user", team_ids=("permitted",), api_key_hash="key"), - ), - (UserAPIKeyAuth(token="key", team_id="unpermitted"), LogVisibility(api_key_hash="key")), - ), -) -async def test_log_visibility_uses_user_and_permitted_teams_instead_of_key_team_membership( - auth: UserAPIKeyAuth, - expected: LogVisibility, -) -> None: - async def permitted_teams(caller: UserAPIKeyAuth) -> tuple[str, ...]: - assert caller is auth - return ("permitted",) - - assert await log_visibility(auth, permitted_teams) == expected - - -@pytest.mark.asyncio -async def test_missing_team_permissions_preserve_authenticated_user_and_key_visibility() -> None: - async def no_teams(auth: UserAPIKeyAuth) -> tuple[str, ...]: - return () - - auth: Final = UserAPIKeyAuth(user_id="user", token="key", team_id="team") - assert await log_visibility(auth, no_teams) == LogVisibility(user_id=auth.user_id, api_key_hash="key") - - -@pytest.mark.asyncio -async def test_team_membership_without_authenticated_identity_does_not_grant_log_access() -> None: - with pytest.raises(HTTPException) as error: - await log_visibility(UserAPIKeyAuth(team_id="team")) - assert error.value.status_code == 403 diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 506de58e438..6aaa536a945 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -14,6 +14,8 @@ from fastapi.testclient import TestClient import litellm import litellm.proxy.proxy_server as ps +from litellm.proxy.auth.authorization import OwnedRows +from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup, load_permitted_log_team_ids def _default_date_range(): @@ -1656,10 +1658,7 @@ async def test_ui_view_spend_logs_explicit_user_filter_cannot_escape_own_scope(c "litellm.proxy.proxy_server.prisma_client", make_ui_spend_logs_mock_prisma([caller_log], lambda _where: [], query_observer=observe_query), ) - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - AsyncMock(return_value=[]), - ) + monkeypatch.setitem(app.dependency_overrides, get_log_team_lookup, lambda: AsyncMock(return_value=())) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, user_id="caller@example.com" ) @@ -1715,10 +1714,7 @@ async def test_ui_view_spend_logs_without_user_filter_includes_permitted_team_sc "litellm.proxy.proxy_server.prisma_client", make_ui_spend_logs_mock_prisma([caller_log, member_log, outside_log], filter_by_scope), ) - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - AsyncMock(return_value=["team-9"]), - ) + monkeypatch.setitem(app.dependency_overrides, get_log_team_lookup, lambda: AsyncMock(return_value=("team-9",))) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin@example.com" ) @@ -1738,21 +1734,13 @@ async def test_ui_view_spend_logs_without_user_filter_includes_permitted_team_sc @pytest.mark.asyncio -async def test_permitted_team_scope_falls_back_to_own_user_when_lookup_fails(monkeypatch): - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - AsyncMock(side_effect=RuntimeError("database unavailable")), - ) +async def test_permitted_team_scope_falls_back_to_own_user_when_lookup_fails(): + from litellm.proxy.auth.authorization import resolve_owned_read_scope - permitted_team_ids = await spend_management_endpoints._get_permitted_team_ids_for_spend_logs_or_empty( - prisma_client=MagicMock(), - user_api_key_dict=UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, - user_id="caller@example.com", - ), - ) + async def unavailable(): + raise RuntimeError("database unavailable") - assert permitted_team_ids == () + assert await resolve_owned_read_scope("caller", unavailable) == OwnedRows("caller") @pytest.mark.asyncio @@ -1876,10 +1864,7 @@ async def test_ui_view_spend_logs_user_filter_intersects_permitted_team_scope(cl "litellm.proxy.proxy_server.prisma_client", make_ui_spend_logs_mock_prisma([member_log, other_team_log], filter_by_user_and_scope), ) - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - AsyncMock(return_value=["team-9"]), - ) + monkeypatch.setitem(app.dependency_overrides, get_log_team_lookup, lambda: AsyncMock(return_value=("team-9",))) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, user_id="team-admin" ) @@ -2129,61 +2114,6 @@ async def test_ui_view_session_spend_logs_rehydrates_metadata_jsonb_text(client, app.dependency_overrides.pop(ps.user_api_key_auth, None) -@pytest.mark.asyncio -async def test_ui_view_session_spend_logs_scopes_non_admin_to_own_logs(client, monkeypatch): - own_log = { - "id": "log1", - "request_id": "req1", - "session_id": "session-123", - "user": "user-1", - "startTime": "2024-01-01T00:00:00Z", - } - - class MockDB: - async def count(self, *args, **kwargs): - assert kwargs.get("where") == {"session_id": "session-123", "user": "user-1"} - return 1 - - async def query_raw(self, sql_query, session_id, page_size, skip, scoped_user): - assert session_id == "session-123" - assert scoped_user == "user-1" - assert '"user" = $4' in sql_query - return [own_log] - - class MockPrismaClient: - def __init__(self): - self.db = MockDB() - self.db.litellm_spendlogs = self.db - - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MockPrismaClient()) - - async def no_permitted_teams(*args, **kwargs): - return [] - - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - no_permitted_teams, - ) - - app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, user_id="user-1" - ) - - try: - response = client.get( - "/spend/logs/session/ui", - params={"session_id": "session-123", "page": 1, "page_size": 50}, - headers={"Authorization": "Bearer sk-test"}, - ) - - assert response.status_code == 200 - data = response.json() - assert data["total"] == 1 - assert [row["request_id"] for row in data["data"]] == ["req1"] - finally: - app.dependency_overrides.pop(ps.user_api_key_auth, None) - - @pytest.mark.asyncio async def test_ui_view_session_spend_logs_includes_permitted_team_logs(client, monkeypatch): class MockDB: @@ -2200,7 +2130,7 @@ async def test_ui_view_session_spend_logs_includes_permitted_team_logs(client, m async def query_raw(self, sql_query, session_id, page_size, skip, scoped_user, team_ids): assert session_id == "session-123" assert scoped_user == "user-1" - assert team_ids == ["team-9"] + assert tuple(team_ids) == ("team-9",) assert '("user" = $4 OR team_id = ANY($5::text[]))' in sql_query return [ { @@ -2222,10 +2152,7 @@ async def test_ui_view_session_spend_logs_includes_permitted_team_logs(client, m async def permitted_teams(*args, **kwargs): return ["team-9"] - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - permitted_teams, - ) + monkeypatch.setitem(app.dependency_overrides, get_log_team_lookup, lambda: permitted_teams) app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( user_role=LitellmUserRoles.INTERNAL_USER, user_id="user-1" @@ -2658,31 +2585,6 @@ async def test_ui_view_spend_logs_request_id_rejects_foreign_row_inserted_after_ app.dependency_overrides.pop(ps.user_api_key_auth, None) -def _make_payload_lookup_prisma(rows): - """Emulate the detail endpoint's SQL over an in-memory corpus: the owner - pre-check, the caller scope on ``"user"`` and permitted teams, and the - exact-request_id-first ordering with LIMIT 1.""" - - class MockDB: - async def query_raw(self, sql_query, *params): - if 'SELECT DISTINCT "user", team_id' in sql_query: - return _emulate_spend_log_owner_lookup(rows, sql_query, params) - lookup_id = params[0] - matches = [r for r in rows if lookup_id in (r["request_id"], r["litellm_call_id"])] - if '"user" = $2' in sql_query: - team_ids = params[2] if "ANY($3::text[])" in sql_query else () - matches = [r for r in matches if r["user"] == params[1] or r["team_id"] in team_ids] - if "ORDER BY (request_id = $1) DESC" in sql_query: - matches = sorted(matches, key=lambda r: r["request_id"] == lookup_id, reverse=True) - return matches[:1] - - class MockPrisma: - def __init__(self): - self.db = MockDB() - - return MockPrisma() - - def _payload_row(request_id, litellm_call_id, user, prompt): return { "request_id": request_id, @@ -2696,36 +2598,6 @@ def _payload_row(request_id, litellm_call_id, user, prompt): } -@pytest.mark.asyncio -async def test_ui_view_request_response_collision_serves_callers_own_row(client, monkeypatch): - """The attacker's row carries the victim's request_id as its client-set call id - and was written first. Each tenant's detail lookup of that id serves only their - own payload, and an admin's lookup resolves the exact request_id match rather - than whichever colliding row the database happens to return first.""" - prisma = _make_payload_lookup_prisma( - [ - _payload_row("attacker-req", "victim-req", "attacker_user", "attacker prompt"), - _payload_row("victim-req", "victim-call-id", "victim_user", "victim prompt"), - ] - ) - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma) - try: - for role, user_id, own_prompt, other_prompt in ( - (LitellmUserRoles.INTERNAL_USER, "victim_user", "victim prompt", "attacker prompt"), - (LitellmUserRoles.INTERNAL_USER, "attacker_user", "attacker prompt", "victim prompt"), - (LitellmUserRoles.PROXY_ADMIN, "admin", "victim prompt", "attacker prompt"), - ): - app.dependency_overrides[ps.user_api_key_auth] = lambda role=role, user_id=user_id: UserAPIKeyAuth( - user_role=role, user_id=user_id - ) - response = client.get("/spend/logs/ui/victim-req", headers={"Authorization": "Bearer sk-test"}) - assert response.status_code == 200, response.text - assert own_prompt in response.text - assert other_prompt not in response.text - finally: - app.dependency_overrides.pop(ps.user_api_key_auth, None) - - @pytest.mark.asyncio async def test_ui_view_request_response_rejects_foreign_row_inserted_after_owner_check(client, monkeypatch): """Backstop behind the SQL scope on the detail endpoint (the mock ignores the @@ -2818,11 +2690,15 @@ async def test_ui_view_request_response_custom_logger_is_keyed_by_callers_own_re that id as its request_id. The custom logger is asked for the caller's own stored request_id, so the caller gets their payload rather than a 403 from the foreign payload's owner check, and the foreign payload is never fetched.""" - prisma = _make_payload_lookup_prisma( - [ - _payload_row("shared-id", "other-call-id", "other_user", "other tenant prompt"), - _payload_row("caller-req", "shared-id", "caller_user", "caller prompt"), - ] + prisma = MagicMock( + db=MagicMock( + query_raw=AsyncMock( + side_effect=[ + [{"user": "other_user", "team_id": None}, {"user": "caller_user", "team_id": None}], + [_payload_row("caller-req", "shared-id", "caller_user", "caller prompt")], + ] + ) + ) ) cold_storage = { "shared-id": { @@ -3161,10 +3037,7 @@ async def test_ui_view_spend_logs_search_keeps_non_admin_scope(client, monkeypat "litellm.proxy.proxy_server.prisma_client", make_ui_spend_logs_mock_prisma(logs, _search_filter_fn(logs, captured)), ) - monkeypatch.setattr( - "litellm.proxy.spend_tracking.spend_management_endpoints._get_permitted_team_ids_for_spend_logs", - AsyncMock(return_value=[]), - ) + monkeypatch.setitem(app.dependency_overrides, get_log_team_lookup, lambda: AsyncMock(return_value=())) ownership_check = AsyncMock() monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._assert_user_can_view_request_id", @@ -3405,9 +3278,7 @@ async def test_ui_view_spend_logs_with_used_client_oauth_token_filter(client, mo start_date, end_date = _default_date_range() - app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( - user_role=LitellmUserRoles.PROXY_ADMIN - ) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) try: for flag, expected_ids in (("true", ["req-seat"]), ("false", ["req-key"])): response = client.get( @@ -7906,3 +7777,115 @@ def test_capture_rate_reports_an_unreadable_bill_as_502(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) assert response.status_code == 502 assert "HTTP 401" in response.json()["detail"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_id", "owner_user", "owner_team", "permitted", "expected"), + [ + ("caller", "caller", "broken", False, True), + ("caller", "other", "allowed", True, True), + ("caller", "other", "allowed", False, False), + ("caller", "other", None, True, False), + (None, None, None, True, False), + (None, None, "allowed", True, True), + ], +) +async def test_shared_owner_policy_preserves_own_user_and_team_access( + user_id, owner_user, owner_team, permitted, expected +): + from litellm.proxy.auth.authorization import can_read_log_owner + + async def lookup(team_id): + if team_id == "broken": + raise RuntimeError("team lookup failed") + return permitted + + assert await can_read_log_owner(user_id, owner_user, owner_team, lookup) is expected + + +@pytest.mark.asyncio +async def test_shared_owner_policy_propagates_team_lookup_failure(): + from litellm.proxy.auth.authorization import can_read_log_owner + + async def unavailable(team_id): + raise RuntimeError("team lookup failed") + + with pytest.raises(RuntimeError, match="team lookup failed"): + await can_read_log_owner("caller", "other", "team", unavailable) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("params", "expected_status"), + [ + ({"start_date": "invalid", "end_date": "invalid"}, 400), + ({"request_id": "foreign"}, 403), + ], +) +async def test_log_team_dependency_preserves_checks_before_permission_lookup( + client, monkeypatch, params, expected_status +): + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + team_reads = [] + + class TeamTable: + async def find_many(self, where): + team_reads.append(where) + return [] + + cache = UserApiKeyCache() + await cache.async_set_cache( + key="caller", value=LiteLLM_UserTable(user_id="caller", teams=["team"]), model_type=LiteLLM_UserTable + ) + prisma = MagicMock( + db=MagicMock( + query_raw=AsyncMock(return_value=[{"user": "other", "team_id": None}]), + litellm_teamtable=TeamTable(), + ) + ) + monkeypatch.setattr(ps, "prisma_client", prisma) + monkeypatch.setattr(ps, "user_api_key_cache", cache) + monkeypatch.setitem( + app.dependency_overrides, + ps.user_api_key_auth, + lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="caller"), + ) + + response = client.get("/spend/logs/ui", params=params, headers={"Authorization": "Bearer sk-test"}) + + assert response.status_code == expected_status, response.text + assert team_reads == [] + + +@pytest.mark.asyncio +async def test_management_team_lookup_without_memberships_keeps_own_user_scope(): + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.auth.authorization import resolve_owned_read_scope + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + cache = UserApiKeyCache() + await cache.async_set_cache( + key="caller", value=LiteLLM_UserTable(user_id="caller", teams=[]), model_type=LiteLLM_UserTable + ) + auth = UserAPIKeyAuth(user_id="caller", user_role=LitellmUserRoles.INTERNAL_USER) + team_reads = [] + + class TeamTable: + async def find_many(self, where): + team_reads.append(where) + return [] + + prisma = MagicMock(db=MagicMock(litellm_teamtable=TeamTable())) + + async def lookup(): + return await load_permitted_log_team_ids( + auth, prisma_client=prisma, user_api_key_cache=cache, proxy_logging_obj=ps.proxy_logging_obj + ) + + assert await lookup() == () + scope = await resolve_owned_read_scope(auth.user_id, lookup) + assert scope == OwnedRows("caller") + assert team_reads == [] diff --git a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py index 6752c91e9f2..93fae093340 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py @@ -11,7 +11,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest - +from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup from litellm.proxy.spend_tracking.spend_tracking_utils import ( get_spend_by_team, get_spend_by_team_and_customer, @@ -180,6 +180,7 @@ async def test_spend_logs_ui_wraps_params_in_at_time_zone_utc(monkeypatch): mock_request.url.path = "/spend/logs/ui" await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -209,9 +210,7 @@ def _make_ui_spend_logs_mock(count_total, page_rows): """ mock_prisma = MagicMock() mock_prisma.db = MagicMock() - mock_prisma.db.query_raw = AsyncMock( - side_effect=[[{"total_count": count_total}], page_rows] - ) + mock_prisma.db.query_raw = AsyncMock(side_effect=[[{"total_count": count_total}], page_rows]) mock_prisma.db.litellm_spendlogs = MagicMock() mock_prisma.db.litellm_spendlogs.count = AsyncMock(return_value=0) return mock_prisma @@ -244,6 +243,7 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -264,17 +264,13 @@ async def test_spend_logs_ui_uses_bounded_count_not_full_scan(monkeypatch): count_sql = count_call[0][0] assert "COUNT(*) OVER ()" not in count_sql assert "LIMIT" in count_sql and "FROM (" in count_sql, ( - "the total must come from a bounded subquery count, not a full-window " - f"scan. SQL was:\n{count_sql}" - ) - assert count_call[0][-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1, ( - "the bounded count must probe at most cap+1 rows" + f"the total must come from a bounded subquery count, not a full-window scan. SQL was:\n{count_sql}" ) + assert count_call[0][-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1, "the bounded count must probe at most cap+1 rows" page_sql = mock_prisma.db.query_raw.call_args_list[1][0][0] assert "COUNT(*) OVER ()" not in page_sql, ( - "the page query must not carry a window count that forces a full-window " - f"scan. SQL was:\n{page_sql}" + f"the page query must not carry a window count that forces a full-window scan. SQL was:\n{page_sql}" ) assert "GROUP BY" not in count_sql and "DISTINCT ON" not in page_sql, ( "without group_by_session the endpoint must keep raw per-call pagination" @@ -302,9 +298,7 @@ async def test_spend_logs_ui_caps_total_for_large_result_sets(monkeypatch): ) page_rows = [{"request_id": "req-1", "metadata": "{}", "session_id": None}] - mock_prisma = _make_ui_spend_logs_mock( - count_total=SPEND_LOGS_PAGINATION_COUNT_CAP + 1, page_rows=page_rows - ) + mock_prisma = _make_ui_spend_logs_mock(count_total=SPEND_LOGS_PAGINATION_COUNT_CAP + 1, page_rows=page_rows) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) auth = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") @@ -312,6 +306,7 @@ async def test_spend_logs_ui_caps_total_for_large_result_sets(monkeypatch): mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -358,6 +353,7 @@ async def test_spend_logs_ui_empty_page_reports_zero_total(monkeypatch): mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -406,6 +402,7 @@ async def test_spend_logs_ui_out_of_range_page_keeps_total(monkeypatch): mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -553,6 +550,7 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -620,6 +618,7 @@ async def test_spend_logs_ui_group_by_session_offset_pages_for_other_sorts(monke mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, @@ -669,6 +668,7 @@ async def test_spend_logs_ui_request_id_lookup_with_grouping_returns_exact_row(m mock_request.url.path = "/spend/logs/ui" response = await ui_view_spend_logs( + log_team_lookup=await get_log_team_lookup(), request=mock_request, api_key=None, user_id=None, diff --git a/tests/unit/proxy/test_tracing_endpoints.py b/tests/unit/proxy/test_tracing_endpoints.py index fd05d54dcb6..d901222aaf4 100644 --- a/tests/unit/proxy/test_tracing_endpoints.py +++ b/tests/unit/proxy/test_tracing_endpoints.py @@ -5,7 +5,7 @@ Tests for the agent tracing endpoints (litellm/proxy/tracing_endpoints.py). from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager from types import ModuleType -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock import pytest @@ -14,12 +14,14 @@ from fastapi.testclient import TestClient from litellm.proxy import tracing_endpoints from litellm.proxy._types import LitellmUserRoles, ProxyLifespanState, UserAPIKeyAuth +from litellm.proxy.auth.authorization import OwnedRows, ReadScope +from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.tracing_runtime import manage_tracing, provide_storage from litellm.rust_bridge import loader from litellm.rust_bridge.trace_queries import SPAN_DETAIL, SpanDetailParams from litellm.rust_bridge.trace_query_responses import TraceQueryHelp, TraceSQLResponse -from litellm.rust_bridge.traces import AdminQueryScope, ClickHouseStorage, TraceStorageConfig +from litellm.rust_bridge.traces import AllQueryScope, ClickHouseStorage, TraceStorageConfig from litellm.tracing import TraceReceiver, TracingPayloadTooLargeError from litellm.tracing.store import TraceStore from litellm.tracing.types import TraceScope @@ -57,7 +59,11 @@ QUERY_HELP: Final[Mapping[str, object]] = { TEAM_KEY = UserAPIKeyAuth( - token="hashed-key", team_id="team-research", org_id="org-1", user_role=LitellmUserRoles.INTERNAL_USER + user_id="user", + token="hashed-key", + team_id="team-research", + org_id="org-1", + user_role=LitellmUserRoles.INTERNAL_USER, ) TRACE_RESPONSE: Final = { "summary": { @@ -97,38 +103,47 @@ SPAN_DETAIL_RESPONSE: Final = { ( pytest.param( UserAPIKeyAuth(token="admin-key", team_id="team-a", user_role=LitellmUserRoles.PROXY_ADMIN), - TraceScope(all_teams=1, user_id="", team_ids=(), api_key_hash=""), + TraceScope(all_teams=1, user_id="", team_ids=()), True, id="admin", ), pytest.param( UserAPIKeyAuth(token="view-key", team_id="team-a", user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), - TraceScope(all_teams=1, user_id="", team_ids=(), api_key_hash=""), + TraceScope(all_teams=1, user_id="", team_ids=()), False, id="view-only-admin", ), pytest.param( TEAM_KEY, - TraceScope(all_teams=0, user_id="", team_ids=(), api_key_hash="hashed-key"), + TraceScope(all_teams=0, user_id="user", team_ids=()), True, id="team-key", ), pytest.param( - UserAPIKeyAuth(token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER), - TraceScope(all_teams=0, user_id="", team_ids=(), api_key_hash="hashed-key"), + UserAPIKeyAuth(user_id="user", token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER), + TraceScope(all_teams=0, user_id="user", team_ids=()), True, id="teamless-key", ), + pytest.param( + UserAPIKeyAuth(token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER), + None, + True, + id="key-without-user-can-only-write", + ), ), ) def test_trace_read_and_write_permissions( - client: TestClient, receiver: MagicMock, auth: UserAPIKeyAuth, scope: TraceScope, can_write: bool + client: TestClient, receiver: MagicMock, auth: UserAPIKeyAuth, scope: TraceScope | None, can_write: bool ) -> None: client.app.dependency_overrides[user_api_key_auth] = lambda: auth read: Final = client.get("/v1/traces?start_ms=1&end_ms=2") - assert read.status_code == 200, read.text - receiver.list_traces.assert_awaited_once_with(scope=scope, start_ms=1, end_ms=2, cursor=None) + assert read.status_code == (403 if scope is None else 200), read.text + if scope is None: + receiver.list_traces.assert_not_awaited() + else: + receiver.list_traces.assert_awaited_once_with(scope=scope, start_ms=1, end_ms=2, cursor=None) write: Final = client.post("/v1/traces", json={}) assert write.status_code == (200 if can_write else 403), write.text @@ -160,6 +175,11 @@ def client() -> TestClient: app = FastAPI() app.include_router(tracing_endpoints.router) app.dependency_overrides[user_api_key_auth] = lambda: TEAM_KEY + + async def lookup(auth: UserAPIKeyAuth) -> tuple[str, ...]: + return () + + app.dependency_overrides[get_log_team_lookup] = lambda: lookup return TestClient(app) @@ -227,7 +247,7 @@ def test_list_traces_passes_scope_window_and_cursor(client, receiver): assert response.status_code == 200 assert response.json() == {"data": [], "next_cursor": None} receiver.list_traces.assert_awaited_once_with( - scope={"all_teams": 0, "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, + scope={"all_teams": 0, "user_id": "user", "team_ids": ()}, start_ms=1, end_ms=2, cursor="abc", @@ -247,9 +267,7 @@ def test_get_trace_404_and_200(client, receiver): response = client.get("/v1/traces/t1") assert response.status_code == 200 assert response.json() == TRACE_RESPONSE - receiver.get_trace.assert_awaited_with( - "t1", {"all_teams": 0, "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, "" - ) + receiver.get_trace.assert_awaited_with("t1", {"all_teams": 0, "user_id": "user", "team_ids": ()}, "") def test_get_span_404_and_200(client, receiver): @@ -258,9 +276,7 @@ def test_get_span_404_and_200(client, receiver): response = client.get("/v1/traces/t1/spans/s1") assert response.status_code == 200 assert response.json()["span_id"] == "s1" - receiver.get_span.assert_awaited_with( - "t1", "s1", {"all_teams": 0, "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, "" - ) + receiver.get_span.assert_awaited_with("t1", "s1", {"all_teams": 0, "user_id": "user", "team_ids": ()}, "") def test_get_span_serves_ui_content_from_stored_payloads(client): @@ -284,9 +300,7 @@ def test_get_span_serves_ui_content_from_stored_payloads(client): def test_trace_detail_passes_scoped_reference(client, receiver): receiver.get_trace.return_value = TRACE_RESPONSE assert client.get("/v1/traces/t1?trace_ref=run-one").status_code == 200 - receiver.get_trace.assert_awaited_with( - "t1", {"all_teams": 0, "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, "run-one" - ) + receiver.get_trace.assert_awaited_with("t1", {"all_teams": 0, "user_id": "user", "team_ids": ()}, "run-one") def test_invalid_export_and_cursor_are_client_errors(client, receiver): @@ -298,12 +312,34 @@ def test_invalid_export_and_cursor_are_client_errors(client, receiver): assert client.get("/v1/traces?cursor=broken").status_code == 400 -def test_teamless_key_without_token_gets_403_on_reads(client, receiver): - client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER - ) - assert client.get("/v1/traces").status_code == 403 - receiver.list_traces.assert_not_called() +@pytest.mark.parametrize( + "auth", + ( + UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER), + UserAPIKeyAuth(token="key"), + UserAPIKeyAuth(token="key", team_id="unpermitted"), + UserAPIKeyAuth(user_id="", token="key"), + ), +) +def test_key_without_user_cannot_read_traces(client: TestClient, auth: UserAPIKeyAuth) -> None: + storage: Final = MagicMock(spec=ClickHouseStorage) + client.app.dependency_overrides[user_api_key_auth] = lambda: auth + client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: TraceReceiver(TraceStore(storage)) + client.app.dependency_overrides[tracing_endpoints.provide_trace_query_secret] = lambda: "test-secret" + for path in ( + "/v1/traces", + "/v1/traces/t1", + "/v1/traces/t1/spans/s1", + "/v1/traces/t1/spans/s1/error", + "/v1/traces/query/help", + ): + response: Final = client.get(path) + assert response.status_code == 403, response.text + query: Final = client.post("/v1/traces/query", json={"sql": "SELECT * FROM otel_traces"}) + assert query.status_code == 403, query.text + storage.query.assert_not_called() + storage.query_sql.assert_not_called() + storage.query_help.assert_not_called() def test_view_only_admin_cannot_ingest_traces(client, receiver): @@ -477,9 +513,8 @@ def test_lifespan_receivers_are_app_local() -> None: SPAN_DETAIL, SpanDetailParams( all_teams=0, - user_id="", + user_id=TEAM_KEY.user_id, team_ids=(), - api_key_hash=TEAM_KEY.token, trace_id="t1", span_id="first-span", trace_ref="first-run", @@ -489,9 +524,8 @@ def test_lifespan_receivers_are_app_local() -> None: SPAN_DETAIL, SpanDetailParams( all_teams=0, - user_id="", + user_id=TEAM_KEY.user_id, team_ids=(), - api_key_hash=TEAM_KEY.token, trace_id="t1", span_id="second-span", trace_ref="second-run", @@ -582,14 +616,17 @@ def test_lens_reads_from_injected_storage_without_receiver() -> None: @pytest.mark.parametrize( ("auth", "expected_scope"), ( - (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), {"kind": "admin"}), - (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), {"kind": "admin"}), - (TEAM_KEY, {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}), + (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), {"kind": "all"}), + (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), {"kind": "all"}), + (TEAM_KEY, {"kind": "owned", "user_id": "user", "team_ids": ()}), ( - UserAPIKeyAuth(token="project-key", team_id="team-a", project_id="project-a"), - {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "project-key"}, + UserAPIKeyAuth(user_id="user", token="project-key", team_id="team-a", project_id="project-a"), + {"kind": "owned", "user_id": "user", "team_ids": ()}, + ), + ( + UserAPIKeyAuth(user_id="user", token="solo-key"), + {"kind": "owned", "user_id": "user", "team_ids": ()}, ), - (UserAPIKeyAuth(token="solo-key"), {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "solo-key"}), ), ) def test_sql_and_help_use_authenticated_scope( @@ -609,7 +646,7 @@ def test_sql_and_help_use_authenticated_scope( assert help_result.status_code == 200, help_result.text assert help_result.json() == QUERY_HELP receiver.store.storage.query_help.assert_awaited_once_with(expected_scope, "test-secret") - forged: Final = client.post("/v1/traces/query", json={"sql": "SELECT 1", "scope": {"kind": "admin"}}) + forged: Final = client.post("/v1/traces/query", json={"sql": "SELECT 1", "scope": {"kind": "all"}}) assert forged.status_code == 422, forged.text assert receiver.store.storage.query_sql.await_count == 1 @@ -638,7 +675,7 @@ def test_sql_reports_rejected_queries_and_unavailable_readers( result: Final = client.post("/v1/traces/query", json={"sql": "SELECT 1"}) assert result.status_code == status, result.text receiver.store.storage.query_sql.assert_awaited_once_with( - "SELECT 1", {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, "test-secret" + "SELECT 1", {"kind": "owned", "user_id": "user", "team_ids": ()}, "test-secret" ) @@ -648,7 +685,7 @@ def test_query_help_does_not_fall_back_when_reader_provisioning_fails(client: Te result: Final = client.get("/v1/traces/query/help") assert result.status_code == 503, result.text receiver.store.storage.query_help.assert_awaited_once_with( - {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, "test-secret" + {"kind": "owned", "user_id": "user", "team_ids": ()}, "test-secret" ) @@ -668,10 +705,96 @@ def test_queries_require_a_proxy_secret( return assert result.status_code == 200, result.text receiver.store.storage.query_sql.assert_awaited_once_with( - "SELECT 1", {"kind": "logs", "user_id": "", "team_ids": (), "api_key_hash": "hashed-key"}, secret + "SELECT 1", {"kind": "owned", "user_id": "user", "team_ids": ()}, secret ) +@pytest.mark.parametrize( + ("auth", "teams", "expected"), + ( + (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), ("team-a",), (1, "", ())), + (UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY), ("team-a",), (1, "", ())), + (UserAPIKeyAuth(user_id="user", token="key", team_id="unpermitted"), ("a", "b"), (0, "user", ("a", "b"))), + (UserAPIKeyAuth(user_id="user", token="key"), (), (0, "user", ())), + (UserAPIKeyAuth(user_id="user"), ("a",), (0, "user", ("a",))), + ), +) +def test_shared_trace_permissions_reach_read_and_sql_boundaries( + client: TestClient, + auth: UserAPIKeyAuth, + teams: tuple[str, ...], + expected: tuple[Literal[0, 1], str, tuple[str, ...]], +) -> None: + async def lookup(caller: UserAPIKeyAuth) -> tuple[str, ...]: + assert caller is auth + return teams + + team_lookup: Final = AsyncMock(side_effect=lookup) + storage: Final = MagicMock(spec=ClickHouseStorage) + storage.query = AsyncMock(return_value=[{"span_id": "s1", "input": "", "output": "", "attributes": {}}]) + storage.query_sql = AsyncMock(return_value=TraceSQLResponse.model_validate(SQL_ENVELOPE)) + storage.query_help = AsyncMock(return_value=TraceQueryHelp.model_validate(QUERY_HELP)) + client.app.dependency_overrides[user_api_key_auth] = lambda: auth + client.app.dependency_overrides[get_log_team_lookup] = lambda: team_lookup + client.app.dependency_overrides[tracing_endpoints.provide_receiver] = lambda: TraceReceiver(TraceStore(storage)) + client.app.dependency_overrides[tracing_endpoints.provide_trace_query_secret] = lambda: "test-secret" + + response: Final = client.get("/v1/traces/t1/spans/s1?trace_ref=run-one") + assert response.status_code == 200, response.text + assert response.json()["span_id"] == "s1" + storage.query.assert_awaited_once_with( + SPAN_DETAIL, + SpanDetailParams( + all_teams=expected[0], + user_id=expected[1], + team_ids=expected[2], + trace_id="t1", + span_id="s1", + trace_ref="run-one", + ), + ) + sql_response: Final = client.post("/v1/traces/query", json={"sql": "SELECT * FROM otel_traces"}) + assert sql_response.status_code == 200, sql_response.text + assert sql_response.json() == SQL_ENVELOPE + assert client.get("/v1/traces/query/help").json() == QUERY_HELP + query_scope: Final = ( + {"kind": "all"} + if expected[0] + else { + "kind": "owned", + "user_id": expected[1], + "team_ids": expected[2], + } + ) + storage.query_sql.assert_awaited_once_with("SELECT * FROM otel_traces", query_scope, "test-secret") + storage.query_help.assert_awaited_once_with(query_scope, "test-secret") + assert team_lookup.await_count == ( + 3 + if auth.user_id and auth.user_role not in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + else 0 + ) + + +@pytest.mark.parametrize( + ("scope", "expected"), + ( + (OwnedRows(None), ("", ())), + (OwnedRows("user"), ("user", ())), + (OwnedRows("user", ("a", "b")), ("user", ("a", "b"))), + ), +) +def test_trace_storage_permissions_map_owned_rows( + scope: ReadScope, + expected: tuple[str, tuple[str, ...]], +) -> None: + assert tracing_endpoints._trace_scope(scope) == TraceScope(all_teams=0, user_id=expected[0], team_ids=expected[1]) + assert tracing_endpoints.trace_query_scope(scope) == { + "kind": "owned", + "user_id": expected[0], + "team_ids": expected[1], + } + + class _NativeConfig: def __init__(self, database: str, url: str, retention_days: int) -> None: pass @@ -685,7 +808,7 @@ class _NativeReturningHelp(ModuleType): def __init__(self, config: _NativeConfig) -> None: pass - async def query_help(self, scope: AdminQueryScope, secret: str) -> Mapping[str, object]: + async def query_help(self, scope: AllQueryScope, secret: str) -> Mapping[str, object]: return help_payload self.NativeTraceConfig: Final = _NativeConfig @@ -698,7 +821,7 @@ class _NativeReturningHelp(ModuleType): async def test_storage_validates_the_native_query_help_value(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(loader, "_cached_bridge", _NativeReturningHelp(QUERY_HELP)) storage: Final = ClickHouseStorage(TraceStorageConfig("http://clickhouse:8123")) - assert await storage.query_help({"kind": "admin"}, "secret") == TraceQueryHelp.model_validate(QUERY_HELP) + assert await storage.query_help({"kind": "all"}, "secret") == TraceQueryHelp.model_validate(QUERY_HELP) @pytest.mark.parametrize( @@ -726,4 +849,4 @@ async def test_storage_rejects_native_query_help_that_drifts_from_the_contract( monkeypatch.setattr(loader, "_cached_bridge", _NativeReturningHelp({**QUERY_HELP, **drift})) storage: Final = ClickHouseStorage(TraceStorageConfig("http://clickhouse:8123")) with pytest.raises(RuntimeError, match="invalid response"): - await storage.query_help({"kind": "admin"}, "secret") + await storage.query_help({"kind": "all"}, "secret") diff --git a/tests/unit/rust_bridge/test_trace_queries.py b/tests/unit/rust_bridge/test_trace_queries.py index 5ffe9ea0404..76ead3582a4 100644 --- a/tests/unit/rust_bridge/test_trace_queries.py +++ b/tests/unit/rust_bridge/test_trace_queries.py @@ -15,7 +15,6 @@ def test_named_query_rejects_offsets_outside_the_native_integer_range(offset: in "all_teams": 0, "user_id": "", "team_ids": ["team"], - "api_key_hash": "key", "trace_id": "trace", "trace_ref": "ref", "span_id": "span", @@ -31,7 +30,6 @@ def test_named_query_rejects_parameters_for_a_different_query() -> None: all_teams=0, user_id="", team_ids=("team",), - api_key_hash="key", trace_id="trace", trace_ref="ref", span_id="span", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index df968e0bc9f..722f10cd571 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -46901,8 +46901,11 @@ export interface components { expression: string; /** Key */ key: string; - /** Type */ - type: string; + /** + * Type + * @constant + */ + type: "String"; }; /** TraceQueryAttributes */ TraceQueryAttributes: { @@ -46916,8 +46919,11 @@ export interface components { fields: components["schemas"]["TraceQueryAttributeField"][]; /** Scope */ scope: string; - /** Table */ - table: string; + /** + * Table + * @enum {string} + */ + table: "otel_traces" | "agent_traces_by_key" | "spend_logs"; /** Truncated */ truncated: boolean; }; @@ -46977,8 +46983,11 @@ export interface components { sampled_rows: number; /** Scope */ scope: string; - /** Table */ - table: string; + /** + * Table + * @enum {string} + */ + table: "otel_traces" | "agent_traces_by_key" | "spend_logs"; /** Truncated */ truncated: boolean; }; @@ -46989,7 +46998,7 @@ export interface components { /** Path */ path: (string | number)[]; /** Types */ - types: string[]; + types: ("array" | "boolean" | "integer" | "null" | "number" | "object" | "string")[]; }; /** TraceQueryNormalizedField */ TraceQueryNormalizedField: { @@ -46999,8 +47008,11 @@ export interface components { meaning: string; /** Name */ name: string; - /** Table */ - table: string; + /** + * Table + * @enum {string} + */ + table: "otel_traces" | "agent_traces_by_key" | "spend_logs"; /** Type */ type: string; }; @@ -47035,8 +47047,11 @@ export interface components { TraceQueryTable: { /** Columns */ columns: components["schemas"]["TraceQueryColumn"][]; - /** Name */ - name: string; + /** + * Name + * @enum {string} + */ + name: "otel_traces" | "agent_traces_by_key" | "spend_logs"; }; /** TraceSQLResponse */ TraceSQLResponse: { From 8b11b682f46fab627972d7f129bb0dceed6a88a2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:39:06 -0700 Subject: [PATCH 09/42] fix(proxy-extras): retry P3009 when a peer already recovered the named migration row (#44283) * fix(proxy-extras): retry P3009 when a peer already recovered the named migration row * fix(proxy-extras): pin P3009 recovery locals as Final and cover a recovered sibling row * test(proxy-extras): stop the P3009 ledger fakes shadowing the partition detector cursor * fix(proxy-extras): annotate the migration ledger reader locals as Final --------- Co-authored-by: yuneng --- .../litellm_proxy_extras/utils.py | 68 ++++--- .../test_litellm_proxy_extras_utils.py | 172 +++++++++++++++++- 2 files changed, 216 insertions(+), 24 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 2062ca93fb3..245244250ee 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -353,9 +353,8 @@ class ProxyExtrasDBManager: pass @staticmethod - def _failed_migration_logs(migration_name: str) -> Optional[str]: - """Return failed migration logs, or None if the ledger is unavailable.""" - database_url = os.getenv("DATABASE_URL") + def _read_migration_ledger(query: str, params: tuple[str, ...]) -> "tuple[object, ...] | None": + database_url: Final = os.getenv("DATABASE_URL") if not database_url: return None @@ -364,28 +363,37 @@ class ProxyExtrasDBManager: except ImportError: return None - cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url) - ledger_table = psycopg.sql.SQL("{}.{}").format( - psycopg.sql.Identifier( - ProxyExtrasDBManager._prisma_schema_param(database_url) or "public" - ), + cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url) + ledger_table: Final = psycopg.sql.SQL("{}.{}").format( + psycopg.sql.Identifier(ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"), psycopg.sql.Identifier("_prisma_migrations"), ) try: - with psycopg.connect( - cleaned_url, connect_timeout=10, autocommit=True - ) as conn: - row = conn.execute( - psycopg.sql.SQL( - "SELECT logs FROM {} " - "WHERE migration_name = %s AND finished_at IS NULL " - "AND rolled_back_at IS NULL" - ).format(ledger_table), - (migration_name,), - ).fetchone() + with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn: + row: Final = conn.execute(psycopg.sql.SQL(query).format(ledger_table), params).fetchone() except (psycopg.OperationalError, psycopg.DatabaseError): return None - return (row[0] or "") if row else "" + return tuple(row) if row is not None else () + + @staticmethod + def _failed_migration_logs(migration_name: str, started_at: str) -> Optional[str]: + row: Final = ProxyExtrasDBManager._read_migration_ledger( + "SELECT logs FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz " + "AND finished_at IS NULL AND rolled_back_at IS NULL", + (migration_name, started_at), + ) + if row is None: + return None + return row[0] if row and isinstance(row[0], str) else "" + + @staticmethod + def _failed_migration_recovered(migration_name: str, started_at: str) -> bool: + row: Final = ProxyExtrasDBManager._read_migration_ledger( + "SELECT 1 FROM {} WHERE migration_name = %s AND started_at = %s::timestamptz " + "AND (finished_at IS NOT NULL OR rolled_back_at IS NOT NULL)", + (migration_name, started_at), + ) + return bool(row) @staticmethod def _resolve_specific_migration(migration_name: str): @@ -1102,6 +1110,11 @@ class ProxyExtrasDBManager: return match.group(1) if match else None return None + @staticmethod + def _v2_failed_migration_started_at(stderr: str, migration_name: str) -> "str | None": + match: Final = re.search(rf"`{re.escape(migration_name)}` migration started at ([^\r\n]+?) failed", stderr) + return match.group(1) if match else None + @staticmethod def _v2_roll_back_migration_best_effort(migration_name: str) -> None: from litellm_proxy_extras.migration_lock import migration_environment @@ -1130,8 +1143,11 @@ class ProxyExtrasDBManager: if "P3009" in stderr: migration_name = ProxyExtrasDBManager._v2_failed_migration_name(stderr) - if migration_name: - ledger_logs = ProxyExtrasDBManager._failed_migration_logs(migration_name) + started_at: Final = ( + ProxyExtrasDBManager._v2_failed_migration_started_at(stderr, migration_name) if migration_name else None + ) + if migration_name and started_at: + ledger_logs: Final = ProxyExtrasDBManager._failed_migration_logs(migration_name, started_at) if ledger_logs and _MIGRATION_DEADLOCK_MARKER in ledger_logs: logger.info( "Migration %s failed in a concurrent migrate deploy " @@ -1140,6 +1156,14 @@ class ProxyExtrasDBManager: ) ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name) return budget.spend() + if ProxyExtrasDBManager._failed_migration_recovered(migration_name, started_at): + logger.info( + "Migration %s started at %s was already rolled back or completed by a concurrent " + "migrate deploy, retrying", + migration_name, + started_at, + ) + return budget.spend() raise RuntimeError( "Migration completion could not be verified. LiteLLM startup has stopped.\n\n" f"Prisma migration history (migration name and start time):\n{stderr}\n\n" diff --git a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py index 29c9ec56d91..ea4a25283a1 100644 --- a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py +++ b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py @@ -3,6 +3,7 @@ import os import re import sys import threading +from dataclasses import dataclass from pathlib import Path from typing import Final @@ -709,6 +710,9 @@ class TestSpendLogsPartitionDetectionMissingPsycopg: _ATTEMPT_BUDGET = 4 +_P3009_MIGRATION_NAME = "20260415120000_health_check_latest_per_model_index" +_P3009_STARTED_AT = "2026-10-02 23:20:56.439594 UTC" +_P3009_DEADLOCK_LOGS = "ERROR: deadlock detected\nDETAIL: Process 72 waits for ShareLock on transaction 991" _P3005_STDERR = """Error: P3005 @@ -730,6 +734,80 @@ ERROR: relation "SomeTable" already exists """ +def _p3009_stderr(migration_name: str, started_at: str) -> str: + return ( + "Error: P3009\n\n" + "migrate found failed migrations in the target database, new migrations will not be applied. " + "Read more about how to resolve migration issues in a production database: " + "https://pris.ly/d/migrate-resolve\n" + f"The `{migration_name}` migration started at {started_at} failed\n" + ) + + +@dataclass(frozen=True, slots=True) +class _LedgerRow: + migration_name: str + started_at: str + finished: bool = False + rolled_back: bool = False + logs: str | None = None + + +@dataclass(frozen=True, slots=True) +class _LedgerCursor: + row: tuple[object, ...] | None = None + + def fetchone(self) -> tuple[object, ...] | None: + return self.row + + def fetchall(self) -> tuple[tuple[object, ...], ...]: + return () + + +class _LedgerConnection: + def __init__(self, ledger: "_FakeLedger") -> None: + self.ledger = ledger + + def __enter__(self) -> "_LedgerConnection": + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute(self, query: object, params: tuple[object, ...] = ()) -> _LedgerCursor: + return self.ledger.execute(query, params) + + +class _FakeLedger: + def __init__(self, at_error: tuple[_LedgerRow, ...], after_peer: tuple[_LedgerRow, ...]) -> None: + self.rows = at_error + self.after_peer = after_peer + self._peer_observed = False + + def connect(self, *args: object, **kwargs: object) -> _LedgerConnection: + return _LedgerConnection(self) + + def execute(self, query: object, params: tuple[object, ...]) -> _LedgerCursor: + text: Final = str(query) + if "WHERE migration_name = %s" not in text or not params: + return _LedgerCursor() + if not self._peer_observed: + self.rows = self.after_peer + self._peer_observed = True + matching: Final = tuple( + row + for row in self.rows + if row.migration_name == params[0] and (len(params) == 1 or row.started_at == params[1]) + ) + if "rolled_back_at IS NULL" in text: + unresolved: Final = next((row for row in matching if not row.finished and not row.rolled_back), None) + return _LedgerCursor((unresolved.logs,) if unresolved else None) + if "IS NOT NULL" in text: + resolved: Final = next((row for row in matching if row.finished or row.rolled_back), None) + return _LedgerCursor((1,) if resolved else None) + return _LedgerCursor() + + @pytest.mark.parametrize( "pooled,direct,expected", ( @@ -759,7 +837,15 @@ class _MigrateDeployHarness: `prisma migrate deploy` outcomes, with every recovery command faked out so nothing touches a database or the packaged migrations directory.""" - def __init__(self, monkeypatch, tmp_path, outcomes, repeat_last=False, confirmed_migrations=()): + def __init__( + self, + monkeypatch, + tmp_path, + outcomes, + repeat_last=False, + confirmed_migrations=(), + ledger: "_FakeLedger | None" = None, + ): import subprocess as subprocess_module import litellm_proxy_extras.utils as utils_module @@ -772,7 +858,11 @@ class _MigrateDeployHarness: self._subprocess_module = subprocess_module self.confirmed_migrations = set(confirmed_migrations) - monkeypatch.delenv("DATABASE_URL", raising=False) + if ledger is None: + monkeypatch.delenv("DATABASE_URL", raising=False) + else: + monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x") + monkeypatch.setattr("psycopg.connect", ledger.connect) monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(tmp_path)) monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", self._fake_run) monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: {}) @@ -819,6 +909,84 @@ class _MigrateDeployHarness: return True +class TestConcurrentP3009Recovery: + @pytest.mark.parametrize( + "after_peer", + ( + (_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS),), + ( + _LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, rolled_back=True, logs=_P3009_DEADLOCK_LOGS), + _LedgerRow(_P3009_MIGRATION_NAME, "2026-10-02 23:21:11.539224 UTC"), + ), + (_LedgerRow(_P3009_MIGRATION_NAME, _P3009_STARTED_AT, finished=True, logs=_P3009_DEADLOCK_LOGS),), + ), + ids=("rolled-back", "rolled-back-beside-a-fresh-in-flight-row", "finished"), + ) + def test_a_p3009_row_a_peer_already_recovered_is_retried( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + after_peer: tuple[_LedgerRow, ...], + ) -> None: + deadlocked_row: Final = _LedgerRow( + _P3009_MIGRATION_NAME, + _P3009_STARTED_AT, + logs=_P3009_DEADLOCK_LOGS, + ) + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT), "ok"], + ledger=_FakeLedger(at_error=(deadlocked_row,), after_peer=after_peer), + ) + + assert harness.run() is True + assert len(harness.deploy_calls) == 2 + + @pytest.mark.parametrize( + "ledger_rows", + ( + ( + _LedgerRow( + _P3009_MIGRATION_NAME, + _P3009_STARTED_AT, + logs='ERROR: syntax error at or near "SLECT"', + ), + ), + ( + _LedgerRow( + _P3009_MIGRATION_NAME, + _P3009_STARTED_AT, + logs='ERROR: syntax error at or near "SLECT"', + ), + _LedgerRow( + _P3009_MIGRATION_NAME, + "2026-10-02 23:19:40.120000 UTC", + rolled_back=True, + logs=_P3009_DEADLOCK_LOGS, + ), + ), + ), + ids=("only-row", "beside-a-recovered-earlier-attempt"), + ) + def test_an_unresolved_p3009_row_without_the_deadlock_marker_stops( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ledger_rows: tuple[_LedgerRow, ...], + ) -> None: + harness: Final = _MigrateDeployHarness( + monkeypatch, + tmp_path, + [_p3009_stderr(_P3009_MIGRATION_NAME, _P3009_STARTED_AT)], + ledger=_FakeLedger(at_error=ledger_rows, after_peer=ledger_rows), + ) + + with pytest.raises(RuntimeError, match="Migration completion could not be verified"): + harness.run() + assert len(harness.deploy_calls) == 1 + + class TestMigrateDeployAttemptAccounting: def test_a_push_created_database_finishes_bootstrapping(self, monkeypatch, tmp_path): harness = _MigrateDeployHarness( From c5c0a48ef149a6c4077f23861612f2020c1f01ba Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:05:52 -0700 Subject: [PATCH 10/42] fix(bedrock): price amazon nova 2 pro preview at the standard tier (#44302) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 44 ++++++++++--------- model_prices_and_context_window.json | 44 ++++++++++--------- 2 files changed, 48 insertions(+), 40 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2b613ff1aea..31e150f1f7d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -386,16 +386,17 @@ "supports_vision": true }, "amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.125e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_image_token": 1.25e-06, + "input_cost_per_audio_token": 1.25e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -424,16 +425,17 @@ "supports_vision": true }, "apac.amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.4375e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_image_token": 1.375e-06, + "input_cost_per_audio_token": 1.375e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1.1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -462,16 +464,17 @@ "supports_vision": true }, "eu.amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.4375e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_image_token": 1.375e-06, + "input_cost_per_audio_token": 1.375e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1.1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -500,16 +503,17 @@ "supports_vision": true }, "us.amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.4375e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_image_token": 1.375e-06, + "input_cost_per_audio_token": 1.375e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1.1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2b613ff1aea..31e150f1f7d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -386,16 +386,17 @@ "supports_vision": true }, "amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.125e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_image_token": 1.25e-06, + "input_cost_per_audio_token": 1.25e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -424,16 +425,17 @@ "supports_vision": true }, "apac.amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.4375e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_image_token": 1.375e-06, + "input_cost_per_audio_token": 1.375e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1.1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -462,16 +464,17 @@ "supports_vision": true }, "eu.amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.4375e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_image_token": 1.375e-06, + "input_cost_per_audio_token": 1.375e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1.1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -500,16 +503,17 @@ "supports_vision": true }, "us.amazon.nova-2-pro-preview-20251202-v1:0": { - "cache_read_input_token_cost": 5.46875e-07, - "input_cost_per_token": 2.1875e-06, - "input_cost_per_image_token": 2.1875e-06, - "input_cost_per_audio_token": 2.1875e-06, + "cache_read_input_token_cost": 3.4375e-07, + "input_cost_per_token": 1.375e-06, + "input_cost_per_image_token": 1.375e-06, + "input_cost_per_audio_token": 1.375e-06, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.75e-05, + "output_cost_per_token": 1.1e-05, + "source": "https://aws.amazon.com/nova/pricing/", "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, From f63d989ff9437809d509f276001e0fbe135aa198 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 2 Oct 2026 19:37:08 -0700 Subject: [PATCH 11/42] feat: add Bespoke Nimble gateway and OSS classifier support (#44246) * feat: add Bespoke Nimble gateway and OSS classifier support * feat: accept Ollama's nimble model name for the Bespoke provider * test: exempt the POST-only bespoke decisions route from the all-methods check test_pass_through_routes_support_all_methods requires every built-in pass-through route to accept every HTTP method unless it is listed in PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES. /bespoke/v1/systemone is POST-only like /laya/v1/systemone, so the test failed at this branch and passed at the merge base. List it alongside Laya. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 --- litellm/llms/laya/common_utils.py | 48 +------- litellm/llms/oss_decision.py | 56 ++++++++++ ...odel_prices_and_context_window_backup.json | 42 +++++++ litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 24 ++++ litellm/proxy/_types.py | 1 + litellm/proxy/auth/auth_utils.py | 11 +- .../auto_router_permissions.py | 2 +- .../llm_passthrough_endpoints.py | 25 ++++- .../pass_through_endpoints.py | 26 +++-- .../pass_through_endpoints/success_handler.py | 2 +- .../complexity_router/complexity_router.py | 14 +-- .../complexity_router/config.py | 22 ++-- .../complexity_router/jev_classifier.py | 6 +- model_prices_and_context_window.json | 42 +++++++ provider_endpoints_support.json | 7 ++ .../test_pass_through_unit_tests.py | 1 + tests/unit/llms/laya/test_common_utils.py | 42 +------ tests/unit/llms/test_oss_decision.py | 60 ++++++++++ tests/unit/proxy/auth/test_auth_utils.py | 20 ++-- .../test_model_management_endpoints.py | 6 +- .../test_auto_router_permissions.py | 8 +- ...st_typesafe_passthrough_logging_handler.py | 51 +++++---- .../test_llm_pass_through_endpoints.py | 103 ++++++++++-------- .../complexity_router/test_jev_classifier.py | 44 ++++---- .../test_auto_router_model_naming.py | 1 + .../add_model/AutoRouterClassifierTabs.tsx | 8 +- .../JevClassifierConfig.integration.test.tsx | 25 +++-- .../add_model/JevClassifierConfig.tsx | 21 ++-- ...d_auto_router_routing_test_request.test.ts | 21 +++- .../build_complexity_router_config.test.ts | 33 +++++- .../add_model/jev_classifier_config.ts | 27 +++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 47 +++++++- 33 files changed, 577 insertions(+), 270 deletions(-) create mode 100644 litellm/llms/oss_decision.py create mode 100644 tests/unit/llms/test_oss_decision.py diff --git a/litellm/llms/laya/common_utils.py b/litellm/llms/laya/common_utils.py index f400eef22d3..3e423a9e742 100644 --- a/litellm/llms/laya/common_utils.py +++ b/litellm/llms/laya/common_utils.py @@ -1,51 +1,7 @@ from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Final, Literal, TypeAlias +from typing import Final -from pydantic import AnyHttpUrl, BaseModel, TypeAdapter, ValidationError - -from litellm.secret_managers.main import get_secret_str - -LayaCheckpoint: TypeAlias = Literal["english", "multilingual", "typed-decisions"] - - -def validate_laya_model(value: object) -> LayaCheckpoint: - try: - return TypeAdapter(LayaCheckpoint).validate_python(value) - except ValidationError as exc: - raise ValueError("Laya model must be 'english', 'multilingual', or 'typed-decisions'") from exc - - -def validate_laya_request(body: Mapping[str, object]) -> LayaCheckpoint: - if "custom_body" in body: - raise ValueError("custom_body is not supported for Laya requests") - if body.get("stream"): - raise ValueError("Streaming is not supported for Laya requests") - return validate_laya_model(body.get("model")) - - -@dataclass(frozen=True, slots=True) -class LayaConnection: - api_base: str - api_key: str | None = field(repr=False) - - -def validate_laya_api_base(value: str) -> str: - try: - url: Final = TypeAdapter(AnyHttpUrl).validate_python(value) - except ValidationError as exc: - raise ValueError("Laya api_base must be an HTTP or HTTPS server URL") from exc - if url.username or url.password or url.query or url.fragment: - raise ValueError("Laya api_base must not contain credentials, a query, or a fragment") - return str(url).rstrip("/") - - -def laya_connection(api_base: str | None = None, api_key: str | None = None) -> LayaConnection: - base: Final = api_base if api_base is not None else get_secret_str("LAYA_API_BASE") - if not base: - raise ValueError("Laya requires api_base or LAYA_API_BASE pointing to a self-hosted server") - key: Final = api_key if api_base is not None else api_key or get_secret_str("LAYA_API_KEY") - return LayaConnection(api_base=validate_laya_api_base(base), api_key=key) +from pydantic import BaseModel, TypeAdapter, ValidationError class _LayaRouting(BaseModel): diff --git a/litellm/llms/oss_decision.py b/litellm/llms/oss_decision.py new file mode 100644 index 00000000000..1483adad93f --- /dev/null +++ b/litellm/llms/oss_decision.py @@ -0,0 +1,56 @@ +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from pydantic import AnyHttpUrl, TypeAdapter, ValidationError + +from litellm.secret_managers.main import get_secret_str + +OssDecisionProvider: TypeAlias = Literal["laya", "bespoke"] +OSS_DECISION_MODELS: Final = MappingProxyType( + { + "laya": ("english", "multilingual", "typed-decisions"), + "bespoke": ("nimble-latest", "nimble", "bespokelabs/Bespoke-Nimble-9B"), + } +) + + +def validate_oss_model(provider: OssDecisionProvider, value: object) -> str: + if not isinstance(value, str) or value not in OSS_DECISION_MODELS[provider]: + raise ValueError(f"{provider} model must be one of {', '.join(OSS_DECISION_MODELS[provider])}") + return value + + +def validate_oss_request(provider: OssDecisionProvider, body: Mapping[str, object]) -> str: + if "custom_body" in body: + raise ValueError(f"custom_body is not supported for {provider} requests") + if body.get("stream"): + raise ValueError(f"Streaming is not supported for {provider} requests") + return validate_oss_model(provider, body.get("model")) + + +@dataclass(frozen=True, slots=True) +class OssDecisionConnection: + api_base: str + api_key: str | None = field(repr=False) + + +def validate_oss_api_base(provider: OssDecisionProvider, value: str) -> str: + try: + url: Final = TypeAdapter(AnyHttpUrl).validate_python(value) + except ValidationError as exc: + raise ValueError(f"{provider} api_base must be an HTTP or HTTPS server URL") from exc + if url.username or url.password or url.query or url.fragment: + raise ValueError(f"{provider} api_base must not contain credentials, a query, or a fragment") + return str(url).rstrip("/") + + +def oss_connection( + provider: OssDecisionProvider, api_base: str | None = None, api_key: str | None = None +) -> OssDecisionConnection: + base: Final = api_base if api_base is not None else get_secret_str(f"{provider.upper()}_API_BASE") + if not base: + raise ValueError(f"{provider} requires api_base or {provider.upper()}_API_BASE pointing to its server") + key: Final = api_key if api_base is not None else api_key or get_secret_str(f"{provider.upper()}_API_KEY") + return OssDecisionConnection(api_base=validate_oss_api_base(provider, base), api_key=key) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 31e150f1f7d..21bdee73464 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -72632,6 +72632,48 @@ "supports_audio_input": true, "supports_video_input": true }, + "bespoke/nimble-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, + "bespoke/nimble": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ollama.com/library/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately" + } + }, + "bespoke/bespokelabs/Bespoke-Nimble-9B": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/english": { "input_cost_per_token": 0.0, "litellm_provider": "laya", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 837552e522f..54d757d75aa 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -230,6 +230,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/transcribe", "/typesafe/", "/laya/", + "/bespoke/", "/openrouter/", "/vertex-ai/", "/vertex_ai/", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 3346b0c9ff8..727ecb235bb 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -26318,6 +26318,30 @@ ] } }, + "/bespoke/v1/systemone": { + "post": { + "operationId": "bespoke_proxy_route_bespoke_v1_systemone_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Bespoke Proxy Route", + "tags": [ + "llm_passthrough" + ] + } + }, "/cohere/{endpoint}": { "delete": { "description": "[Docs](https://docs.litellm.ai/docs/pass_through/cohere)", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 2084f6b6ee3..59302e9f08e 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -511,6 +511,7 @@ class LiteLLMRoutes(enum.Enum): "/mistral", "/typesafe", "/laya", + "/bespoke", "/openrouter", "/milvus", "/gigachat", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index e5a1430b3d8..7ef82009184 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1883,15 +1883,16 @@ def _extract_model_candidates_from_request( llm_router: Router | None = None, team_id: str | None = None, ) -> list[str]: - if route.rstrip("/") == "/laya/v1/systemone": - from litellm.llms.laya.common_utils import validate_laya_model + if route.rstrip("/") in ("/laya/v1/systemone", "/bespoke/v1/systemone"): + from litellm.llms.oss_decision import validate_oss_model + provider: Final = "bespoke" if route.startswith("/bespoke/") else "laya" try: - laya_request: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data) - laya_model: Final = validate_laya_model(laya_request.get("model")) + decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data) + decision_model: Final = validate_oss_model(provider, decision_request.get("model")) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - return _dedupe_model_candidates((f"laya/{laya_model}",)) + return _dedupe_model_candidates((f"{provider}/{decision_model}",)) if route == "/cost/predict-cache": prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload return _dedupe_model_candidates(prediction_models) diff --git a/litellm/proxy/management_helpers/auto_router_permissions.py b/litellm/proxy/management_helpers/auto_router_permissions.py index e0d8fda5b1c..5845194fa9b 100644 --- a/litellm/proxy/management_helpers/auto_router_permissions.py +++ b/litellm/proxy/management_helpers/auto_router_permissions.py @@ -74,7 +74,7 @@ class _MemberOpenSourceClassifierConfig(BaseModel): model_config = ConfigDict(extra="forbid") - provider: Literal["jev", "laya"] = "jev" + provider: Literal["jev", "laya", "bespoke"] = "jev" model: str api_key: None = None api_base: None = None diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index ed2ea475c7a..0cff558985b 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -58,8 +58,8 @@ from litellm.llms.deepgram.common_utils import ( deepgram_listen_websocket_target, ) from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base -from litellm.llms.laya.common_utils import laya_connection, validate_laya_request from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path +from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.passthrough.main import AsyncPassthroughStreamingResponse from litellm.proxy._types import * @@ -646,17 +646,32 @@ async def laya_proxy_route( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + return await _oss_decision_proxy_route("laya", request, fastapi_response, user_api_key_dict) + + +@router.post("/bespoke/v1/systemone", tags=["Bespoke Nimble Pass-through", "pass-through"]) +async def bespoke_proxy_route( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> Response: + return await _oss_decision_proxy_route("bespoke", request, fastapi_response, user_api_key_dict) + + +async def _oss_decision_proxy_route( + provider: OssDecisionProvider, request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth ) -> Response: body: Final = TypeAdapter(dict[str, object]).validate_python(await _read_request_body(request)) try: - _ = validate_laya_request(body) + _ = validate_oss_request(provider, body) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc try: - connection: Final = laya_connection() + connection: Final = oss_connection(provider) except ValueError as exc: raise HTTPException( - status_code=503, detail="Laya server is not configured correctly; check LAYA_API_BASE" + status_code=503, detail=f"{provider} server is not configured correctly; check {provider.upper()}_API_BASE" ) from exc base_url: Final = httpx.URL(connection.api_base) updated_url: Final = base_url.copy_with( @@ -671,7 +686,7 @@ async def laya_proxy_route( endpoint="v1/systemone", target=str(updated_url), custom_headers=MappingProxyType({**authorization, "Content-Type": "application/json"}), - custom_llm_provider="laya", + custom_llm_provider=provider, is_streaming_request=False, ) return TypeAdapter(Response, config=ConfigDict(arbitrary_types_allowed=True)).validate_python( diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 865374a0430..a5b414e5a18 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -65,7 +65,7 @@ from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) from litellm.llms.custom_httpx.http_handler import get_async_httpx_client -from litellm.llms.laya.common_utils import validate_laya_request +from litellm.llms.oss_decision import validate_oss_request from litellm.passthrough import BasePassthroughUtils from litellm.proxy._types import ( ConfigFieldInfo, @@ -387,7 +387,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): @staticmethod def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType: parsed_url: Final = urlparse(url) - if custom_llm_provider == "typesafe" and parsed_url.path.removesuffix("/").endswith("/v1/systemone"): + if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith( + "/v1/systemone" + ): return EndpointType.DECISIONS if ( ("generateContent") in url @@ -1163,10 +1165,10 @@ async def pass_through_request( pricing_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) _strip_client_pricing_overrides(pricing_body) _parsed_body = pricing_body - if custom_llm_provider == "laya": - laya_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) - checkpoint: Final = validate_laya_request(laya_request) - _parsed_body["model"] = f"laya/{checkpoint}" + if custom_llm_provider in ("laya", "bespoke"): + decision_request: Final = TypeAdapter(Mapping[str, object]).validate_python(_parsed_body) + checkpoint: Final = validate_oss_request(custom_llm_provider, decision_request) + _parsed_body["model"] = f"{custom_llm_provider}/{checkpoint}" ### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ### # Passthrough endpoints are opt-in only for guardrails @@ -1223,17 +1225,19 @@ async def pass_through_request( call_type="pass_through_endpoint", endpoint_type=endpoint_type, ) - if custom_llm_provider == "laya": + if custom_llm_provider in ("laya", "bespoke"): hook_body: Final = TypeAdapter(dict[str, object]).validate_python(_parsed_body) hook_model: Final = hook_body.get("model") - laya_body: Final = MappingProxyType( + decision_body: Final = MappingProxyType( { **hook_body, - "model": hook_model.removeprefix("laya/") if isinstance(hook_model, str) else hook_model, + "model": hook_model.removeprefix(f"{custom_llm_provider}/") + if isinstance(hook_model, str) + else hook_model, } ) - _ = validate_laya_request(laya_body) - _parsed_body = TypeAdapter(dict[str, object]).validate_python(laya_body) + _ = validate_oss_request(custom_llm_provider, decision_body) + _parsed_body = TypeAdapter(dict[str, object]).validate_python(decision_body) resolved_timeout: Final = resolve_pass_through_request_timeout(timeout) async_client_obj: Final = get_async_httpx_client( llm_provider=httpxSpecialProvider.PassThroughEndpoint, diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 3c4733d0bf0..da3e28e25e4 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -336,7 +336,7 @@ class PassThroughEndpointLogging: kwargs = transcribe_handler_result["kwargs"] # rebind-ok: elif-chain contract elif ( self.is_typesafe_route(custom_llm_provider) - or custom_llm_provider == "laya" + or custom_llm_provider in ("laya", "bespoke") or self.is_openrouter_decisions_route(url_route, custom_llm_provider) ): from .llm_provider_handlers.typesafe_passthrough_logging_handler import ( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 39fb237917c..3610991a20d 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1309,15 +1309,15 @@ class ComplexityRouter(CustomLogger): @staticmethod def _build_jev_client(config: OpenSourceClassifierConfig) -> JevClassifierClient: - if config.provider == "laya": - from litellm.llms.laya.common_utils import laya_connection + if config.provider in ("laya", "bespoke"): + from litellm.llms.oss_decision import oss_connection - connection: Final = laya_connection(config.api_base, config.api_key) + connection: Final = oss_connection(config.provider, config.api_base, config.api_key) return HttpJevClassifierClient( api_key=connection.api_key, api_base=connection.api_base, http_client=get_async_httpx_client(httpxSpecialProvider.PassThroughEndpoint), - provider="laya", + provider=config.provider, ) api_key: Final = config.api_key or get_secret_str("TYPESAFE_API_KEY") if not api_key: @@ -2228,7 +2228,7 @@ class ComplexityRouter(CustomLogger): if not self._tier_pools().get(tier_name): raise ValueError(f"Jev classifier returned tier {tier_name!r}, which has no models configured") model: Final = response.model or config.model - accounting_provider: Final = "laya" if config.provider == "laya" else "typesafe" + accounting_provider: Final = "typesafe" if config.provider == "jev" else config.provider verdict: Final = JevVerdict( label=answer.choice, probabilities=answer.probabilities, @@ -2243,8 +2243,8 @@ class ComplexityRouter(CustomLogger): tier=tier, score=None, signals=( - f"{'laya' if config.provider == 'laya' else 'jev'}-classifier:{tier_name}", - f"{'laya' if config.provider == 'laya' else 'jev'}-confidence={answer.confidence:.6f}", + f"{config.provider}-classifier:{tier_name}", + f"{config.provider}-confidence={answer.confidence:.6f}", *( f"tier-probability:{label}={probability:.6f}" for label, probability in answer.probabilities.items() diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index 41f389db7d8..88907731468 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -698,17 +698,17 @@ def normalize_classifier_config_aliases(config: Mapping[str, object]) -> Mapping class OpenSourceClassifierConfig(BaseModel): model_config = ConfigDict(extra="forbid", frozen=True) - provider: Literal["jev", "laya"] = "jev" + provider: Literal["jev", "laya", "bespoke"] = "jev" model: str = "jev-latest" - api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted Laya") + api_key: str | None = Field(default=None, description="Provider API key; optional for self-hosted providers") api_base: str | None = Field( default=None, - description="Provider API base; defaults to TYPESAFE_API_BASE or LAYA_API_BASE for the selected provider", + description="Provider API base; defaults to the selected provider API_BASE environment variable", ) timeout_ms: int = Field(default=3000, ge=1) instructions: str | None = Field( default=None, - description="Replaces the built-in Jev question instructions", + description="Replaces the built-in classification instructions", ) circuit_breaker_enabled: bool = True circuit_breaker_cooldown_seconds: float = Field(default=30.0, gt=0.0) @@ -729,17 +729,19 @@ class OpenSourceClassifierConfig(BaseModel): @classmethod def _reject_blank_api_key(cls, value: str | None) -> str | None: if value is not None and not value.strip(): - raise ValueError("opensource_classifier_config.api_key must be non-empty; omit it to use TYPESAFE_API_KEY") + raise ValueError( + "opensource_classifier_config.api_key must be non-empty; omit it to use the provider environment key" + ) return value @model_validator(mode="after") def _keep_the_environment_key_on_the_environment_base(self) -> "OpenSourceClassifierConfig": - if self.provider == "laya": - from litellm.llms.laya.common_utils import validate_laya_api_base, validate_laya_model + if self.provider in ("laya", "bespoke"): + from litellm.llms.oss_decision import validate_oss_api_base, validate_oss_model - _ = validate_laya_model(self.model) + _ = validate_oss_model(self.provider, self.model) if self.api_base is not None: - _ = validate_laya_api_base(self.api_base) + _ = validate_oss_api_base(self.provider, self.api_base) return self if self.api_base is not None and self.api_key is None: raise ValueError( @@ -1150,7 +1152,7 @@ class ComplexityRouterConfig(BaseModel): "an LLM tier-selection call, a Switchyard-compatible capability forecast, a joint Fuse V2 forecast, " "a custom classifier plugin, 'heuristic_first', which scores locally and only pays for the LLM classifier when the " "local scorer does not confidently land a cheap tier, or 'hybrid', which trusts the local scorer " - "everywhere except when its score lands near a tier boundary, or 'oss_classifier', a structured choice call using Jev or Laya" + "everywhere except when its score lands near a tier boundary, or 'oss_classifier', a structured choice call using Jev, Laya or Bespoke Nimble" ), ) llm_v2_config: LLMV2Config | None = Field( diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index 073d87c25a6..2b97ae824dd 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -84,7 +84,7 @@ class HttpJevClassifierClient: api_key: str | None, api_base: str, http_client: AsyncHTTPHandler, - provider: Literal["typesafe", "laya"] = "typesafe", + provider: Literal["typesafe", "laya", "bespoke"] = "typesafe", ) -> None: self._api_key = api_key self._api_base = api_base.rstrip("/") @@ -201,7 +201,7 @@ class JevVerdict(NamedTuple): confidence: float model: str cost: float | None - provider: Literal["typesafe", "laya"] = "typesafe" + provider: Literal["typesafe", "laya", "bespoke"] = "typesafe" class _RegistryPricing(BaseModel): @@ -225,7 +225,7 @@ def build_jev_request( def jev_classifier_cost( - response: JevSystemOneResponse, configured_model: str, provider: Literal["typesafe", "laya"] = "typesafe" + response: JevSystemOneResponse, configured_model: str, provider: Literal["typesafe", "laya", "bespoke"] = "typesafe" ) -> float | None: usage: Final = response.usage if usage is None: diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 31e150f1f7d..21bdee73464 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -72632,6 +72632,48 @@ "supports_audio_input": true, "supports_video_input": true }, + "bespoke/nimble-latest": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, + "bespoke/nimble": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://ollama.com/library/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model under the name Ollama serves it as; infrastructure costs are paid separately" + } + }, + "bespoke/bespokelabs/Bespoke-Nimble-9B": { + "input_cost_per_token": 0.0, + "litellm_provider": "bespoke", + "max_input_tokens": 8192, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://github.com/bespokelabsai/nimble", + "supported_endpoints": [ + "/v1/systemone" + ], + "metadata": { + "notes": "Self-hosted decision model; infrastructure costs are paid separately" + } + }, "laya/english": { "input_cost_per_token": 0.0, "litellm_provider": "laya", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index d18f8d2e6d1..223711a92b6 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1477,6 +1477,13 @@ "rerank": false } }, + "bespoke": { + "display_name": "Bespoke Nimble (`bespoke`)", + "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", + "endpoints": { + "systemone": true + } + }, "laya": { "display_name": "Laya (`laya`)", "url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers", diff --git a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py index 7fb23223845..82cb652950c 100644 --- a/tests/pass_through_unit_tests/test_pass_through_unit_tests.py +++ b/tests/pass_through_unit_tests/test_pass_through_unit_tests.py @@ -416,6 +416,7 @@ PROTOCOL_CONSTRAINED_PASS_THROUGH_ROUTES = { "/transcribe/{operation}": {"POST"}, "/tinyfish/{endpoint:path}": {"GET", "POST"}, "/laya/v1/systemone": {"POST"}, + "/bespoke/v1/systemone": {"POST"}, } diff --git a/tests/unit/llms/laya/test_common_utils.py b/tests/unit/llms/laya/test_common_utils.py index c9ee0062cd2..408bd300beb 100644 --- a/tests/unit/llms/laya/test_common_utils.py +++ b/tests/unit/llms/laya/test_common_utils.py @@ -1,48 +1,8 @@ from collections.abc import Mapping -from typing import Final import pytest -from litellm.llms.laya.common_utils import laya_connection, laya_response_model - - -@pytest.mark.parametrize( - ("base", "key", "expected_base", "expected_key"), - [ - (None, None, "http://laya.test/root", "laya-env-key"), - ("http://custom.test/", None, "http://custom.test", None), - ("http://custom.test/", "explicit-key", "http://custom.test", "explicit-key"), - ], -) -def test_laya_credentials_stay_with_their_configured_destination( - monkeypatch: pytest.MonkeyPatch, - base: str | None, - key: str | None, - expected_base: str, - expected_key: str | None, -) -> None: - monkeypatch.setenv("LAYA_API_BASE", "http://laya.test/root/") - monkeypatch.setenv("LAYA_API_KEY", "laya-env-key") - monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-this") - connection: Final = laya_connection(base, key) - assert (connection.api_base, connection.api_key) == (expected_base, expected_key) - assert "key" not in repr(connection) - - -@pytest.mark.parametrize( - "base", - ["", "ftp://laya.test", "http://user:password@laya.test", "https://laya.test?key=x", "http://laya.test/#x"], -) -def test_laya_rejects_ambiguous_server_urls(base: str) -> None: - with pytest.raises(ValueError, match="Laya"): - laya_connection(base) - - -def test_laya_missing_server_does_not_fall_back_to_typesafe(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv("LAYA_API_BASE", raising=False) - monkeypatch.setenv("TYPESAFE_API_BASE", "https://typesafe.test") - with pytest.raises(ValueError, match="LAYA_API_BASE"): - laya_connection() +from litellm.llms.laya.common_utils import laya_response_model @pytest.mark.parametrize( diff --git a/tests/unit/llms/test_oss_decision.py b/tests/unit/llms/test_oss_decision.py new file mode 100644 index 00000000000..05c5d2bbff5 --- /dev/null +++ b/tests/unit/llms/test_oss_decision.py @@ -0,0 +1,60 @@ +from typing import Final + +import pytest + +from litellm.llms.oss_decision import OssDecisionProvider, oss_connection, validate_oss_request + +pytestmark: Final = pytest.mark.parametrize("provider", ["laya", "bespoke"]) + + +@pytest.mark.parametrize( + ("base", "key", "expected_base", "expected_key"), + [ + (None, None, "http://decision.test/root", "oss-env-key"), + ("http://custom.test/", None, "http://custom.test", None), + ("http://custom.test/", "explicit-key", "http://custom.test", "explicit-key"), + ], +) +def test_oss_credentials_stay_with_their_configured_destination( + monkeypatch: pytest.MonkeyPatch, + provider: OssDecisionProvider, + base: str | None, + key: str | None, + expected_base: str, + expected_key: str | None, +) -> None: + monkeypatch.setenv(f"{provider.upper()}_API_BASE", "http://decision.test/root/") + monkeypatch.setenv(f"{provider.upper()}_API_KEY", "oss-env-key") + monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-this") + monkeypatch.setenv("NIMBLE_API_KEY", "never-send-nimble-search-key") + connection: Final = oss_connection(provider, base, key) + assert (connection.api_base, connection.api_key) == (expected_base, expected_key) + assert "key" not in repr(connection) + + +@pytest.mark.parametrize( + "base", + ["", "ftp://laya.test", "http://user:password@laya.test", "https://laya.test?key=x", "http://laya.test/#x"], +) +def test_oss_rejects_ambiguous_server_urls(provider: OssDecisionProvider, base: str) -> None: + with pytest.raises(ValueError, match=provider): + oss_connection(provider, base) + + +def test_oss_missing_server_does_not_fall_back_to_typesafe( + monkeypatch: pytest.MonkeyPatch, provider: OssDecisionProvider +) -> None: + monkeypatch.delenv(f"{provider.upper()}_API_BASE", raising=False) + monkeypatch.setenv("TYPESAFE_API_BASE", "https://typesafe.test") + monkeypatch.setenv("NIMBLE_API_BASE", "https://nimble-search.test") + with pytest.raises(ValueError, match=f"{provider.upper()}_API_BASE"): + oss_connection(provider) + + +def test_oss_request_accepts_the_name_ollama_serves_nimble_under_only_for_bespoke(provider: OssDecisionProvider) -> None: + body: Final = {"model": "nimble"} + if provider == "bespoke": + assert validate_oss_request(provider, body) == "nimble" + return + with pytest.raises(ValueError, match=f"{provider} model must be one of"): + validate_oss_request(provider, body) diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 6cf4456a0ff..bc4a6e0155d 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -463,16 +463,22 @@ def test_get_model_from_request_no_request_extracts_model(): ) -@pytest.mark.parametrize("model", ["english", "multilingual", "typed-decisions"]) -@pytest.mark.parametrize("route", ["/laya/v1/systemone", "/laya/v1/systemone/"]) -def test_laya_native_model_uses_the_classifier_permission_identity(model: str, route: str) -> None: - assert get_model_from_request(request_data={"model": model}, route=route) == f"laya/{model}" +@pytest.mark.parametrize("provider,model", [ + ("laya", "english"), ("laya", "multilingual"), ("laya", "typed-decisions"), + ("bespoke", "nimble-latest"), ("bespoke", "bespokelabs/Bespoke-Nimble-9B"), +]) +@pytest.mark.parametrize("suffix", ["", "/"]) +def test_oss_native_model_uses_the_classifier_permission_identity(provider: str, model: str, suffix: str) -> None: + assert get_model_from_request( + request_data={"model": model}, route=f"/{provider}/v1/systemone{suffix}" + ) == f"{provider}/{model}" -@pytest.mark.parametrize("model", [None, "", "auto", "laya/english", "unknown", ["english"], 7]) -def test_laya_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(model: object) -> None: +@pytest.mark.parametrize("provider", ["laya", "bespoke"]) +@pytest.mark.parametrize("model", [None, "", "auto", "laya/english", "bespoke/nimble-latest", "unknown", ["english"], 7]) +def test_oss_native_model_cannot_implicitly_select_an_unauthorized_checkpoint(provider: str, model: object) -> None: with pytest.raises(HTTPException) as denied: - get_model_from_request(request_data={"model": model}, route="/laya/v1/systemone") + get_model_from_request(request_data={"model": model}, route=f"/{provider}/v1/systemone") assert denied.value.status_code == 400 diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 7eda03c560b..19cc8bf15ab 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -7706,6 +7706,10 @@ class TestTeamMemberAutoRouterWrites: @pytest.mark.parametrize( "stored_provider,stored_base,supplied,expected_transport", [ + ("bespoke", "https://decision.test", {"provider": "bespoke", "model": "nimble-latest"}, {"api_base": "https://decision.test", "api_key": "stored-secret"}), + ("bespoke", "https://decision.test", {"provider": "bespoke", "model": "nimble-latest", "api_base": "https://new.test"}, {}), + ("bespoke", "https://decision.test", {"provider": "laya", "model": "english"}, {}), + ("laya", "https://decision.test", {"provider": "bespoke", "model": "nimble-latest"}, {}), ("laya", "https://decision.test", {"provider": "laya", "model": "english"}, {"api_base": "https://decision.test", "api_key": "stored-secret"}), ("laya", "https://decision.test", {"provider": "laya", "model": "english", "api_base": "https://decision.test"}, {"api_base": "https://decision.test", "api_key": "stored-secret"}), ( @@ -7743,7 +7747,7 @@ class TestTeamMemberAutoRouterWrites: "model": "auto_router/complexity_router", "complexity_router_config": self._classifier_config( { - "provider": stored_provider, "model": "english" if stored_provider == "laya" else "jev-latest", + "provider": stored_provider, "model": {"laya": "english", "bespoke": "nimble-latest"}.get(stored_provider, "jev-latest"), "api_base": stored_base, "api_key": "stored-secret", }, stored_legacy, diff --git a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py index b60fd4ac7ad..1ccfbab7b1f 100644 --- a/tests/unit/proxy/management_helpers/test_auto_router_permissions.py +++ b/tests/unit/proxy/management_helpers/test_auto_router_permissions.py @@ -144,6 +144,8 @@ def test_tier_config_is_normalized_and_unknown_router_extras_are_rejected() -> N ({"api_base": "https://collector.invalid", "api_key": ""}, "opensource_classifier_config.api_key"), ({"provider": "laya", "model": "english", "api_base": "https://collector.invalid"}, "api_base"), ({"provider": "laya", "model": "english", "api_key": "sk-member"}, "api_key"), + ({"provider": "bespoke", "model": "nimble-latest", "api_base": "https://collector.invalid"}, "api_base"), + ({"provider": "bespoke", "model": "nimble-latest", "api_key": "sk-member"}, "api_key"), ], ) @pytest.mark.parametrize("legacy", [False, True]) @@ -162,7 +164,7 @@ def test_members_cannot_move_the_jev_classifier_off_the_proxys_typesafe_account( assert denied.value.detail == f"Invalid member auto-router configuration at {rejected_at}." -@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-preview"), ("laya", "english")]) +@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-preview"), ("laya", "english"), ("bespoke", "nimble-latest")]) @pytest.mark.parametrize("legacy", [False, True]) def test_members_can_still_tune_the_jev_classifier(provider: str, model: str, legacy: bool) -> None: validated: Final = validate_member_auto_router_config( @@ -348,7 +350,7 @@ async def test_member_dependencies_require_plain_configured_models(target: str) @pytest.mark.asyncio @pytest.mark.parametrize("restricted", ["key", "team", None]) -@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-latest"), ("laya", "english")]) +@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-latest"), ("laya", "english"), ("bespoke", "nimble-latest")]) async def test_jev_evaluation_requires_model_access_but_no_completion_deployment( catalog: Router, restricted: str | None, provider: str, model: str ) -> None: @@ -376,7 +378,7 @@ async def test_jev_evaluation_requires_model_access_but_no_completion_deployment @pytest.mark.asyncio @pytest.mark.parametrize("restricted", ["member", "project", "organization", None]) -@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-latest"), ("laya", "english")]) +@pytest.mark.parametrize(("provider", "model"), [("typesafe", "jev-latest"), ("laya", "english"), ("bespoke", "nimble-latest")]) async def test_jev_evaluation_obeys_each_containing_scope( catalog: Router, restricted: str | None, provider: str, model: str ) -> None: diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py index acf05dcdfde..7961d2a911b 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py @@ -142,60 +142,65 @@ def test_success_handler_dispatches_to_typesafe_handler(): @pytest.mark.asyncio @pytest.mark.parametrize("guardrail_cost", [0.0, 0.25]) @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"]) -@pytest.mark.parametrize("routing_model", ["multilingual", None]) -async def test_laya_gateway_accounts_for_checkpoint_usage_and_registered_cost( - monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float +@pytest.mark.parametrize("provider,requested,routing_model", [ + ("laya", "english", "multilingual"), ("laya", "english", None), + ("bespoke", "nimble-latest", None), + ("bespoke", "bespokelabs/Bespoke-Nimble-9B", None), +]) +async def test_oss_gateway_accounts_for_checkpoint_usage_and_registered_cost( + monkeypatch: pytest.MonkeyPatch, routing_model: str | None, metadata_slot: str, guardrail_cost: float, + provider: str, requested: str ) -> None: - checkpoint: Final = routing_model or "english" - model: Final = f"laya/{checkpoint}" + checkpoint: Final = routing_model or requested + model: Final = f"{provider}/{checkpoint}" input_rate: Final = 0.002 output_rate: Final = 0.005 monkeypatch.setitem(litellm.model_cost, model, { "input_cost_per_token": input_rate, "output_cost_per_token": output_rate, - "litellm_provider": "laya", "mode": "evaluation", + "litellm_provider": provider, "mode": "evaluation", }) start: Final = datetime.now() logging_obj: Final = Logging( - model="english", messages=[], stream=False, call_type="pass_through_endpoint", - start_time=start, litellm_call_id="laya-accounting", function_id="laya-accounting", kwargs={}, + model=requested, messages=[], stream=False, call_type="pass_through_endpoint", + start_time=start, litellm_call_id="oss-accounting", function_id="oss-accounting", kwargs={}, ) from fastapi import Request from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.pass_through_endpoints.pass_through_endpoints import HttpPassThroughEndpointHelpers request: Final = Request({ - "type": "http", "method": "POST", "path": "/laya/v1/systemone", + "type": "http", "method": "POST", "path": f"/{provider}/v1/systemone", "headers": [], "query_string": b"", }) auth: Final = UserAPIKeyAuth( - api_key="laya-budget-key", token="laya-budget-key", - model_max_budget={"laya/english": {"budget_limit": 0.01, "time_period": "1d"}}, + api_key="oss-budget-key", token="oss-budget-key", + model_max_budget={f"{provider}/{requested}": {"budget_limit": 0.01, "time_period": "1d"}}, ) - request_body: Final = {"model": "english", metadata_slot: {"model_group": "unbounded-client-choice"}} + request_body: Final = {"model": requested, metadata_slot: {"model_group": "unbounded-client-choice"}} logging_kwargs: Final = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint( request=request, user_api_key_dict=auth, logging_obj=logging_obj, - passthrough_logging_payload={"url": "https://laya.test/v1/systemone"}, _parsed_body=request_body, + passthrough_logging_payload={"url": f"https://{provider}.test/v1/systemone"}, _parsed_body=request_body, ) logging_kwargs["litellm_params"]["metadata"]["standard_logging_guardrail_information"] = [ {"guardrail_name": "trusted-hook", "guardrail_cost": guardrail_cost}, ] logging_obj.update_environment_variables( - model="english", user="unknown", optional_params={}, + model=requested, user="unknown", optional_params={}, litellm_params=logging_kwargs["litellm_params"], call_type="pass_through_endpoint", ) body: Final = { - "model": "laya-rl-agent", "usage": {"input_tokens": 10, "output_tokens": 3}, + "model": "laya-rl-agent" if provider == "laya" else requested, "usage": {"input_tokens": 10, "output_tokens": 3}, **({"routing": {"model": routing_model}} if routing_model else {}), } normalized: Final = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload( - httpx_response=httpx.Response(200, request=httpx.Request("POST", "https://laya.test/v1/systemone"), json=body), - response_body=body, request_body={"model": "english"}, logging_obj=logging_obj, - url_route="https://laya.test/v1/systemone", result="{}", start_time=start, - end_time=datetime.now(), cache_hit=False, custom_llm_provider="laya", **logging_kwargs, + httpx_response=httpx.Response(200, request=httpx.Request("POST", f"https://{provider}.test/v1/systemone"), json=body), + response_body=body, request_body={"model": requested}, logging_obj=logging_obj, + url_route=f"https://{provider}.test/v1/systemone", result="{}", start_time=start, + end_time=datetime.now(), cache_hit=False, custom_llm_provider=provider, **logging_kwargs, ) logged: Final = normalized["kwargs"] expected_cost: Final = 10 * input_rate + 3 * output_rate - assert (logged["model"], logged["custom_llm_provider"]) == (model, "laya") + assert (logged["model"], logged["custom_llm_provider"]) == (model, provider) assert logged["response_cost"] == pytest.approx(expected_cost) assert logged["combined_usage_object"].model_dump(exclude_none=True) == { "prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13, @@ -203,7 +208,7 @@ async def test_laya_gateway_accounts_for_checkpoint_usage_and_registered_cost( assert logging_obj.model_call_details["model"] == model assert logging_obj.model_call_details["response_cost"] == pytest.approx(expected_cost) assert logged["standard_logging_object"]["model"] == model - assert logged["standard_logging_object"]["model_group"] == "laya/english" + assert logged["standard_logging_object"]["model_group"] == f"{provider}/{requested}" assert logged["standard_logging_object"]["response_cost"] == pytest.approx(expected_cost + guardrail_cost) from litellm.caching.caching import DualCache @@ -211,10 +216,10 @@ async def test_laya_gateway_accounts_for_checkpoint_usage_and_registered_cost( from litellm.proxy.hooks.model_max_budget_limiter import _PROXY_VirtualKeyModelMaxBudgetLimiter budget_limiter: Final = _PROXY_VirtualKeyModelMaxBudgetLimiter(DualCache()) - assert await budget_limiter.is_key_within_model_budget(auth, "laya/english") + assert await budget_limiter.is_key_within_model_budget(auth, f"{provider}/{requested}") await budget_limiter.async_log_success_event(logged, None, start, datetime.now()) with pytest.raises(BudgetExceededError): - await budget_limiter.is_key_within_model_budget(auth, "laya/english") + await budget_limiter.is_key_within_model_budget(auth, f"{provider}/{requested}") def test_openrouter_decisions_response_is_priced_from_request_model_registry_row(): diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 22171f4ffb0..cad065c64d1 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -7298,6 +7298,8 @@ class TestTypeSafePassthroughRoute: "provider, endpoint, is_decision_request", ( ("typesafe", "systemone", True), + ("laya", "systemone", True), + ("bespoke", "systemone", True), ("typesafe", "systemone/", True), ("typesafe", "systemone?trace=1", True), ("typesafe", "systemone/?trace=1", True), @@ -7316,7 +7318,7 @@ class TestTypeSafePassthroughRoute: self, client: TestClient, monkeypatch: pytest.MonkeyPatch, - provider: Literal["typesafe", "openrouter"], + provider: Literal["typesafe", "openrouter", "laya", "bespoke"], endpoint: str, is_decision_request: bool, quota_scope: Literal["key", "project_output"], @@ -7337,12 +7339,15 @@ class TestTypeSafePassthroughRoute: monkeypatch.setattr(proxy_server, "proxy_logging_obj", ProxyLogging(user_api_key_cache=cache)) monkeypatch.setenv("OPENROUTER_API_KEY", "openrouter-test-key") monkeypatch.setenv("OPENROUTER_API_BASE", "https://typesafe.example/base") - model: Final = "jev-latest" if provider == "typesafe" else "test-generative-model" + monkeypatch.setenv("LAYA_API_BASE", "https://typesafe.example/base") + monkeypatch.setenv("BESPOKE_API_BASE", "https://typesafe.example/base") + model: Final = {"typesafe": "jev-latest", "laya": "english", "bespoke": "nimble-latest"}.get(provider, "test-generative-model") + permission_model: Final = f"{provider}/{model}" if provider in ("laya", "bespoke") else model auth: Final = UserAPIKeyAuth( api_key="sk-limited", tpm_limit=token_limit if quota_scope == "key" else None, project_id="test-project" if quota_scope == "project_output" else None, - project_metadata={"model_otpm_limit": {model: token_limit}} if quota_scope == "project_output" else {}, + project_metadata={"model_otpm_limit": {permission_model: token_limit}} if quota_scope == "project_output" else {}, ) monkeypatch.setitem(proxy_server.app.dependency_overrides, user_api_key_auth, lambda: auth) body: Final = ( @@ -7409,36 +7414,44 @@ class TestTypeSafePassthroughRoute: ) -class TestLayaPassthroughRoute: +@pytest.mark.parametrize("provider", ["laya", "bespoke"]) +class TestOssDecisionPassthroughRoute: @pytest.fixture - def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + def checkpoint(self, provider: str) -> str: + return "english" if provider == "laya" else "nimble-latest" + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch, provider: str) -> Iterator[TestClient]: from litellm.proxy.proxy_server import app - monkeypatch.setenv("LAYA_API_BASE", "http://laya.test/base") + monkeypatch.setenv(f"{provider.upper()}_API_BASE", f"http://{provider}.test/base") monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-typesafe-key") - monkeypatch.delenv("LAYA_API_KEY", raising=False) + monkeypatch.delenv(f"{provider.upper()}_API_KEY", raising=False) monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) yield TestClient(app) - @pytest.mark.parametrize("api_key", [None, "laya-provider-key"]) - def test_laya_forwards_native_decisions_without_gateway_or_typesafe_credentials( - self, client: TestClient, monkeypatch: pytest.MonkeyPatch, api_key: str | None + @pytest.mark.parametrize("api_key", [None, "oss-provider-key"]) + def test_oss_forwards_native_decisions_without_gateway_or_typesafe_credentials( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, api_key: str | None, provider: str, checkpoint: str ) -> None: if api_key is not None: - monkeypatch.setenv("LAYA_API_KEY", api_key) + monkeypatch.setenv(f"{provider.upper()}_API_KEY", api_key) body: Final = { - "model": "english", + "model": checkpoint, "state": "refund", "questions": {"department": {"type": "choice", "criteria": {"billing": "refunds"}}}, } - answer: Final = {"model": "laya-rl-agent", "routing": {"model": "english"}, "answers": {}} + answer: Final = { + "model": "laya-rl-agent" if provider == "laya" else checkpoint, "answers": {}, + **({"routing": {"model": checkpoint}} if provider == "laya" else {}), + } with respx.mock(assert_all_called=True) as upstream: - route: Final = upstream.post("http://laya.test/base/v1/systemone?trace=yes").respond(200, json=answer) + route: Final = upstream.post(f"http://{provider}.test/base/v1/systemone?trace=yes").respond(200, json=answer) response: Final = client.post( - "/laya/v1/systemone?trace=yes", + f"/{provider}/v1/systemone?trace=yes", json=body, headers={"Authorization": "Bearer sk-virtual", "x-pass-authorization": "Bearer attacker"}, ) @@ -7448,26 +7461,26 @@ class TestLayaPassthroughRoute: assert sent.headers.get("authorization") == (f"Bearer {api_key}" if api_key else None) assert json.loads(sent.content) == body - def test_laya_missing_server_fails_without_contacting_another_provider( - self, client: TestClient, monkeypatch: pytest.MonkeyPatch + def test_oss_missing_server_fails_without_contacting_another_provider( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, provider: str, checkpoint: str ) -> None: - monkeypatch.delenv("LAYA_API_BASE") + monkeypatch.delenv(f"{provider.upper()}_API_BASE") with respx.mock(assert_all_called=False) as upstream: - response: Final = client.post("/laya/v1/systemone", json={"model": "english"}) + response: Final = client.post(f"/{provider}/v1/systemone", json={"model": checkpoint}) assert response.status_code == 503 - assert "LAYA_API_BASE" in response.text + assert f"{provider.upper()}_API_BASE" in response.text assert len(upstream.calls) == 0 - def test_laya_does_not_forward_unsupported_endpoints(self, client: TestClient) -> None: + def test_oss_does_not_forward_unsupported_endpoints(self, client: TestClient, provider: str, checkpoint: str) -> None: with respx.mock(assert_all_called=False) as upstream: - response: Final = client.post("/laya/v1/evaluate", json={"model": "english"}) + response: Final = client.post(f"/{provider}/v1/evaluate", json={"model": checkpoint}) assert response.status_code == 404 assert len(upstream.calls) == 0 @pytest.mark.parametrize("model", [None, "auto", "jev-latest"]) - def test_laya_rejects_implicit_checkpoint_selection(self, client: TestClient, model: str | None) -> None: + def test_oss_rejects_implicit_checkpoint_selection(self, client: TestClient, model: str | None, provider: str) -> None: with respx.mock(assert_all_called=False) as upstream: - response: Final = client.post("/laya/v1/systemone", json={"model": model}) + response: Final = client.post(f"/{provider}/v1/systemone", json={"model": model}) assert response.status_code == 400 assert len(upstream.calls) == 0 @@ -7475,19 +7488,19 @@ class TestLayaPassthroughRoute: "controls", [{"custom_body": {"model": "multilingual", "state": "refund"}}, {"stream": True}, {"stream": "true"}], ) - def test_laya_rejects_controls_that_change_authorized_body_or_usage_accounting( - self, client: TestClient, controls: Mapping[str, object] + def test_oss_rejects_controls_that_change_authorized_body_or_usage_accounting( + self, client: TestClient, controls: Mapping[str, object], provider: str, checkpoint: str ) -> None: with respx.mock(assert_all_called=False) as upstream: - route: Final = upstream.post("http://laya.test/base/v1/systemone").respond(200, json={"answers": {}}) - response: Final = client.post("/laya/v1/systemone", json={"model": "english", **controls}) + route: Final = upstream.post(f"http://{provider}.test/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post(f"/{provider}/v1/systemone", json={"model": checkpoint, **controls}) assert response.status_code == 400 assert not route.called @pytest.mark.parametrize("metadata_slot", ["metadata", "litellm_metadata"]) - def test_laya_hooks_enforce_canonical_model_limits_and_keep_native_wire_body( - self, client: TestClient, monkeypatch: pytest.MonkeyPatch, metadata_slot: str + def test_oss_hooks_enforce_canonical_model_limits_and_keep_native_wire_body( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, metadata_slot: str, provider: str, checkpoint: str ) -> None: from litellm.integrations.custom_logger import CustomLogger from litellm.proxy.hooks.parallel_request_limiter_v3 import _PROXY_MaxParallelRequestsHandler_v3 @@ -7497,7 +7510,7 @@ class TestLayaPassthroughRoute: cache: Final = DualCache() limiter: Final = _PROXY_MaxParallelRequestsHandler_v3(internal_usage_cache=InternalUsageCache(cache)) auth: Final = UserAPIKeyAuth( - api_key="laya-native-rpm", metadata={"model_rpm_limit": {"laya/english": 1}}, + api_key="oss-native-rpm", metadata={"model_rpm_limit": {f"{provider}/{checkpoint}": 1}}, ) def authenticated_key() -> UserAPIKeyAuth: return auth @@ -7509,7 +7522,7 @@ class TestLayaPassthroughRoute: self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict[str, object], call_type: CallTypesLiteral, ) -> dict[str, object]: - assert data["model"] == "laya/english" + assert data["model"] == f"{provider}/{checkpoint}" metadata: Final = data.get(metadata_slot) assert isinstance(metadata, dict) assert "standard_logging_guardrail_information" not in metadata @@ -7519,40 +7532,42 @@ class TestLayaPassthroughRoute: monkeypatch.setattr(litellm, "callbacks", [LimitHook()]) body: Final = { - "model": "english", "state": "refund", + "model": checkpoint, "state": "refund", metadata_slot: { "customer_label": "retained", "model_group": "unbounded-client-choice", "standard_logging_guardrail_information": [{"guardrail_cost": 25.0}], }, } with respx.mock(assert_all_called=True) as upstream: - route: Final = upstream.post("http://laya.test/base/v1/systemone").respond(200, json={"answers": {}}) - first: Final = client.post("/laya/v1/systemone", json=body) - second: Final = client.post("/laya/v1/systemone", json=body) + route: Final = upstream.post(f"http://{provider}.test/base/v1/systemone").respond(200, json={"answers": {}}) + first: Final = client.post(f"/{provider}/v1/systemone", json=body) + second: Final = client.post(f"/{provider}/v1/systemone", json=body) assert first.status_code == 200, first.text assert second.status_code == 429, second.text assert route.call_count == 1 - assert json.loads(route.calls.last.request.content) == {"model": "english", "state": "refund"} + assert json.loads(route.calls.last.request.content) == {"model": checkpoint, "state": "refund"} - def test_laya_preserves_trusted_hook_checkpoint_changes( - self, client: TestClient, monkeypatch: pytest.MonkeyPatch + def test_oss_preserves_trusted_hook_checkpoint_changes( + self, client: TestClient, monkeypatch: pytest.MonkeyPatch, provider: str, checkpoint: str ) -> None: from litellm.integrations.custom_logger import CustomLogger + changed_checkpoint: Final = "multilingual" if provider == "laya" else "bespokelabs/Bespoke-Nimble-9B" + class CheckpointHook(CustomLogger): async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, cache: DualCache, data: dict[str, object], call_type: CallTypesLiteral, ) -> dict[str, object]: - assert data["model"] == "laya/english" - return {**data, "model": "laya/multilingual"} + assert data["model"] == f"{provider}/{checkpoint}" + return {**data, "model": f"{provider}/{changed_checkpoint}"} monkeypatch.setattr(litellm, "callbacks", [CheckpointHook()]) with respx.mock(assert_all_called=True) as upstream: - route: Final = upstream.post("http://laya.test/base/v1/systemone").respond(200, json={"answers": {}}) - response: Final = client.post("/laya/v1/systemone", json={"model": "english", "state": "refund"}) + route: Final = upstream.post(f"http://{provider}.test/base/v1/systemone").respond(200, json={"answers": {}}) + response: Final = client.post(f"/{provider}/v1/systemone", json={"model": checkpoint, "state": "refund"}) assert response.status_code == 200, response.text - assert json.loads(route.calls.last.request.content) == {"model": "multilingual", "state": "refund"} + assert json.loads(route.calls.last.request.content) == {"model": changed_checkpoint, "state": "refund"} class TestFalAIPassthroughRoute: diff --git a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py index affcfdc789c..418bf522b6a 100644 --- a/tests/unit/router_strategy/complexity_router/test_jev_classifier.py +++ b/tests/unit/router_strategy/complexity_router/test_jev_classifier.py @@ -451,7 +451,7 @@ def test_jev_config_requires_classifier_config() -> None: ) @pytest.mark.parametrize( ("provider", "model", "canonical_provider"), - [(None, "jev-latest", "jev"), ("typesafe", "jev-latest", "jev"), ("jev", "jev-latest", "jev"), ("laya", "english", "laya")], + [(None, "jev-latest", "jev"), ("typesafe", "jev-latest", "jev"), ("jev", "jev-latest", "jev"), ("laya", "english", "laya"), ("bespoke", "nimble-latest", "bespoke")], ) def test_classifier_aliases_load_and_serialize_one_canonical_config( classifier_type: str, config_key: str, provider: str | None, model: str, canonical_provider: str @@ -474,45 +474,47 @@ def test_classifier_aliases_load_and_serialize_one_canonical_config( assert incoming == original -@pytest.mark.parametrize("config", [{"provider": "laya"}, {"provider": "laya", "model": " "}]) -def test_laya_requires_its_own_checkpoint(config: Mapping[str, object]) -> None: - with pytest.raises(ValueError, match="Laya model must be"): - JevClassifierConfig.model_validate(config) +@pytest.mark.parametrize("provider", ["laya", "bespoke"]) +@pytest.mark.parametrize("model", [None, " "]) +def test_oss_requires_its_own_checkpoint(provider: str, model: str | None) -> None: + with pytest.raises(ValueError, match=f"{provider} model must be"): + JevClassifierConfig.model_validate({"provider": provider, **({"model": model} if model is not None else {})}) @pytest.mark.asyncio +@pytest.mark.parametrize("provider,model", [("laya", "english"), ("bespoke", "nimble-latest")]) @pytest.mark.parametrize("custom_base", [False, True]) @pytest.mark.parametrize("legacy", [False, True]) -async def test_laya_routes_with_its_own_credentials_and_accounts_the_checkpoint( - monkeypatch: pytest.MonkeyPatch, custom_base: bool, legacy: bool +async def test_oss_routes_with_its_own_credentials_and_accounts_the_checkpoint( + monkeypatch: pytest.MonkeyPatch, custom_base: bool, legacy: bool, provider: str, model: str ) -> None: monkeypatch.setenv("TYPESAFE_API_KEY", "never-send-typesafe-key") - monkeypatch.setenv("LAYA_API_BASE", "https://laya.test") - monkeypatch.setenv("LAYA_API_KEY", "laya-env-key") + monkeypatch.setenv(f"{provider.upper()}_API_BASE", f"https://{provider}.test") + monkeypatch.setenv(f"{provider.upper()}_API_KEY", "oss-env-key") monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setitem(litellm.model_cost, "laya/english", {"input_cost_per_token": 0.01}) - recorder: Final = _UsageRecorder("laya/english") + monkeypatch.setitem(litellm.model_cost, f"{provider}/{model}", {"input_cost_per_token": 0.01}) + recorder: Final = _UsageRecorder(f"{provider}/{model}") monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) router: Final = ComplexityRouter( - "laya-route", + f"{provider}-route", litellm.Router(model_list=[]), { "classifier_type": "jev" if legacy else "oss_classifier", "jev_classifier_config" if legacy else "opensource_classifier_config": { - "provider": "laya", - "model": "english", - **({"api_base": "https://laya.test"} if custom_base else {}), + "provider": provider, + "model": model, + **({"api_base": f"https://{provider}.test"} if custom_base else {}), }, "tiers": {"SIMPLE": "cheap"}, }, derive_savings_baseline=False, ) with respx.mock(assert_all_called=True) as upstream: - route: Final = upstream.post("https://laya.test/v1/systemone").respond( + route: Final = upstream.post(f"https://{provider}.test/v1/systemone").respond( 200, json={ - "model": "laya-rl-agent", - "routing": {"model": "english"}, + "model": "laya-rl-agent" if provider == "laya" else model, + **({"routing": {"model": model}} if provider == "laya" else {}), "answers": {"tier": _answer().model_dump()}, "usage": {"input_tokens": 31, "output_tokens": 0}, }, @@ -522,11 +524,11 @@ async def test_laya_routes_with_its_own_credentials_and_accounts_the_checkpoint( assert outcome.cause == "jev_classifier" assert outcome.jev_verdict is not None - assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == ("laya", "english") + assert (outcome.jev_verdict.provider, outcome.jev_verdict.model) == (provider, model) assert outcome.classifier_cost == pytest.approx(0.31) sent: Final = route.calls.last.request - assert sent.headers.get("authorization") == (None if custom_base else "Bearer laya-env-key") - assert json.loads(sent.content)["model"] == "english" + assert sent.headers.get("authorization") == (None if custom_base else "Bearer oss-env-key") + assert json.loads(sent.content)["model"] == model assert len(recorder.calls) == 1 assert recorder.calls[0]["response_cost"] == pytest.approx(0.31) diff --git a/tests/unit/router_utils/test_auto_router_model_naming.py b/tests/unit/router_utils/test_auto_router_model_naming.py index 4881b850f2a..87b23ce93ae 100644 --- a/tests/unit/router_utils/test_auto_router_model_naming.py +++ b/tests/unit/router_utils/test_auto_router_model_naming.py @@ -39,6 +39,7 @@ SEMANTIC_FIELDS = frozenset({"auto_router_config", "auto_router_default_model", ("typesafe", "jev-preview", "typesafe"), ("jev", "jev-preview", "typesafe"), ("laya", "english", "laya"), + ("bespoke", "nimble-latest", "bespoke"), ], ) def test_open_source_classifier_enumerates_its_accounting_model( diff --git a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx index 3a1e0065530..b722ff5f5ac 100644 --- a/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AutoRouterClassifierTabs.tsx @@ -150,7 +150,7 @@ const AutoRouterClassifierTabs: React.FC = ({ val if (next === "jev") changeType("jev"); }; const changeProvider = (provider: unknown) => { - if (provider !== "jev" && provider !== "laya") return; + if (provider !== "jev" && provider !== "laya" && provider !== "bespoke") return; const defaults = defaultJevClassifierConfig(provider); onChange({ ...value, @@ -173,7 +173,7 @@ const AutoRouterClassifierTabs: React.FC = ({ val {[ { value: "heuristics", label: "Heuristics", description: "Classify locally, with no API call" }, { value: "llm", label: "LLM", description: "Use a judge model to choose a solver" }, - { value: "jev", label: "OSS Classifier", description: "Use Jev or Laya to choose a tier" }, + { value: "jev", label: "OSS Classifier", description: "Use Jev, Laya, or Bespoke Nimble to choose a tier" }, ].map((option) => ( + )} diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx index e175fc934b9..1beacef7cf4 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.integration.test.tsx @@ -97,9 +97,13 @@ function Form() { describe("JEV classifier editor", () => { afterEach(() => vi.mocked(useAuthorized).mockReset()); - it.each(["jev", "laya"] as const)( + it.each([ + ["jev", "Jev", "jev-test"], + ["laya", "Laya", "multilingual"], + ["bespoke", "Bespoke Nimble", "bespokelabs/Bespoke-Nimble-9B"], + ] as const)( "preserves %s, custom tiers and context through save, reload and probe", - async (provider) => { + async (provider, label, model) => { renderWithProviders(
); expect(screen.getByLabelText("Judge model")).toBeInTheDocument(); expect(screen.getByText("Reasoning Effort")).toBeInTheDocument(); @@ -117,12 +121,13 @@ describe("JEV classifier editor", () => { expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("english"); fireEvent.click(screen.getByRole("radio", { name: "Jev" })); expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-latest"); - if (provider === "laya") { - fireEvent.click(screen.getByRole("radio", { name: "Laya" })); + fireEvent.click(screen.getByRole("radio", { name: label })); + if (provider === "bespoke") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("nimble-latest"); + if (provider !== "jev") { await userEvent.click(screen.getByLabelText("Classifier Model")); - await userEvent.click(screen.getByRole("option", { name: "multilingual" })); + await userEvent.click(screen.getByRole("option", { name: model })); } else { - fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: "jev-test" } }); + fireEvent.change(screen.getByLabelText("Classifier Model"), { target: { value: model } }); } fireEvent.change(screen.getByLabelText("Classifier Timeout (ms)"), { target: { value: "4200" } }); fireEvent.change(screen.getByLabelText("Context Window Size"), { target: { value: "6" } }); @@ -131,9 +136,9 @@ describe("JEV classifier editor", () => { fireEvent.click(screen.getByRole("button", { name: "Customize tiers" })); fireEvent.click(screen.getByRole("button", { name: "Save and reload" })); expect(screen.getByRole("radio", { name: /^OSS Classifier$/ })).toBeChecked(); - expect(screen.getByRole("radio", { name: provider === "laya" ? "Laya" : "Jev" })).toBeChecked(); - if (provider === "laya") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent("multilingual"); - else expect(screen.getByLabelText("Classifier Model")).toHaveValue("jev-test"); + expect(screen.getByRole("radio", { name: label })).toBeChecked(); + if (provider !== "jev") expect(screen.getByLabelText("Classifier Model")).toHaveTextContent(model); + else expect(screen.getByLabelText("Classifier Model")).toHaveValue(model); expect(screen.getByLabelText("Classifier Timeout (ms)")).toHaveValue(4200); expect(screen.getByLabelText("Context Window Size")).toHaveValue("6"); expect(screen.getByRole("switch", { name: "Classifier circuit breaker" })).not.toBeChecked(); @@ -145,7 +150,7 @@ describe("JEV classifier editor", () => { classifier_type: "oss_classifier", opensource_classifier_config: { provider, - model: provider === "laya" ? "multilingual" : "jev-test", + model, timeout_ms: 4200, circuit_breaker_enabled: false, circuit_breaker_cooldown_seconds: 50, diff --git a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx index 22e8708acc7..3818e3ae9c2 100644 --- a/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx +++ b/ui/litellm-dashboard/src/components/add_model/JevClassifierConfig.tsx @@ -7,7 +7,14 @@ import { Textarea } from "@/components/ui/textarea"; import ClassifierCircuitBreakerConfig from "./ClassifierCircuitBreakerConfig"; import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { defaultJevClassifierConfig, LAYA_MODELS } from "./jev_classifier_config"; +import { defaultJevClassifierConfig, OSS_CLASSIFIER_MODELS } from "./jev_classifier_config"; + +const providerDescriptions = { + jev: "Uses TypeSafe System One Choice evaluation with your configured tiers", + laya: "Uses Laya with your configured tiers. Set LAYA_API_BASE on the gateway to connect your Laya server.", + bespoke: + "Uses Bespoke Nimble with your configured tiers. Set BESPOKE_API_BASE on the gateway to connect your Nimble server.", +}; export default function JevClassifierConfig({ value, @@ -18,26 +25,22 @@ export default function JevClassifierConfig({ }) { const id = useId(); const config = value.jev_classifier_config ?? defaultJevClassifierConfig(); - const isLaya = config.provider === "laya"; + const models = config.provider && config.provider !== "jev" ? OSS_CLASSIFIER_MODELS[config.provider] : undefined; const update = (patch: Partial) => onChange({ ...value, jev_classifier_config: { ...config, ...patch } }); return (
-

- {isLaya - ? "Uses Laya with your configured tiers. Set LAYA_API_BASE on the gateway to connect your Laya server." - : "Uses TypeSafe System One Choice evaluation with your configured tiers"} -

+

{providerDescriptions[config.provider ?? "jev"]}

- {isLaya ? ( + {models ? (