From f498176a276b1f16b9c8ae6a265efe8c56ffd188 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Fri, 2 Oct 2026 17:27:13 -0700 Subject: [PATCH] feat(lens): add sample previews and improve setup and worker feedback (#44268) * feat(lens): add interactive traces and investigations demo * fix(lens): limit sample previews to setup screens * fix(lens): simplify tracing setup and align sample previews * fix(ui): unify Lens and ROI demo notices * fix(lens): distinguish preparation from zero selected runs * fix(lens): report incompatible workers and address setup review --- deploy/lens/README.md | 6 +- deploy/lens/compose.yaml | 2 +- litellm/proxy/lens/worker.py | 31 +- tests/unit/proxy/lens/test_worker.py | 40 ++ .../public/assets/logos/google-adk.png | Bin 0 -> 11976 bytes .../public/assets/logos/hermes.png | Bin 0 -> 1100 bytes .../public/assets/logos/openclaw.png | Bin 0 -> 1015 bytes .../public/assets/logos/strands.svg | 4 + .../hooks/models/useModelCostMap.ts | 3 +- .../lens/_components/LensOverview.tsx | 58 -- .../lens/_components/LensProgress.tsx | 2 +- .../_components/LensView.integration.test.tsx | 35 +- .../(dashboard)/lens/_components/LensView.tsx | 37 +- .../lens/_components/LensWelcome.tsx | 13 +- .../LensWorkspace.integration.test.tsx | 137 +++++ .../lens/_components/LensWorkspace.tsx | 117 ++++ .../lens/_components/WorkerSetup.tsx | 2 +- .../lens/_components/lensData.test.ts | 22 + .../(dashboard)/lens/_components/lensData.ts | 9 + .../lens/_components/lensDemoData.test.ts | 55 ++ .../lens/_components/lensDemoData.ts | 483 +++++++++++++++++ .../src/app/(dashboard)/lens/page.tsx | 54 +- .../ROICalculatorView.integration.test.tsx | 26 + .../_components/ROICalculatorView.tsx | 25 +- .../src/components/lens/LensDemoContext.tsx | 12 + .../src/components/lens/LensPreviewButton.tsx | 18 + .../src/components/shared/DemoNotice.tsx | 16 + .../src/components/shared/PageHeader.tsx | 22 +- .../view_logs/TraceView/AgentTracesPage.tsx | 9 +- .../AgentTracesSection.integration.test.tsx | 509 ++++++++++++++++++ .../TraceView/AgentTracesSection.test.tsx | 502 +---------------- .../TraceView/AgentTracesSection.tsx | 36 +- .../view_logs/TraceView/DetailContent.tsx | 17 +- .../view_logs/TraceView/DetailPane.tsx | 17 +- .../view_logs/TraceView/TimeRangeControls.tsx | 28 +- .../view_logs/TraceView/TraceDrawer.tsx | 17 +- .../view_logs/TraceView/TracePreview.tsx | 70 --- ... => TracingSetupCard.integration.test.tsx} | 111 ++-- .../view_logs/TraceView/TracingSetupCard.tsx | 498 +++++++---------- .../view_logs/TraceView/spanProvider.ts | 4 +- .../view_logs/TraceView/tracingSetupGuides.ts | 501 +++++++++++++++++ .../view_logs/TraceView/useAgentTraces.ts | 12 +- 42 files changed, 2427 insertions(+), 1133 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/google-adk.png create mode 100644 ui/litellm-dashboard/public/assets/logos/hermes.png create mode 100644 ui/litellm-dashboard/public/assets/logos/openclaw.png create mode 100644 ui/litellm-dashboard/public/assets/logos/strands.svg create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts create mode 100644 ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/DemoNotice.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx rename ui/litellm-dashboard/src/components/view_logs/TraceView/{TracingSetupCard.test.tsx => TracingSetupCard.integration.test.tsx} (63%) create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/tracingSetupGuides.ts diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 315d072b501..4b65c88afcd 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -19,11 +19,13 @@ The URL, database, and retention settings can also come from `CLICKHOUSE_URL`, ` Retention changes require a proxy restart. ClickHouse removes expired rows during background merges, not immediately at startup. Enable request/response logging to analyze LLM requests. Lens can only inspect content you actually retain -In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in +In **Lens > Investigations**, click **Connect worker**, choose an analysis model and monthly limit, then **Get install command**. Use **Advanced options** to select an existing virtual key or change the proxy URL if the server running Docker needs a different network address. Copy the command and run it on your server. The dashboard shows **Worker connected** when the container checks in The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis -The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds +The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. CI also publishes immutable `:sha-` tags for successful worker builds on `main`. Keep the worker image compatible with your gateway version + +After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected: diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml index 4d1224fd41e..fc9850fcb04 100644 --- a/deploy/lens/compose.yaml +++ b/deploy/lens/compose.yaml @@ -1,6 +1,6 @@ services: lens-worker: - image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:67eba741c1b97c749975c5c38e2370a603e1105babc908d613c1b79d7b995393} + image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b} environment: LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} diff --git a/litellm/proxy/lens/worker.py b/litellm/proxy/lens/worker.py index 62f8295e7d3..051b4a09392 100644 --- a/litellm/proxy/lens/worker.py +++ b/litellm/proxy/lens/worker.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import Final import httpx +from pydantic import BaseModel, ConfigDict, ValidationError from .analysis import analyze_sample from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult, Progress, Result, Sample @@ -15,6 +16,17 @@ from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult logger: Final = logging.getLogger("litellm.lens.worker") +class ClaimedJobIdentity(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + id: str + + +class ClaimIdentity(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + lens_id: str + job: ClaimedJobIdentity + + def failure_message(error: Exception) -> str: if isinstance(error, (OSError, sqlite3.Error)): return "Worker temporary storage failed. Increase its capacity or reduce analysis parallelism." @@ -74,9 +86,24 @@ class LensWorker: async def run_once(self) -> bool: response: Final = await self.client.post("/lens/worker/claim", params=MappingProxyType({"protocol_version": 2})) response.raise_for_status() - if response.json() is None: + payload: Final = response.json() + if payload is None: return False - claim: Final = Claim.model_validate(response.json()) + try: + claim: Final = Claim.model_validate(payload) + except ValidationError: + identity: Final = ClaimIdentity.model_validate(payload) + failure: Final = await self.client.post( + f"/lens/worker/{identity.lens_id}/{identity.job.id}/result", + json=Result( + coverage=Coverage(), + error="The worker could not read this investigation. Update the worker to match the gateway, then retry.", + ).model_dump(), + ) + if failure.status_code != 409: + failure.raise_for_status() + logger.warning("Worker could not read a claimed investigation; reported a version compatibility failure") + return True prefix: Final = f"/lens/worker/{claim.lens_id}/{claim.job.id}" async def model(body: ModelRequest) -> ModelResult: diff --git a/tests/unit/proxy/lens/test_worker.py b/tests/unit/proxy/lens/test_worker.py index dce3fc04d45..0e64b0f10c8 100644 --- a/tests/unit/proxy/lens/test_worker.py +++ b/tests/unit/proxy/lens/test_worker.py @@ -3,6 +3,7 @@ from typing import Final import httpx import pytest +from pydantic import ValidationError from litellm.proxy.lens.models import ( Claim, @@ -81,6 +82,45 @@ async def test_idle_worker_does_not_start_an_analysis() -> None: assert await LensWorker(client).run_once() is False +@pytest.mark.asyncio +@pytest.mark.parametrize("result_status", (200, 409)) +async def test_incompatible_claim_reports_failure_instead_of_leaving_the_investigation_running( + result_status: int, +) -> None: + claim: Final = Claim(lens_id="lens", job=queue_job(lens(), NOW, "job").jobs[0], findings=()) + payload: Final = claim.model_dump(mode="json") | { + "job": claim.job.model_dump(mode="json") | { + "settings": claim.job.settings.model_dump() | {"future_setting": "private content"}, + }, + } + saved: Final = SimpleQueue[Result]() + + def handle(request: httpx.Request) -> httpx.Response: + if request.url.path == "/lens/worker/claim": + return httpx.Response(200, json=payload) + assert request.url.path == "/lens/worker/lens/job/result" + saved.put(Result.model_validate_json(request.content)) + return httpx.Response(result_status, json=True) + + async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client: + assert await LensWorker(client).run_once() is True + assert saved.get_nowait().error == ( + "The worker could not read this investigation. Update the worker to match the gateway, then retry." + ) + assert saved.empty() + + +@pytest.mark.asyncio +async def test_claim_without_an_identity_does_not_report_failure_for_another_investigation() -> None: + def handle(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/lens/worker/claim" + return httpx.Response(200, json={"job": {"settings": {"future_setting": True}}}) + + async with httpx.AsyncClient(base_url="https://proxy.test", transport=httpx.MockTransport(handle)) as client: + with pytest.raises(ValidationError): + await LensWorker(client).run_once() + + @pytest.mark.asyncio @pytest.mark.parametrize("model_status", (200, 402, 503)) async def test_worker_reads_claimed_activity_and_reports_analysis_or_failure(model_status: int) -> None: diff --git a/ui/litellm-dashboard/public/assets/logos/google-adk.png b/ui/litellm-dashboard/public/assets/logos/google-adk.png new file mode 100644 index 0000000000000000000000000000000000000000..9f967caa3008bdb13b446e1545cb16eef4e29149 GIT binary patch literal 11976 zcmdUV`9IWO^#7e1%h<-)q7aE}p^_!*5ZRJRcFI;LYh_K$*g`7VQKYs#;_MksZKKj=80bt?!`wIgvGx(rF*nLwyZBW)Lun7G_=c;9_1wchS%Yh?3 z0J7H&bhOL@VJnlyPck|}=GSrRCfnE}2Cnb!@|AK}8UIM`y61Ay#cwR?fxgPR@kQI( zBChC;4EVrSsP7S`@`*Bi#Ak zrR2HA`8Q??u5L{|(hXU~53*Jn39dR5oWmw2qs`Q?J2TH-l})ro32|1 z2{pQd^C?FcdP+Pl!<`xTzGCmjyM;BUKdwu4Q&wGVX>om_miWp6 z9=tSE$viQqJFj)Wl(50bVYE!S`Q?hMm*m4`>tS6Fict5FBl>1TacEM$h8dHfHv zjY82p`Gxx@KVD8-iLlZWAss_C*Pjn^zq$B``$2tKMDs^-{DPapaylML4C9pH2WWwYl zqyqRHMw%71@kMBV7V>cw@Vu>j`a*9B&a-o4l&&bozpOH-5=LAALt|f})Lk#fKOZ5h zC?k>PdC3}SPq|&kC9(G0KtbVMwch-u%bGXS$gKd_SR8(g4PRU#&Qwr;p(PQcUO|Vw z_*MqTKh$M-L7E|Hc9bK zL6@-7r(r{p*H@d>D}7@L)07ZuiqPi`p%2oSTaxL&;(v&LrXNDkW7vpOngWUlhdR0= zJ=08B^B?e~2r>NQ{XO?MXI5&PV-+x=?FA>^wC5lcd&rW!cpm;Pb(Yb}zY5G=e5hCA z?ICOG7|^6Q{&<_1(h&wN+QM+Vx!%UiQgLuXj>llL#E_a}^Ndl`&aq4|)KWBfZ?Qb| zkBn?vQIQ$;0!0mXottJv?sjk|uh3kI!!Bc6IGkuwI3Z$G4DGeqI9XeNd4!?D&&{Pu?w`xr@h8M9OboS<#NK)bE=hG7z) z-z<>9^W4Q=@L{q}Ookn_3$?qKa*vIcYr>?q_q~m4D@6s3 znA6X%4$rSBs=QGtJyq6x;ID_4VaCU>UD>Hh+cWchn{7Hc@V?1L^iJ94;q~hwDU*UQsU?!FTy4sNWNF^H=wD+Z!V?Mc z_rtIcA(_5ATN7#37Cg#1_ixo^uc~ys)L6RN&-2{(8#i1|U%{Wpw@j!(`9useJ?3U{ z#b|bz5YgjAf7+*H)4jmuF2vmQ%zrg$p$}}(Y|ygWoBuWl%ox_jBx{`Ha1M5`gNEP; zp0($BaH=oBSbh!R-r#*(c;pF5`=U|@6aA+!xG?-|G!?lGv#qPV@1XJN7EC*AovcDe zzG$Icei+ewd)xiO;S_FKp@q!u(X%htw^b(C@UD-wuy6%0#ZjSabpJx^1Ib+31wi+^ z-#zZyTvt$ROy<_B@h&FHI>JPDR&rCG zQqUfuh9O(QkyZtpn;bRNVBJAJ*EPof-i0o&+-J*NjdL?_s4lN8s!#K~2tyaOq-x4| z#^Q~4c*lG1+EDSfiu?`;f0xXVOSAnNYi-_HoOH2bc%WPJ)1>RR%OPeL)wnh5D>~0W zWeZ`&+H(=VqYW4c)}Wy4*lF#RYUdY2H86l-@r=61bQf+8Ge5q zwO~l_xI4Jhs3m+x*cZU>D($4I|x%AqddUr^=ATj80<9yjVHO%N_G(mOd4E-;r6Yx>EFo49H_V@7U$@+#wb$# zk_kx}A}ROE6{)F!EWyyjk-U*kF{WoXhM98I-5It`((>Au{xjZHzWf`BUzp6i_4X@g7A36i zh6hXIYp^D1e8tdV`J;yAjd67uAk{V{%J+j_+ZWcD1J-_g=C7N8O;JX>Hm`q$6YgyY z?)u_iW9q?iNOlg3pB-FY|DbAAs7N&U66wqBFvr6uU+LE#!AXdQ)#>W*1)`!hSGe{N z#hz$pZa<6G!O(xkb5R1QA=ieQt;@FPms;jv-D%J0^5hOBwV(>ak*uZ*#pYOd8iknM z??MLz?kpTcMpknZ{ul(uhY`%Uftb*Q;FujA3Lnky4x7Ch!>WC)Ro03c-p^$f2 z3@^g4InP^hbTnW^!3#5qp^JQd3Ue)v(qjykJ8>LTX^2{>oLLT%a4|O?xGj7eEITrSWG7)(iv$nU5ZS(H`e{ z_jKe6E@+VaNfW#pc}^|aQ8Gl4WSBwp257|@<{9mD7{03JW0Ngc;LG%KEKZ${T*EssD0!Io*OepD|@ zOW+V!7BVk>ee>uDSNyW;!dQOi)i5q@@Qk&Wwq1S_5JRQ{&#&+>l@SMP{`V*@R;!2@8n$!S}(%VL&8J#ikMXo_GTML29KC5Q<}G% zn%(Vu`d*J7DNIiMZW|&?sX6bSAa4m3TcFvpwtWtMvNvxf1R2}R_Nk7>U;lv>W?TU8 zaz48Ie0+#MyKnEf$>(A)*QlP{$;x0`%Q>g!&5$fV_sg2$5plgST6*A~xipR*FR7u- zImW>Z^cq(IcH{NXe?O{^e^99OEs*$;%&OOQpo06c_3^MFwW{j=*WkwNv}c;+poZ5e z=aQI0?9>KA8`x7WA^XExI3O?@1ZD)|mle9}UKwr9eRZ~qXi<;a4W7EA=skveB#cz^ z4lv76nLVlpZ_t8~@mH$1L*6!QgnF`oeXS2An?W7dUyR&K=(|R@)3NC*DUi>3J5*QoeoUA<8pf-(dQYOy_Hj*MwwXC6J)S=_0Jse&9jlyZvMYgk zx0i7U1u4WN3AsnjUq5cn-OL*gfe+?K&)_7OJYxI6MJ80tFRy)~yF@>Ew9WKR?ELj( zGFusDW`gv1t}mMp%nw7YEL|dG)(aJO{yIP5XHK!i0-0;lA4HhM3f1EQDL3 zMVVon2Nn#To}P%i0Z*X&lOnSxbLrm;s{%Kz|^e<9&#|@M)9bQ3M>AqX(nU zAi{&yyd97dP?TGAf*F`;=`-CL9Q?iB7qgeMbUzTY^!URAKR6s!56 zB^#Dc4oZ_w$P>G<{HUCkfT{arm%WMc~Cw@ciO}|H-FSIgf3T2kJB`NxB z*F$dw=rLb^U~7?%J5@#7Sa0mgO$p32d(mg+4^xH=WaRHyJ;SA7-P;MrQSF`Q`g?<^$M{*9yT;$eDL6GSWVosxJwUW>`s>GT!^sdH269=8bYl@nnA`SBUyhJEqY0y&IjSCnxzQR%?f~Wd^$II;*3zCY?v)ZC(SP z_Je-knxfRe+==k8ly*wVQNDui=uU$VA2CVle!a)lE)Hs<)K+KCWji}ZJPo8$cdKcF zSK;EQ5B~cZK|v;u_3V2q)$a$m>b{HrAaizLH!nz|PrB6h0kOlnG&18)_)*xA%Z&La z4RLOL4+@8clpr%XZVUFM4)xV4#bv1*V;`@Dzs^OFpZwk{EB8`(HFwd(g`VAN7q3~j z!NSwAIn{L9pd-1D+DxSH7t_W+%Y)fj-$^~u7##rGr3W^`cM3YeJ8c+QKlPUSdUr|Y zE^US$T~!52Hp)i#sI)MP+9bxUfiY4t|zA7%I$41E^1KE^jrvAzp$fPoV6$SRsl1C92rBJMl9Z~-K1{Q~y@#S72Cdk0uu zD;pG6iHbr_&>-iJ4bOXy8L1rVgidZQ9I#hZ;i@P??JbUm(jzpZzx98So<5j>J-u2F z!12OtQU8_ceYzGe+V=z6Ur=$${<&~JJy;V~-B+smEy;9I?;li&tYXFR0#d~CvSwr{ zE_Foed8g@C1q(qZ`i5SuNtB%HxQ0I+K#J{pvt6O7`rpkaaQgzod{|}bhdsuO=S;tae@2(nfA8^ESl9zw1FDIEu<4|lv;-SHCU)5iJX_&D#+&?-<>ox?_ySYq5c*j+^?GwEzK~&ZqD1D&>Bjt zdR@ex$isdQ-kalxxHBgFH)1JHQuM{_^dnkjAv2053E{R*m53i9*p%OHP%r`~D0-~P z6^%FVx-B2ibhb2wK%kmL;-?b2$b3N1u&|+B<3zXOLVZwp@o0Pdj59$0GQU<|?L^J#L|e~^XxBmf+JhnM7bl@6?13m&ww98M1R zv(+Bb<5#%TjMAt zDt%N09?rcqgJHMUXAVhj3kNt0)NaN8K}7Go&y2|nC`^_3fgy)niqq*4df*(}fB@{v zHBHju2T{j+uM6++o>Bo}Dt|POn4}kO7U&3Hz2`N0&)>iIv?CG}dJInASmv4?r^Xjv zvDc(1y3=4eP+%)+1bh47q=CwMRf`t`kkzT0!o@GWXu)ZtMN$wIgQB-w1p3{@a+xdXa`R#$ifdml4_5gi@Zzqnhc4v0Y?xs0M_kPCr(C^STCh%>7c>SS2F`~Q@ba+jxqdO*LtOp$S^&eB4UtodGT>sm;zI3fW zdF`dHR6!R>I&8r3(Vbg91AS10nsr16+GOp7Om-sb13{;!;yk|o%@E03GQEL1_%S!U z2ofN%fh9YSKfxRRiRzmE?gcZm?4D_=LGQ)!(qr-%USl%*EZ-ol28_Bra$XM}t%nzd z&rhk0EQxn@NXt(&#|s`@o6HdY!z&K)9yRv10!BJc?^tizcE756n_{QWKe)R)y|ToN zj*ZZyQeKSgvbgkn8%4S7v!7TOy1NS=g z5uV8qTu!TWG2*Q+)mfG6_8xX5H4AkdSU`-SZo6*?QT2hA(jA&AanY42|>K=&TL+L0u!ASy+e|oC{cfx-8y=asqE75?J zngYeCTBjMot17jm05%#ydWamiPYr zCCgxTJgG>j9Tm_`zE7jKI*|@5@4-cph+Ct z4hSqsn5I1)`W$Y&C%-ewE{K0AB7iLE$>M&uvm79ea}pli9pQ&c>(f{y{yYJJcs96V zDz(Zi-E+5U%HG=-_-ywJi@7c^0NnExs_Xlz?d-g8k1pEz=zTm2{P?mR(DSV{9Kx@F zHLt7BaI5GFM4cTya2$Y<5Ej2Vhz)eg>gXDmQJVMazU`YB`pvtz)XjstntvJbl;&x_ zGX253rEx6qAuB%=UA^ir*R_te1>j(O6M+q=qaM6#|8nY}r*dkM{YL#~m_y)c6=3yM znBOO$kYo+Jb81!wPvOk2Qk>}{$8Yt+NCji}5oTG&(kbn#V(?nR*m}ij>bNu~@5FEK z3qtpz+sMinu`XN~+^-`DfnYC*Y#3kOT32AQY46WKB5ljHmG z1<0MK$fsdK6M8VbxDyM^ASub04y)kL;_&VQUO zfp-F76%jP(aJK3F!nHg^+xOe_oL$I;23h-b1feV9l8#aDI|qAM89~c*brGFjuSjlb zoDPn?y(+)ZCe94#T4cFIb*GNx-#8-_@9fKnO-*#6`lo6Z>V}N%9l%&Y_{%EG z``H`qz7D>%V9tOzyoa4JvVjhes&uf$)Sq*AQ-1OL@gK-|)(aly1YH7PY5*oN;4tUS zNPFp?_cY`*J;2*T_|s|hfYB4DueoZe@&^gdLN*2diqiKupWlz^z_V5RMq`mvcN1exQOiaW=LbT{Qu@!LvR%KL-JhBFgXdJ#xS|7(Bbn$iOqJWm zye&fBPFqOkUpbF*t%8etg?@i;G16!966SKp4?mta?jo<=Fz>p$O?RJ8W6ivdskHT1fP^@+w!!HO73tHPljRV`%h)d%USPb<@?1+o0!i~2iwr{U5-*( z$JNQX1P{A2=Oe6LQYSJDj2`8>x@2>8Yt%*0hxK#7eLu5Oud!0yXZ(&9n8d4%{&o#% zXv-68ljkF8?C!OkZAn$;J1A|L{;J^)dxVFRPRQ?MbC=}%&kMiXps@?7L5Z^Fk2ODg z?iid%TgIOncD9?LA+>4E&`c;xM zx92Oe&swvdw7M||UR)Q7)=ZIAv@H&o2XI9@O6N?(rpYnciEs;C!ZiVxG{V=FcpYNzxOp1C1-vseA<9G?*<2B8n3_)Iye(76=P}}9&tE!Ro%~jPp?A%W z$OiEQtj`X_4(ep{Bj1MAwOk$Z{@U$uk=KFkxIKSs*w_(fz`NH?RdP}hwWPP5RbH;M zPPz#@?pLKAXk5@P1hqH^Mcv6xl;#AN#2zNa(mdj{%M%x)9)lJ{rRbBe!U8dVR{`OC!Pd0<@x}YnvkY*i z{nBVOIafQx7|G~l%nc->1P_dV;k1cm_}0)ObZb~(#v|}-yro(Hhg-x5?WM<9U&*%p z;9UV>UHs-%iUL!;d?hE!L+RBblNWcSE+e>7+IygP>Kuw{eUa$cTqCi_beu371WBJg z&J5n2PdcUA!E{(Evp}4-IbW!(QPRJ3F&3IQE0kV-mwejos{GUHiarMFP&<5ORzU|% z30CWoe8zq)_P4$rk`c-`o9MHRXfzlOl~_Bg4e0x?@Lx#GbQ)=HdeIizBX|Gt_qLbg zWQEwWF);xvZ^JXb<3ArAiSO=yHGiyVn3&xjZ2Mz*YP&2Gtu~v~#^|ZV`%8f=*c!HV zkr7B3_&PRse7=9>u3cP^r_)rEfim%t1i*9Z9Hj*7pE$QEq}q)hy|-N>=^}a~65;*` z4t!33BTggCRitK=Mp2}tCWg7 z{9UiQ>AE(p@`|xYTO5OAw1A!poY65y?z2fOp8jbP&$Tzv6s(I6k`eQYL`Xljs6aru zGi~0B{kT;{K2&-$Z_g zeLDL_lSXOEvrw|AfR#))w;*Jygv^=tC4--o4(ict~9opJ255QEw%*o~G7aXYeGyO2k3e(ST){$nylG*SfyM+el z$@oAQDkU%ZfmTjiXQb`4wb35PnT^mk0=v-Cmw}rBwt)gvfjEVa^LV;qL zQN6#HyF#qu(cy9b?LDV=&|4kDiHY|kGSwO~cTWn$7{^{%4$v$ewl4eR9}n&7c6Z9J z`Pus2HO7yU$i$O}VsSr97(55>u9d<>TI9JR`snU!)R1!bkg0O|V1IKi9kR@vfsie(xulFf`^I!EzGYhTpP`6hpTR^vfeZv>D3H)6 zbnWe41L(@5blf0=Aof3(OS3g>4Z7cUxd@=fFoW6A8DqcYP_O_5@k@~FJI6t}m<}1d zU(k0Ybl1lrL&JtfMY=~Gk`ez5MUiZ47@uNm=vovC5a}ZkXD>rlT>lJ>na~VIjj1&h z34@FnKcvzI$i#4S1Ck=N*%pvw{r|BPhmeoH1XZ2)1^5%tKwHp2F)t)lFpPvI$Onc- z9-_~w01_JNB?U=?j`x|H<}8e%8(R_1;W9^AsdF`4P&wbf`H+eqM^iy3&MmuJyM@Ne+=^OM5o0 zpUF0O32U^BdcN7{!{>IMr&Q-5UmnrM06kCQhH*M_oOvmuAl@@--J))NF3>nf9=TsJ zWp8^LM(%X0=v+Hnx|KY*_fXTmaU543bfZsmn6pIsn~y_+nq1)X#G)IJa)?ko=v< zFn<_dx&6|6Dai5Hm`aY?1uTyM>z^|`*EX_~4t;p(!G8mLW;f1z_p@2q`_5nlcNhI- z-C}bkM3$rXv4AA`mCfy^u152LSCaL2m1V|_sjkseA{Wlbl=bF@6Kj(*q9W zZw#ev@_sGMhm43QNq+Vrq`nPjEYb-OMMDl1dQFFNWAac$XMMj3t38$H}b;I1^*W{-8df4{vZwj8|zWg#4Hic|>#7lU8@oSmJwVH-TCYWJ?lFC+ec zkxw3UBi87kJmLBX>PdtbQ*G&i4`FGtGvEr#$Z+@OVr9KN)Bt+FWH+FVy%Q1%q6ry3 zgk!z}d~S*^Rp8d?0Z0*Npp@(2!GmV!CSN+AZcpX@1NHkM$wxNlCT(->uBrV+2#|n5 zZHGIa-KU~I8!NB8^3JsvWXN>9XK4NaHd zEq9KMmZi(Zv!CSqyoqNqB%4t-FHJOl*$iit$amq7uHu4=|2zG8yVbie&8Bf$yrVti>={CfT>Ts#Zvp{84nBcklqSqq4(x(Q|a5% zV*4qnsW+G$Pz-#*u}`nmCvsb+Rc79&JN$1pNLUnl2tw?ba?#ymabG*DnLU| zD`CV*JV%vc6DPW4@cl3O+m-h-TA=n98HF*5qHcy&*X9=0gsxj+I>SenwB?Qa&aNRa z?;w&~NrJ)jc3f&I=LdV~NmH8|QEnHK60SCjb6CdgE)>*fK|#I1#{Zo`_&-w+|5pK6 aa~f)e-P@me1|joTmVxdSoic5Q@c#!yYc4(j literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/public/assets/logos/hermes.png b/ui/litellm-dashboard/public/assets/logos/hermes.png new file mode 100644 index 0000000000000000000000000000000000000000..de47b728d12a648a0d3affd06e7d3ed75eacd562 GIT binary patch literal 1100 zcmV-S1he~zP)q7y%f8W&ke&%s_tx@FGChvw#-?7=i6_R^alj?rfw76x79^ zx##o|X}Y_ps(Y&d?m#jmnUXy7&*6OA)5jD@3X%o?dlV$a*x?obKUAF&&dq=Vz_fdM zXxkQT+oG;(e0_bPZCfa%pp=5v8lLCjO_Yw~EW4P$_FGXfj$_7g%-Cx>x!dgqb5cq+P4ki+n40OkR!aGBxYn9gRWVId=6Oz%lu}YkNze0G6a~{XeQCt6 zvFtF$004p@u!-mUK0MDuS(Z>rVY}U8zu)6ola=m7MIHfQcBcyZRe)&F(ot|T-UYNv)K$%%F&HZr_&(*LWqHJQ+P$LB91ix`n-_L=fC~Um7)|O&T(=Z4EDy6vHZYSy5 zr)>Yas;c*L!1sNmX^PEe17i%bEVE3=vJ52Qx~`3HAG6sE^Z9)AMaP~W^*|q5YerG@ z;Gw?OYpo}l(f2^7nD=;UKoA5k3rk0}u_1)8)b%{?y%0>5;3-VH_HLaq#wzSn8uoEK z9$}0@UDu;VJUe`hRbzCVG@!G#TQqi9|6W%&#=OvJNk`wON!V4dj*e5Y>r+rl`Ouhj zvF}T{5aM|bWLaii*1erE2K)W~sh8&SxvhL1uGi}z4;G7sZRI9IV2rUY?oS zC6>!2=JPpLtJPq0DdmF!V^ZBWdNc%46j2Casr8KGNs=%OLjqtJhW7f^NRs412ikrq zUa!~GT64GCQ7JXr4RtZj^ZaRvzSo2i1I(Z)ZJLH!Yx=%#7p&Lohs}Z@u-aZ#)ufSc zcp&=)AOo=OI9rw_4u=D>EJGNE`2G9$Xgrltmgm0j+xt>VD5Y#CWQ-Y9!!Qg7dEezh zIYlw8HG?1+ZONl3Vv;0mnubvnjke-xnp!Vvt@~5u{ij(y#rHm!`@BD&&#bD7d7eKc zS3j(2nsTvN^gHCb`!xHwH688k)9ib!)w-8q&+`z5Aw17RUDv4V`rfqzDej)XY0NhO z|90lTI^Z}?>o|@NUj|3GLC)Vz$bm0<)8omzPHZ9 z8x6UBeY5QB{F&G$0Twt9U@AxDsv5I5uwla|ROBz&(Q$~_Pxo;Up0>Bgo~>Hd`n0XB z>oM^`YisMh<;!Cr(02o#39@a4f7rT$o>B(kYD?F9t-88M#Rejr z?~qciC@nZ>a@;4<`$vj1W5NNprZjC$U;~u=FD0^41J{#jh=zKIcnmjE8Tg0gR@x}W zZ)HqRyin`^KnCg=`&{I^-qVVoVl7zuXcV9AIfftKZH7Po-8=KJ_`wL4>^_Q(%}s9H z&d!|}$(KGGR_|Jt>i03=K1 zm*D~KI9p810=I2siL~ipUN}4}6MiELqftbuma5B%X8yVlz@HieV)WuS5-$b~;tRu2 z$yLx@DUEe?#oDH8;`dA{HD5~tVR~S`AQ)qwVzOKySR6otbB_UPtRhR3wifT}dqlyz zrqAL0NF;zru=r>?4I`HW&kIjz40H?(JkbD13Tz_^6;xOHp&%RzspOiFI}XN7Q;Ca0 zBZcWW+myD3JWd-zd_{rHAGEZbh`N)nX1$<&j;CxEpoqCmj3~(8wy|#fLss*(4x_ulyuiu zq#6^0H2yu6{Eo9Wu$dCANlgHim{2|X3jki}k?e{CbkDHM1!1?UHQO`cULnLEQc5wU z31F&Tki>sfjj9f%g#CiCZ>763%2{q=)B!pH^i+2$cF~{BLN^6TRUG z(QWJchX5{Ado1@OP25$|>Z<@26PKv_i3zE`DSV3>8gL;?cE|!eyQL4cH$Y;WNNAHS lL~(o_$LW!U_jrp6@fsk=FmPoZ + + + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts index 2d82eedf25c..d9824b4753e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts @@ -4,8 +4,9 @@ import { createQueryKeys } from "../common/queryKeysFactory"; const modelCostMapKeys = createQueryKeys("modelCostMap"); -export const useModelCostMap = () => { +export const useModelCostMap = (enabled = true) => { return useQuery>({ + enabled, queryKey: modelCostMapKeys.list({}), queryFn: async () => await modelCostMap(), staleTime: 60 * 1000, // 1 minute diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx index 2e2a0318d85..c6cd8485cbe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx @@ -67,64 +67,6 @@ export function InvestigationList({ ); } -export function InvestigationExample({ onClose }: { onClose: () => void }) { - return ( - { - if (!open) onClose(); - }} - > - - - Example investigation - -
-
- - - 3 of 20 conversations -
-

- Failed lookups leave customers without answers -

- - The agent retries the same failed order lookup, then ends the conversation without an answer or a handoff. - -
-
-
-

After three failed lookups, the agent replies:

-
“I will check that for you.”
-
-
- - See the trace - -
    -
  1. - Customer · Where is my order? -
    2. -
  2. - Order lookup · Service unavailable -
    3. -
  3. - Two retries · Same error, no new information -
    4. -
  4. - Agent · I will check that for you. Conversation - ends. -
    5. -
-
-
- - - ); -} - export function MonitoringSetup({ settings, ready, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx index c2b9d227276..8249f284ada 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensProgress.tsx @@ -53,7 +53,7 @@ export function LensProgress({ job, onCancel }: { job: Job; onCancel?: () => voi >
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx index a3ec7877223..b3d7366bbab 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.integration.test.tsx @@ -231,39 +231,18 @@ it("runs saved settings immediately without opening setup", async () => { expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); }); -it("lets a new user inspect example evidence and return to setup without starting an investigation", async () => { +it("offers the interactive demo without starting an investigation", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); vi.mocked(apiClient.get).mockImplementation(async (path) => { if (path === "/lens") return { lenses: [], workers: [], tracing_enabled: false }; return { traces: false, requests: false }; }); + const onDemo = vi.fn(); const user = userEvent.setup(); - renderWithProviders(); - - await user.click(await screen.findByRole("button", { name: "View an example" })); - const example = within(screen.getByRole("dialog", { name: "Example investigation" })); - expect(example.getByRole("heading", { name: "Failed lookups leave customers without answers" })).toBeVisible(); - const evidence = example.getByText(/Where is my order\?/); - expect(evidence).not.toBeVisible(); - await user.click(example.getByText("See the trace")); - expect(evidence).toBeVisible(); - expect(example.getByText(/Service unavailable/)).toBeVisible(); - await user.click(example.getByText("See the trace")); - expect(evidence).not.toBeVisible(); - - await user.click(example.getByRole("button", { name: "Close" })); - await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); - expect(screen.getByRole("link", { name: "Set up traces" })).toBeVisible(); - expect(screen.getByRole("button", { name: "Connect worker" })).toBeDisabled(); - expect(screen.getByRole("button", { name: "New investigation" })).toBeDisabled(); - - await user.click(screen.getByRole("button", { name: "View an example" })); - expect(screen.getByRole("dialog", { name: "Example investigation" })).toBeVisible(); - expect(screen.getByText(/Where is my order\?/)).not.toBeVisible(); - await user.keyboard("{Escape}"); - await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); - expect(screen.getByRole("button", { name: "View an example" })).toBeVisible(); + renderWithProviders(); + await user.click(await screen.findByRole("button", { name: "Preview sample" })); + expect(onDemo).toHaveBeenCalledOnce(); expect(apiClient.post).not.toHaveBeenCalled(); }); @@ -276,13 +255,14 @@ it("guides a first-time administrator into worker connection and lens setup", as return { traces: true, requests: false, data: [] }; }); const user = userEvent.setup(); - renderWithProviders(); + renderWithProviders(); const guide = within(await screen.findByRole("region", { name: "Find what needs attention" })); expect(apiClient.get).toHaveBeenCalledWith("/lens/activity/available", { accessToken: "test" }); expect(await guide.findByRole("link", { name: "View traces" })).toHaveAttribute( "href", expect.stringMatching(/^\/ui\/lens\/?\?tab=traces$/), ); + expect(await guide.findByRole("button", { name: "Preview sample" })).toBeVisible(); await user.click(guide.getByRole("button", { name: "Connect worker" })); const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); expect(connection.getByRole("button", { name: "Get install command" })).toBeVisible(); @@ -305,6 +285,7 @@ it("guides a first-time administrator into worker connection and lens setup", as }); }); await waitFor(() => expect(guide.getByRole("button", { name: "New investigation" })).toBeEnabled()); + expect(guide.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); await user.click(guide.getByRole("button", { name: "New investigation" })); expect(await screen.findByRole("dialog", { name: "Which activity should we investigate?" })).toBeVisible(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx index 09a8607813d..cd78effe0f8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensView.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import type { components } from "@/lib/http/schema"; import { useEffect, useState } from "react"; @@ -28,7 +29,7 @@ import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs"; import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetDescription } from "@/components/ui/sheet"; import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; import { LensFinding } from "./LensFinding"; -import { apiClient } from "@/components/networking"; +import { apiClient as liveApiClient } from "@/components/networking"; import { TracePanel } from "./TracePanel"; import { LensSetup } from "./LensSetup"; import { LensRuns } from "./LensRuns"; @@ -37,13 +38,7 @@ import { WorkerSetup } from "./WorkerSetup"; import { useAnalysisKeyInfo } from "./AnalysisKeyDetails"; import { uiHref } from "@/utils/uiHref"; import { ApiError } from "@/lib/http/client"; -import { - InvestigationList, - InvestigationExample, - MonitoringSetup, - InvestigationSummary, - InvestigationFailure, -} from "./LensOverview"; +import { InvestigationList, MonitoringSetup, InvestigationSummary, InvestigationFailure } from "./LensOverview"; import { LensWelcome } from "./LensWelcome"; import { workerConnected, @@ -70,10 +65,19 @@ function emptyFindingTitle(active: boolean, scanned: boolean, status?: string) { return scanned ? "No matching findings" : "Ready for the first analysis"; } -export function LensView({ accessToken, readOnly = false }: { accessToken: string; readOnly?: boolean }) { +export function LensView({ + accessToken, + readOnly = false, + onDemo, +}: { + accessToken: string; + readOnly?: boolean; + onDemo?: () => void; +}) { + const demo = useLensDemo(); + const apiClient = demo?.client ?? liveApiClient; const client = useQueryClient(); const [workerSetup, setWorkerSetup] = useState(false); - const [showExample, setShowExample] = useState(false); const [monitoring, setMonitoring] = useState(false); const [now, setNow] = useState(Date.now); useEffect(() => { @@ -85,6 +89,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryKey: key, queryFn: () => apiClient.get("/lens", { accessToken }), refetchInterval: (current) => { + if (demo) return false; const running = current.state.data?.lenses.some((item) => item.jobs.some((job) => ["queued", "running"].includes(job.status)), ); @@ -100,7 +105,10 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryFn: () => apiClient.get<{ data: import("./lensData").AnalysisModelInfo[] }>("/model_group/info", { accessToken }), }); - const [selected, setSelected] = useQueryState("lens", parseAsString.withOptions({ history: "push" })); + const [liveSelected, setLiveSelected] = useQueryState("lens", parseAsString.withOptions({ history: "push" })); + const [demoSelected, setDemoSelected] = useState(null); + const selected = demo ? demoSelected : liveSelected; + const setSelected = demo ? setDemoSelected : setLiveSelected; const [editing, setEditing] = useState<"new" | "edit" | "duplicate" | null>(null); const [batchId, setBatchId] = useState("latest"); const [historyOffset, setHistoryOffset] = useState(0); @@ -146,7 +154,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryKey: ["lens-activity-available", accessToken], queryFn: () => apiClient.get<{ traces: boolean; requests: boolean }>("/lens/activity/available", { accessToken }), enabled: loaded, - refetchInterval: 5000, + refetchInterval: demo ? (false as const) : 5000, }; const activity = useQuery(activityOptions); const tracesReady = activity.data?.traces === true && !activity.error; @@ -157,7 +165,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin queryKey: ["lens-history", lens?.id, historyOffset, accessToken], enabled: !!lens, queryFn: () => apiClient.get(`/lens/${lens?.id}/runs`, { accessToken, query: { offset: historyOffset } }), - refetchInterval: 10000, + refetchInterval: demo ? (false as const) : 10000, }; const history = useQuery(historyQuery); const historical = useQuery({ @@ -297,7 +305,7 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin }} onConnect={() => setWorkerSetup(true)} onCreate={() => setEditing("new")} - onExample={() => setShowExample(true)} + onDemo={activity.isSuccess && !ready ? onDemo : undefined} /> )} {showReadiness && !ready && ( @@ -687,7 +695,6 @@ export function LensView({ accessToken, readOnly = false }: { accessToken: strin } /> )} - {showExample && setShowExample(false)} />} {monitoring && lens && ( void; onCreate: () => void; onRetry: () => void; - onExample: () => void; + onDemo?: () => void; }) { const activityReady = tracesReady || requestsReady; const workerReady = activityReady && connected; @@ -39,16 +40,14 @@ export function LensWelcome({ className: "col-start-2 w-fit sm:col-start-auto", }); return ( -
+
+ {onDemo && }

Find what needs attention

Check how your agents behave. Get findings you can trace back to what happened.

-
  1. diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx new file mode 100644 index 00000000000..8b7531c9bc2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.integration.test.tsx @@ -0,0 +1,137 @@ +import { fireEvent, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { LensWorkspace } from "./LensWorkspace"; +import { createLensDemoData } from "./lensDemoData"; + +const network = vi.fn(); +beforeEach(() => { + testQueryClient.clear(); + vi.stubGlobal("fetch", network); + network.mockReset(); + network.mockImplementation(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); + if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: false }); + return Response.json({ data: [], traces: false, requests: false }); + }); +}); + +describe("Lens interactive demo", () => { + it("opens without tracing, filters sample runs, and restores the live view without mixing data", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { + onUrlUpdate, + }); + expect(await screen.findByText("Enable tracing")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Preview sample" })); + expect(await screen.findByText("Where is order #1042?")).toBeVisible(); + expect(screen.getByText("You’re viewing demo data")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Set up tracing" })).not.toBeInTheDocument(); + network.mockClear(); + await user.click(screen.getByRole("button", { name: "Refresh", exact: true })); + expect(screen.getByText("Where is order #1042?")).toBeVisible(); + fireEvent.change(screen.getByPlaceholderText("Search input or trace ID"), { target: { value: "headphones" } }); + expect(screen.getByText("Can I return my headphones?")).toBeVisible(); + expect(screen.queryByText("Where is order #1042?")).not.toBeInTheDocument(); + fireEvent.change(screen.getByRole("textbox", { name: "Search runs" }), { target: { value: "" } }); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by agent" }), "support_agent"); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by status" }), "Failed"); + expect(within(screen.getByRole("table", { name: "Agent runs" })).getAllByRole("row")).toHaveLength(4); + await user.click(screen.getByRole("tab", { name: "Investigations", exact: true })); + expect(await screen.findByRole("button", { name: /Support quality/ })).toBeVisible(); + expect(screen.queryByRole("button", { name: "New investigation" })).not.toBeInTheDocument(); + expect(network).not.toHaveBeenCalled(); + expect(onUrlUpdate).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(await screen.findByText("Enable tracing")).toBeVisible(); + expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); + expect(screen.queryByText("Can I return my headphones?")).not.toBeInTheDocument(); + }); + + it("connects findings and history to their original trace without live requests or URL changes", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { + searchParams: "?tab=investigations", + onUrlUpdate, + }); + await screen.findByRole("button", { name: "Preview sample" }); + await user.click(screen.getByRole("button", { name: "Preview sample" })); + await user.click(await screen.findByRole("button", { name: /Support quality/ })); + network.mockClear(); + await user.click(screen.getByRole("button", { name: /Repeated lookups leave customers without an answer/ })); + const finding = screen.getByRole("dialog"); + expect(within(finding).getByText(/The support agent retries/)).toBeVisible(); + const summaries = within(finding).getAllByText("support_agent", { exact: true }); + await user.click(summaries[0]); + await user.click(within(finding).getAllByRole("button", { name: /Open original step/ })[0]); + expect(await screen.findByRole("complementary", { name: "Span details" })).toHaveTextContent( + "I will check that for you.", + ); + await user.click(screen.getByRole("button", { name: "Copy for agent" })); + expect(await navigator.clipboard.readText()).toContain("I will check that for you."); + expect(await navigator.clipboard.readText()).not.toContain("Authorization"); + await user.click(screen.getByRole("tab", { name: "Attributes" })); + expect(await screen.findByText("gen_ai.agent.name")).toBeVisible(); + await user.click( + within(screen.getByRole("dialog", { name: "Original run" })).getByRole("button", { name: "Close", exact: true }), + ); + await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Close", exact: true })); + await user.click(screen.getByRole("tab", { name: "History" })); + await user.click(screen.getAllByRole("button", { name: /runs reviewed/ })[1]); + expect(await screen.findByText(/1 linked run · high priority/)).toBeVisible(); + expect(network).not.toHaveBeenCalled(); + expect(onUrlUpdate).not.toHaveBeenCalled(); + await user.click(screen.getByRole("button", { name: "Exit demo" })); + expect(await screen.findByRole("heading", { name: "Find what needs attention" })).toBeVisible(); + }); + + it("has no demo entry for existing investigations, populated traces, or connecting another agent", async () => { + const user = userEvent.setup(); + const data = createLensDemoData(); + const saved = data.lenses[0]; + network.mockImplementation(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: true }); + if (path.endsWith("/runs")) return Response.json(saved.jobs); + if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) }); + return Response.json({ data: [], traces: true, requests: false }); + }); + renderWithProviders(, { + searchParams: `?tab=investigations&lens=${saved.id}`, + }); + expect(await screen.findByRole("heading", { name: saved.settings.name })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Traces" })); + expect(await screen.findByText("Where is order #1042?")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Set up tracing" })); + expect(await screen.findByRole("heading", { name: "Connect another agent" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + }); + + it("shows the header preview only for the active tab that still needs setup", async () => { + const user = userEvent.setup(); + const saved = createLensDemoData().lenses[0]; + network.mockImplementation(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: false }); + if (path.endsWith("/runs")) return Response.json(saved.jobs); + if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); + return Response.json({ data: [], traces: false, requests: false }); + }); + renderWithProviders(); + expect(await screen.findByRole("button", { name: "Preview sample" })).toBeVisible(); + const tabs = within(screen.getByRole("tablist", { name: "Lens" })); + await user.click(tabs.getByRole("tab", { name: "Investigations" })); + expect(await screen.findByRole("button", { name: new RegExp(saved.settings.name) })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(tabs.getByRole("tab", { name: "Traces" })); + await user.click(await screen.findByRole("button", { name: "Preview sample" })); + expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx new file mode 100644 index 00000000000..9c7d2ca3db8 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensWorkspace.tsx @@ -0,0 +1,117 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Aperture } from "lucide-react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { parseAsString, parseAsStringLiteral, useQueryState } from "nuqs"; +import AgentTracesPage from "@/components/view_logs/TraceView/AgentTracesPage"; +import { DemoNotice } from "@/components/shared/DemoNotice"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { LensDemoContext, useLensDemo } from "@/components/lens/LensDemoContext"; +import { LensPreviewTarget } from "@/components/lens/LensPreviewButton"; +import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles"; +import { LensView } from "./LensView"; +import { createLensDemo } from "./lensDemoData"; + +type Tab = "traces" | "investigations"; +type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean }; + +export function LensWorkspace(props: WorkspaceProps) { + const [demoTab, setDemoTab] = useState(null); + return demoTab ? ( + setDemoTab(null)} /> + ) : ( + + ); +} + +function DemoSession({ initialTab, onExit }: { initialTab: Tab; onExit: () => void }) { + const [demo] = useState(createLensDemo); + const [client] = useState( + () => new QueryClient({ defaultOptions: { queries: { retry: false, staleTime: Infinity } } }), + ); + useEffect( + () => () => { + client.clear(); + }, + [client], + ); + return ( + + + + + + ); +} + +function LensContent({ + accessToken, + userRole, + readOnly, + initialTab = "traces", + onDemo, + onExit, +}: WorkspaceProps & { initialTab?: Tab; onDemo?: (tab: Tab) => void; onExit?: () => void }) { + const demo = useLensDemo(); + const [tab, setTab] = useQueryState( + "tab", + parseAsStringLiteral(["traces", "investigations"]).withOptions({ history: "push" }), + ); + const [lensId] = useQueryState("lens", parseAsString); + const [demoTab, setDemoTab] = useState(initialTab); + const [previewTarget, setPreviewTarget] = useState(null); + const defaultTab = lensId ? "investigations" : "traces"; + const activeTab = demo ? demoTab : tab ?? defaultTab; + const openDemo = onDemo ? () => onDemo(activeTab) : undefined; + return ( + +
    +
    +

    +

    +
    +
    + {demo && } + (demo ? setDemoTab(value as Tab) : void setTab(value as Tab))} + className="min-h-0 flex-1 gap-4" + > + + + Traces + + + Investigations + + + + + + + {demo || isProxyAdminTierRole(userRole) ? ( + + ) : ( +

    + Investigations require proxy administrator access. You can still view your traces. +

    + )} +     +     +
    +     + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx index 4554174f37b..931dd7c17d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/WorkerSetup.tsx @@ -21,7 +21,7 @@ import { AnalysisKey, AnalysisAccessFields, createAnalysisKey, type AnalysisAcce import type { LensList, WorkerCreated } from "./lensData"; export const LENS_WORKER_IMAGE = - "ghcr.io/berriai/litellm-lens-worker@sha256:67eba741c1b97c749975c5c38e2370a603e1105babc908d613c1b79d7b995393"; + "ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b"; function initialProxyAddress(): string { const url = new URL(proxyBaseUrl || serverRootPath, window.location.origin); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts index 84e0fe7e6e6..3be6164fce2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.test.ts @@ -58,6 +58,28 @@ const job: Job = { }; describe("Analysis progress", () => { + it.each(["Collecting executions", "Reading executions"])( + "does not turn an unreported run count into zero runs during %s", + (stage) => { + expect(analysisProgress({ ...job, stage })).toEqual({ + step: -1, + title: "Preparing activity", + done: 0, + total: 0, + detail: "Loading the runs selected for this investigation.", + }); + }, + ); + + it("shows the selected count as soon as the worker reports it", () => { + expect(analysisProgress({ ...job, coverage: { ...coverage, selected: 7 } })).toMatchObject({ + step: 0, + done: 0, + total: 7, + detail: "0 of 7 selected runs reviewed", + }); + }); + it("measures review progress against the sample, not all eligible runs", () => { const expected = { step: 0, done: 7, total: 20, detail: "7 of 20 selected runs reviewed" }; expect( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts index c066577817b..1e30dc75a38 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensData.ts @@ -108,6 +108,15 @@ export function analysisProgress(job: Job) { : `${investigated} patterns checked against the original activity`, }; } + if (!selected) { + return { + step: -1, + title: "Preparing activity", + done: 0, + total: 0, + detail: "Loading the runs selected for this investigation.", + }; + } return { step: 0, title: "Reviewing activity", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts new file mode 100644 index 00000000000..eabae2928bd --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it, vi } from "vitest"; +import { createLensDemo, createLensDemoData } from "./lensDemoData"; +import type { TracePage } from "@/components/view_logs/TraceView/traceTypes"; +import { evidenceTarget } from "./lensData"; + +describe("Lens demo data", () => { + it("links every finding to the quoted original step and assessed run", () => { + const data = createLensDemoData(); + for (const lens of data.lenses) { + for (const job of lens.jobs) { + for (const finding of job.findings ?? []) { + for (const evidence of finding.evidence) { + const target = evidenceTarget(evidence.execution_id); + const run = data.runs.find(({ trace }) => trace.summary.trace_id === target?.id); + const step = run?.details.find((span) => span.span_id === evidence.span_id); + expect(step?.output).toContain(evidence.quote); + expect(job.sample?.executions.map((execution) => execution.id)).toContain(evidence.execution_id); + expect( + job.assessments.find((assessment) => assessment.execution_id === evidence.execution_id)?.[ + finding.kind === "issue" ? "issue_checks" : "pattern_checks" + ], + ).toContain(finding.check_id); + } + } + } + } + }); + + it("keeps trace totals, timestamps, and agent names consistent", () => { + const data = createLensDemoData(); + for (const { trace } of data.runs) { + expect(trace.summary.span_count).toBe(trace.spans.length); + expect(trace.summary.agent_names).toContain(trace.agents[0].name); + expect(trace.summary.error_count).toBe(trace.spans.filter((span) => span.status === "error").length); + for (const span of trace.spans) { + expect(span.start_offset_ms + span.duration_ms).toBeLessThanOrEqual(trace.summary.duration_ms); + } + } + }); + + it("filters time windows locally and rejects writes or unknown reads without network access", async () => { + const network = vi.spyOn(globalThis, "fetch"); + const now = Date.now(); + const { client } = createLensDemo(now); + const all = await client.get("/v1/traces"); + const recent = await client.get("/v1/traces", { query: { start_ms: now - 3600_000, end_ms: now } }); + expect(recent.data.length).toBeGreaterThan(0); + expect(recent.data.length).toBeLessThan(all.data.length); + expect(recent.data.every((trace) => Date.parse(trace.start_time) >= now - 3600_000)).toBe(true); + await expect(client.post("/lens", { body: {} })).rejects.toMatchObject({ status: 403 }); + await expect(client.get("/lens/real-investigation")).rejects.toMatchObject({ status: 404 }); + expect(network).not.toHaveBeenCalled(); + network.mockRestore(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts new file mode 100644 index 00000000000..8b4e1645c9e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/lensDemoData.ts @@ -0,0 +1,483 @@ +import { createApiClient } from "@/lib/http/client"; +import type { LensDemo } from "@/components/lens/LensDemoContext"; +import type { Trace, Span, SpanDetail } from "@/components/view_logs/TraceView/traceTypes"; +import type { Lens, Finding, Job, Settings } from "./lensData"; + +type Scenario = { + agent: string; + question: string; + tool: string; + result: string; + answer: string; + failed?: boolean; +}; + +const scenarios: Scenario[] = [ + { + agent: "support_agent", + question: "Where is order #1042?", + tool: "lookup_order", + result: "Order service timed out", + answer: "I will check that for you.", + failed: true, + }, + { + agent: "research_agent", + question: "Which database should we use for analytics?", + tool: "search_docs", + result: "Column stores are designed for analytical queries. No comparative benchmark is available.", + answer: "The column store is 40% faster than the row store for your workload.", + }, + { + agent: "support_agent", + question: "Can I return my headphones?", + tool: "lookup_return_policy", + result: "Returns are accepted within 30 days. This order was delivered 10 days ago.", + answer: "Yes. Your order is eligible for a return. I can help you start it.", + }, + { + agent: "release_agent", + question: "Is the checkout release ready to ship?", + tool: "read_test_results", + result: "Checkout: 120 passed, 0 failed. Rollback plan approved.", + answer: "All 120 checkout tests passed and the rollback plan is approved. The release is ready.", + }, + { + agent: "support_agent", + question: "My order #1088 has not arrived", + tool: "lookup_order", + result: "Order service timed out", + answer: "I will check that for you.", + failed: true, + }, + { + agent: "research_agent", + question: "How long do we retain audit logs?", + tool: "search_docs", + result: "Audit log retention is 90 days on the current plan.", + answer: "Audit logs are retained for 90 days, according to the plan documentation.", + }, + { + agent: "release_agent", + question: "Review the search release", + tool: "read_test_results", + result: "Search: 86 passed, 1 failed. Unicode query regression remains open.", + answer: "Hold the release. The Unicode query regression is still failing.", + }, + { + agent: "support_agent", + question: "Update the address on order #1091", + tool: "lookup_order", + result: "Order #1091 has not shipped. Address changes are allowed.", + answer: "Your order has not shipped. Please confirm the new delivery address.", + }, + { + agent: "research_agent", + question: "Can this plan store data in Europe?", + tool: "search_docs", + result: "Regional availability is not documented for this plan.", + answer: "I could not verify European data residency for this plan. Please confirm with support before choosing it.", + }, + { + agent: "support_agent", + question: "Check the delivery date for order #1107", + tool: "lookup_order", + result: "Order service timed out", + answer: "The order service is unavailable. I can connect you to support instead.", + failed: true, + }, + { + agent: "release_agent", + question: "Summarize the authentication release checks", + tool: "read_test_results", + result: "Authentication: 214 passed, 0 failed. Security review approved.", + answer: "All 214 tests passed and the security review is approved.", + }, + { + agent: "research_agent", + question: "Does the API support request retries?", + tool: "search_docs", + result: "Retry requests with exponential backoff after a 429 or 503 response.", + answer: "Yes. Use exponential backoff for 429 and 503 responses, as documented.", + }, +]; + +const executionId = (traceId: string) => btoa(JSON.stringify(["traces", "", traceId])); +const iso = (time: number) => new Date(time).toISOString(); + +function makeTrace(scene: Scenario, index: number, now: number) { + const traceId = (index + 1).toString(16).padStart(32, "0"); + const spanId = (step: number) => `${index + 1}${step}`.padStart(16, "0"); + const toolCount = scene.failed && index !== 9 ? 3 : 1; + const duration = toolCount === 3 ? 6340 : 3180 + index * 137; + const base: Span = { + agent: scene.agent, + duration_ms: duration, + error: null, + error_truncated: false, + framework: "", + input_preview: scene.question, + input_tokens: 0, + output_tokens: 0, + litellm_request_id: null, + model: null, + name: scene.agent, + parent_span_id: null, + span_id: spanId(0), + spend: null, + start_offset_ms: 0, + status: "ok", + type: "agent", + }; + const tools: Span[] = Array.from({ length: toolCount }, (_, i) => ({ + ...base, + span_id: spanId(i + 1), + parent_span_id: base.span_id, + name: scene.tool, + type: "tool", + start_offset_ms: 150 + i * 1600, + duration_ms: scene.failed ? 1500 : 340, + status: scene.failed ? "error" : "ok", + error: scene.failed ? scene.result : null, + })); + const model: Span = { + ...base, + span_id: spanId(5), + parent_span_id: base.span_id, + name: "Generate response", + type: "llm", + model: "demo-chat-model", + start_offset_ms: toolCount * 1600, + duration_ms: 1200, + input_tokens: 520 + index * 41, + output_tokens: 48 + index * 7, + spend: 0.003 + index * 0.0002, + }; + const trace: Trace = { + summary: { + agent_count: 1, + agent_invocations: 1, + agent_names: [scene.agent], + duration_ms: duration, + error_count: scene.failed ? toolCount : 0, + frameworks: [], + input_preview: scene.question, + input_tokens: model.input_tokens, + output_tokens: model.output_tokens, + llm_calls: 1, + models: [model.model!], + name: scene.agent, + service: "demo-agents", + span_count: toolCount + 2, + spend: model.spend, + start_time: iso(now - (index + 1) * 35 * 60_000), + status: scene.failed ? "error" : "ok", + tool_calls: toolCount, + trace_id: traceId, + }, + agents: [ + { + name: scene.agent, + parent_agent: null, + duration_ms: duration, + invocations: 1, + llm_calls: 1, + tool_calls: toolCount, + spend: model.spend, + }, + ], + spans: [base, ...tools, model], + }; + const details: SpanDetail[] = trace.spans.map((span) => ({ + span_id: span.span_id, + input: + span.type === "tool" + ? JSON.stringify({ query: scene.question }) + : JSON.stringify([{ role: "user", content: scene.question }]), + output: span.type === "tool" ? scene.result : JSON.stringify([{ role: "assistant", content: scene.answer }]), + attributes: { + "gen_ai.agent.name": scene.agent, + "service.name": "demo-agents", + demo: "true", + ...(span.model ? { "gen_ai.request.model": span.model } : {}), + }, + })); + return { trace, details }; +} + +export function createLensDemoData(now = Date.now()) { + const runs = scenarios.map((scene, index) => makeTrace(scene, index, now)); + const finding = ({ + id, + check, + title, + description, + suggestion, + indices, + kind = "issue", + }: { + id: string; + check: string; + title: string; + description: string; + suggestion: string; + indices: number[]; + kind?: Finding["kind"]; + }): Finding => ({ + id, + check_id: check, + title, + description, + suggestion, + kind, + priority: kind === "issue" ? "high" : "low", + status: "open", + reason: "", + revision: 1, + first_seen: iso(now - 86_400_000), + last_seen: iso(now - 300_000), + limitation: "", + occurrences: indices.map((i) => executionId(runs[i].trace.summary.trace_id)), + evidence: indices.map((i) => ({ + execution_id: executionId(runs[i].trace.summary.trace_id), + span_id: runs[i].trace.spans.at(-1)!.span_id, + quote: scenarios[i].answer, + role: "support", + })), + }); + const findingInputs: Parameters[0][] = [ + { + id: "failed-lookups", + check: "recover", + title: "Repeated lookups leave customers without an answer", + description: + "The support agent retries the same unavailable order service three times, then promises to check without answering or offering a handoff.", + suggestion: "After repeated failures, explain the problem and offer a handoff.", + indices: [0, 4], + }, + { + id: "safe-handoff", + check: "recover", + title: "A clear handoff helps when the order service is unavailable", + description: + "The agent explains the service outage and offers a support handoff instead of promising an answer it cannot provide.", + suggestion: "Keep this fallback for unavailable services.", + indices: [9], + kind: "pattern", + }, + { + id: "unsupported-claim", + check: "grounding", + title: "Performance claim has no supporting benchmark", + description: + "The answer claims a 40% performance improvement, but the retrieved documentation contains no comparative benchmark.", + suggestion: "Require a benchmark source for numeric performance claims, or remove the comparison.", + indices: [1], + }, + { + id: "uncertainty", + check: "grounding", + title: "Missing information is acknowledged", + description: + "When documentation does not establish regional availability, the agent says so and asks the user to verify it.", + suggestion: "Keep stating when a source does not answer the question.", + indices: [8], + kind: "pattern", + }, + { + id: "release-blocked", + check: "release", + title: "Failing checks correctly block the release", + description: "The release agent identifies an unresolved regression and recommends holding the release.", + suggestion: "Continue requiring passing checks before recommending a release.", + indices: [6], + kind: "pattern", + }, + ]; + const findings = findingInputs.map(finding); + const definitions = [ + { + id: "support", + name: "Support quality", + agent: "support_agent", + check: "recover", + context: + "Answer the customer's question using order information. If a tool fails, explain the problem and offer a handoff.", + instruction: "Look for repeated failed calls and conversations that end without an answer or a handoff.", + }, + { + id: "research", + name: "Research accuracy", + agent: "research_agent", + check: "grounding", + context: "Answer questions using verified documentation. Acknowledge missing information.", + instruction: "Find claims that are not supported by the retrieved sources.", + }, + { + id: "release", + name: "Release readiness", + agent: "release_agent", + check: "release", + context: "Review test results and recommend a release only when all required checks pass.", + instruction: "Check whether failed tests are acknowledged before a release recommendation.", + }, + ]; + const lenses: Lens[] = definitions.map((definition) => { + const settings: Settings = { + name: definition.name, + agent_name: definition.agent, + context: definition.context, + checks: [{ id: definition.check, instruction: definition.instruction, enabled: true }], + source: "traces", + service: "", + filters: [], + team_id: "", + execution_ids: [], + lookback_hours: 24, + sample_size: 0, + sample_percent: 100, + concurrency: 8, + monthly_budget: 100, + interval_minutes: 30, + enabled: false, + model: "demo-analysis-model", + }; + const executions = runs + .filter(({ trace }) => trace.summary.name === definition.agent) + .map(({ trace }) => ({ + id: executionId(trace.summary.trace_id), + trace_ref: "", + metadata: [], + root_seen: true, + service: trace.summary.service, + source: "traces" as const, + trace_id: trace.summary.trace_id, + team_id: "", + name: trace.summary.name, + start_time: trace.summary.start_time, + span_count: trace.summary.span_count, + })); + const relevant = findings.filter((f) => f.check_id === definition.check); + const jobs: Job[] = [0, 1].map((day) => { + const sample = day === 0 ? executions : executions.slice(1); + const selectedIds = new Set(sample.map((item) => item.id)); + const snapshot = relevant + .map((f) => ({ + ...f, + occurrences: f.occurrences.filter((id) => selectedIds.has(id)), + evidence: f.evidence.filter((e) => selectedIds.has(e.execution_id)), + })) + .filter((f) => f.occurrences.length > 0); + return { + id: `${definition.id}-scan-${day}`, + findings: snapshot, + settings, + revision: 1, + assessments: sample.map((e) => ({ + execution_id: e.id, + cannot_assess: false, + issue_checks: snapshot.some((f) => f.kind === "issue" && f.occurrences.includes(e.id)) + ? [definition.check] + : [], + pattern_checks: snapshot.some((f) => f.kind === "pattern" && f.occurrences.includes(e.id)) + ? [definition.check] + : [], + })), + attempts: 1, + error: "", + cost: sample.length * 0.012, + coverage: { + eligible: sample.length, + selected: sample.length, + screened: sample.length, + investigated: sample.length, + inconclusive: 0, + grouping_batches: 1, + grouped_batches: 1, + candidates: snapshot.length, + partial: 0, + unassessable: 0, + }, + status: "completed", + stage: "Complete", + created_at: iso(now - 330_000 - day * 60_000), + finished_at: iso(now - 300_000 - day * 60_000), + start: iso(now - 86_400_000), + end: iso(now - 300_000 - day * 60_000), + sample: { eligible: sample.length, selected: sample.length, executions: sample }, + }; + }); + return { + id: definition.id, + version: 1, + revision: 1, + spent: jobs.reduce((sum, job) => sum + job.cost, 0), + scope: { all_teams: true, api_key_hash: "", team_id: "" }, + settings, + created_at: iso(now - 2 * 86_400_000), + next_run_at: iso(now), + budget_month: iso(now).slice(0, 7), + findings: relevant, + jobs, + }; + }); + return { runs, lenses }; +} + +function readDemoTrace(data: ReturnType, url: URL): unknown { + const parts = url.pathname.split("/").filter(Boolean); + if (parts[0] !== "v1" || parts[1] !== "traces") return undefined; + if (!parts[2]) { + const start = Number(url.searchParams.get("start_ms") ?? 0); + const end = Number(url.searchParams.get("end_ms") ?? Infinity); + return { + data: data.runs + .map((run) => run.trace.summary) + .filter((trace) => Date.parse(trace.start_time) >= start && Date.parse(trace.start_time) <= end), + next_cursor: null, + }; + } + const run = data.runs.find(({ trace }) => trace.summary.trace_id === parts[2]); + if (!parts[3]) return run?.trace; + if (parts[3] === "spans" && !parts[5]) return run?.details.find((span) => span.span_id === parts[4]); + if (parts[3] === "spans" && parts[5] === "error") { + const span = run?.trace.spans.find((item) => item.span_id === parts[4]); + return span + ? { span_id: span.span_id, message: span.error ?? "", total_chars: span.error?.length ?? 0, next_cursor: null } + : undefined; + } + return undefined; +} + +export function createLensDemo(now = Date.now()): LensDemo { + const data = createLensDemoData(now); + const read = (url: URL): unknown => { + const parts = url.pathname.split("/").filter(Boolean); + if (url.pathname === "/lens") return { lenses: data.lenses, workers: [], tracing_enabled: true }; + if (url.pathname === "/lens/activity/available") return { traces: true, requests: false }; + if (url.pathname === "/models" || url.pathname === "/model_group/info") return { data: [] }; + if (parts[0] === "lens" && parts[2] === "runs") { + const jobs = data.lenses.find((lens) => lens.id === parts[1])?.jobs; + return parts[3] + ? jobs?.find((job) => job.id === parts[3]) + : jobs?.slice(Number(url.searchParams.get("offset") ?? 0)); + } + return readDemoTrace(data, url); + }; + return { + client: createApiClient({ + getBaseUrl: () => "https://lens-demo.invalid", + fetchImpl: async (input, init) => { + if (init?.method !== "GET") return Response.json({ detail: "Demo data is read-only" }, { status: 403 }); + const result = read(new URL(String(input))); + return result === undefined + ? Response.json({ detail: "This item is not in the demo" }, { status: 404 }) + : Response.json(result); + }, + }), + copyTrace: (traceId, spanId) => { + const run = data.runs.find(({ trace }) => trace.summary.trace_id === traceId); + return JSON.stringify(spanId ? run?.details.find((span) => span.span_id === spanId) : run, null, 2); + }, + }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx index b29e498600f..da13f48165b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/page.tsx @@ -1,60 +1,10 @@ "use client"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { Aperture } from "lucide-react"; -import { parseAsString, parseAsStringLiteral, useQueryState } from "nuqs"; -import AgentTracesPage from "@/components/view_logs/TraceView/AgentTracesPage"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { isProxyAdminRole, isProxyAdminTierRole } from "@/utils/roles"; -import { LensView } from "./_components/LensView"; +import { LensWorkspace } from "./_components/LensWorkspace"; export default function LensPage() { const { accessToken, userRole, isViewOnly } = useAuthorized(); - const [tab, setTab] = useQueryState( - "tab", - parseAsStringLiteral(["traces", "investigations"]).withOptions({ history: "push" }), - ); - const [lensId] = useQueryState("lens", parseAsString); - const defaultTab = lensId ? "investigations" : "traces"; - const activeTab = tab ?? defaultTab; if (!accessToken) return null; - return ( -
    -

    -

    - void setTab(value as "traces" | "investigations")} - className="min-h-0 flex-1 gap-4" - > - - - Traces - - - Investigations - - - - - - - {isProxyAdminTierRole(userRole ?? "") ? ( - - ) : ( -

    - Investigations require proxy administrator access. You can still view your traces. -

    - )} -     -     -
    - ); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx index 3e71e9fb860..1501938e9c5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.integration.test.tsx @@ -249,6 +249,32 @@ describe("ROICalculatorView", () => { expect(screen.getAllByText("Connect GitHub to get started")).toHaveLength(1); }); + it("clearly identifies the sample report and returns to setup when exiting", async () => { + const emptySettings = { ...settings, has_github_token: false, ready: false, repos: [], estimator_model: "" }; + vi.mocked(apiClient.get).mockImplementation((path: string, options) => { + if (path === "/roi-calculator/settings") return Promise.resolve(emptySettings); + if (path === "/roi-calculator/report") { + return Promise.resolve({ report: options?.query?.mode === "demo" ? { ...summary, mode: "demo" } : null }); + } + return Promise.resolve(idleStatus); + }); + + render(); + fireEvent.click(await screen.findByRole("button", { name: "Preview sample report" })); + + expect(await screen.findByText("You’re viewing demo data")).toBeVisible(); + expect(screen.getByText("Spend per estimated engineering hour")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Run analysis" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Settings" })).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Exit demo" })); + + expect(screen.getByRole("heading", { name: "Connect GitHub to get started" })).toBeVisible(); + expect(screen.queryByText("You’re viewing demo data")).not.toBeInTheDocument(); + expect(apiClient.post).not.toHaveBeenCalled(); + expect(apiClient.put).not.toHaveBeenCalled(); + }); + it("returns to Overview and shows the last sync time when completion is polled from Settings", async () => { const runningStatus = { ...idleStatus, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx index 1f5b136dbb7..40c1104354d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/roi-calculator/_components/ROICalculatorView.tsx @@ -5,6 +5,7 @@ import { Calculator, RefreshCw } from "lucide-react"; import { apiClient } from "@/components/networking"; import { PageHeader } from "@/components/shared/PageHeader"; +import { DemoNotice } from "@/components/shared/DemoNotice"; import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent } from "@/components/ui/card"; @@ -222,6 +223,13 @@ export default function ROICalculatorView({ } title="ROI Calculator" + titleAction={ + !liveSummary && showLiveStatus ? ( + + ) : undefined + } subtitle={ <> {summary @@ -238,22 +246,7 @@ export default function ROICalculatorView({ } /> - {!liveSummary && showLiveStatus && ( - - )} - {sampleSummary && ( - - Sample report - - Example data only. No GitHub or model requests were made. - - - - )} + {sampleSummary && setSampleSummary(null)} />} {liveSummary && showLiveStatus && (

    {status.next_update ? `Next update ${formatSyncedAt(status.next_update)}` : scheduleLabel} diff --git a/ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx b/ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx new file mode 100644 index 00000000000..dd34511bc09 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensDemoContext.tsx @@ -0,0 +1,12 @@ +"use client"; + +import { createContext, useContext } from "react"; +import type { ApiClient } from "@/lib/http/client"; + +export interface LensDemo { + client: ApiClient; + copyTrace: (traceId: string, spanId?: string) => string; +} + +export const LensDemoContext = createContext(null); +export const useLensDemo = () => useContext(LensDemoContext); diff --git a/ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx b/ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx new file mode 100644 index 00000000000..845a6b8d279 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensPreviewButton.tsx @@ -0,0 +1,18 @@ +"use client"; + +import { createContext, useContext } from "react"; +import { createPortal } from "react-dom"; +import { Button } from "@/components/ui/button"; + +export const LensPreviewTarget = createContext(undefined); + +export function LensPreviewButton({ onClick }: { onClick: () => void }) { + const target = useContext(LensPreviewTarget); + const button = ( + + ); + if (target === null) return null; + return target ? createPortal(button, target) : button; +} diff --git a/ui/litellm-dashboard/src/components/shared/DemoNotice.tsx b/ui/litellm-dashboard/src/components/shared/DemoNotice.tsx new file mode 100644 index 00000000000..e435d725db5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/DemoNotice.tsx @@ -0,0 +1,16 @@ +import { Info } from "lucide-react"; +import { Button } from "@/components/ui/button"; + +export function DemoNotice({ onExit }: { onExit?: () => void }) { + return ( +

    +

    +

    + +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx index 81092821efc..6fc5b0f3238 100644 --- a/ui/litellm-dashboard/src/components/shared/PageHeader.tsx +++ b/ui/litellm-dashboard/src/components/shared/PageHeader.tsx @@ -14,11 +14,12 @@ interface PageHeaderProps { subtitle: React.ReactNode; icon: React.ReactNode; primaryAction?: React.ReactNode; + titleAction?: React.ReactNode; tabs?: React.ReactNode | ((slots: EmbeddedTabsSlots) => React.ReactNode); utilities?: React.ReactNode; } -export function PageHeader({ title, subtitle, icon, primaryAction, tabs, utilities }: PageHeaderProps) { +export function PageHeader({ title, subtitle, icon, primaryAction, titleAction, tabs, utilities }: PageHeaderProps) { const leadingControls = primaryAction == null ? null : (
    @@ -31,14 +32,17 @@ export function PageHeader({ title, subtitle, icon, primaryAction, tabs, utiliti return (
    -
    - -

    {title}

    +
    +
    + +

    {title}

    +
    + {titleAction}

    {subtitle}

    diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx index d0c5964a4c3..d46fc372e78 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesPage.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import moment from "moment"; import { useMemo, useState } from "react"; @@ -13,12 +14,15 @@ export default function AgentTracesPage({ isActive = true, readOnly = false, canMintTracingKey = false, + onDemo, }: { accessToken: string; isActive?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; }) { + const demo = useLensDemo(); const [rangeHours, setRangeHours] = useState(DEFAULT_RANGE_HOURS); const [live, setLive] = useState(true); const [anchor, setAnchor] = useState(() => moment()); @@ -36,16 +40,17 @@ export default function AgentTracesPage({ }; return ( -
    +
    diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx new file mode 100644 index 00000000000..816bf5f9267 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.integration.test.tsx @@ -0,0 +1,509 @@ +import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ApiError } from "@/lib/http/client"; + +import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import traceList from "./__fixtures__/trace_list.json"; +import AgentTracesPage from "./AgentTracesPage"; +import { AgentTracesSection, type TimeControls } from "./AgentTracesSection"; +import type { TracePage, TraceSummary } from "./traceTypes"; + +vi.mock("../../networking", () => ({ + apiClient: { get: vi.fn(), post: vi.fn() }, + agentTraceListCall: vi.fn(), + sendOtlpTraceCall: vi.fn(), + agentTraceCall: vi.fn(), + agentTraceSpanCall: vi.fn(), + getProxyBaseUrl: () => "http://localhost:4000", +})); + +vi.mock("./TraceDrawer", () => ({ + RunView: ({ traceId, onBack }: { traceId: string; onBack: () => void }) => ( +
    + run {traceId} + +
    + ), +})); + +import { agentTraceListCall, apiClient } from "../../networking"; + +const runs = (traceList as TracePage).data as TraceSummary[]; + +const renderSection = () => + renderWithProviders( + , + ); + +// A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone. +const renderWindowed = (timeControls?: TimeControls) => + renderWithProviders( + , + ); + +const bucketRunCounts = () => + screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-runs"))); + +describe("AgentTracesSection", () => { + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + beforeEach(() => { + vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({ + left: 0, + width: 600, + top: 0, + height: 56, + right: 600, + bottom: 56, + x: 0, + y: 0, + toJSON: () => ({}), + } as DOMRect); + testQueryClient.clear(); + vi.mocked(agentTraceListCall).mockReset(); + vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); + }); + + it.each([ + [401, "Your session is no longer valid. Sign out and sign in again."], + [403, "Your account does not have access to these traces."], + ])("stops live polling after HTTP %s and explains how to recover", async (status, message) => { + vi.useFakeTimers(); + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Private token details", Number(status), {})); + renderWithProviders( + , + ); + await act(async () => { + await vi.advanceTimersByTimeAsync(60_000); + }); + expect(screen.getByText(`Could not load runs: ${message}`)).toBeVisible(); + expect(screen.queryByText(/Private token details/)).not.toBeInTheDocument(); + expect(agentTraceListCall).toHaveBeenCalledOnce(); + expect(screen.getByTestId("runs-footer")).toHaveTextContent("Update failed"); + }); + + it("renders the setup snippet when the proxy answers 501", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue( + new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }), + ); + renderSection(); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Tracing is not enabled"); + expect(screen.getByRole("button", { name: "Preview sample" })).toBeVisible(); + expect(card).toHaveTextContent("type: clickhouse"); + expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); + expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); + expect(card).not.toHaveTextContent(/langsmith/i); + expect(card).toHaveTextContent("ClickHouse and proxy setup"); + expect(card).toHaveTextContent("Ask your proxy administrator"); + }); + + it("shows the waiting guide when tracing is on but no runs have arrived", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + renderSection(); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Connect your agent"); + expect(card).toHaveTextContent("Waiting for your first trace"); + expect(screen.getByRole("button", { name: "Preview sample" })).toBeVisible(); + expect(card).not.toHaveTextContent("store: clickhouse"); + }); + + it("keeps the trace list available when traces exist outside the current time window", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + vi.mocked(apiClient.get).mockResolvedValue(traceList); + renderSection(); + expect(await screen.findByText("No runs match these filters.")).toBeVisible(); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + expect(apiClient.get).toHaveBeenCalledWith("/v1/traces", { accessToken: "sk-test", query: { start_ms: 0 } }); + }); + + it("checks proxy readiness, waits for an agent, and confirms receipt using actual query results", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); + renderSection(); + const checkSetup = await screen.findByRole("button", { name: "Check setup" }); + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + fireEvent.click(checkSetup); + expect(await screen.findByRole("heading", { name: "Connect your agent" })).toBeVisible(); + expect(screen.getByText("Waiting for your first trace")).toBeVisible(); + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + fireEvent.click(screen.getByRole("button", { name: "Check for traces" })); + expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length); + expect(screen.getByText("Traces received. Select a run to inspect it.")).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + }); + + it("keeps setup visible while checking and explains when tracing is still disabled", async () => { + const failure = new ApiError("Tracing is not enabled", 501, {}); + vi.mocked(agentTraceListCall).mockRejectedValue(failure); + renderSection(); + const checkSetup = await screen.findByRole("button", { name: "Check setup" }); + let rejectCheck: (error: Error) => void = () => {}; + vi.mocked(agentTraceListCall).mockImplementationOnce( + () => + new Promise((_, reject) => { + rejectCheck = reject; + }), + ); + fireEvent.click(checkSetup); + expect(await screen.findByRole("button", { name: "Checking…" })).toBeDisabled(); + expect(screen.getByRole("heading", { name: "Enable tracing" })).toBeVisible(); + await act(async () => rejectCheck(failure)); + expect(await screen.findByText(/Tracing is still unavailable/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); + }); + + it("keeps received traces and the open drawer visible during subsequent fetches", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); + renderSection(); + const checkSetup = await screen.findByRole("button", { name: "Check setup" }); + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + fireEvent.click(checkSetup); + const rows = await screen.findAllByTestId("agent-trace-row"); + fireEvent.click(rows[0]); + const drawer = screen.getByRole("complementary", { name: "Trace details" }); + + let finishRefresh: (page: TracePage) => void = () => {}; + vi.mocked(agentTraceListCall).mockImplementationOnce( + () => + new Promise((resolve) => { + finishRefresh = resolve; + }), + ); + await act(async () => { + void testQueryClient.invalidateQueries({ queryKey: ["agentTraces"] }); + }); + expect(await screen.findByText("Updating…")).toBeVisible(); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + await act(async () => finishRefresh(traceList as TracePage)); + }); + + it("separates a failed history check from the empty list and retries that check", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + vi.mocked(apiClient.get).mockRejectedValue(new ApiError("History unavailable", 503, {})); + renderSection(); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not check earlier traces. History unavailable"); + expect(screen.getByText("No runs match these filters.")).toBeVisible(); + expect(screen.queryByText(/Could not load runs/)).not.toBeInTheDocument(); + + vi.mocked(apiClient.get).mockResolvedValue(traceList); + fireEvent.click(screen.getByRole("button", { name: "Retry trace check" })); + await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); + expect(screen.getByText("No runs match these filters.")).toBeVisible(); + expect(agentTraceListCall).toHaveBeenCalledTimes(1); + }); + + it("treats a proxy without the trace routes (404) like tracing being off", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" })); + renderSection(); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Tracing is not enabled"); + expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); + }); + + it("lists every run with its input, counts and failed column", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + + const rows = await screen.findAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(runs.length); + const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans")); + expect(lead).toBeDefined(); + const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement; + expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument(); + expect(screen.getByText(`${runs.length} runs`)).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument(); + expect(within(failed).getByText("—")).toBeInTheDocument(); + }); + + it("shows the spend returned for a run", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + ...(traceList as TracePage), + data: [{ ...runs[0], spend: 0.025 }], + }); + renderSection(); + + const row = await screen.findByTestId("agent-trace-row"); + expect(within(row).getByText("$0.03")).toBeInTheDocument(); + }); + + it("filters by input text and by trace id", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + await screen.findAllByTestId("agent-trace-row"); + + const search = screen.getByLabelText("Search runs"); + fireEvent.change(search, { target: { value: "acme-404" } }); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); + + const lead = runs.find((r) => r.name === "research_lead") as TraceSummary; + fireEvent.change(search, { target: { value: lead.trace_id.slice(0, 10) } }); + const rows = screen.getAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans"); + }); + + it("uses recorded agent names for the column and filter even when services are shared", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + ...(traceList as TracePage), + data: [ + ...runs.slice(1).map((run) => ({ ...run, service: "shared-app", agent_names: ["research-agent"] })), + { ...runs[0], service: "shared-app", agent_names: ["billing-agent", "review-agent"] }, + ], + }); + const user = userEvent.setup(); + renderSection(); + await screen.findAllByTestId("agent-trace-row"); + + expect(screen.getByRole("columnheader", { name: "Agent" })).toBeInTheDocument(); + const agentFilter = screen.getByRole("combobox", { name: "Filter by agent" }); + expect(agentFilter).toHaveTextContent("All agents"); + + await chooseSelectOption(user, agentFilter, "billing-agent"); + const rows = screen.getAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("billing-agent"); + expect(rows[0]).not.toHaveTextContent("shared-app"); + + await chooseSelectOption(user, agentFilter, "review-agent"); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); + + await chooseSelectOption(user, agentFilter, "All agents"); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + }); + + it("shows each run's agent name with the logo of the SDK that produced it", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ + ...(traceList as TracePage), + data: [ + { ...runs[0], agent_names: ["research-bot"], frameworks: ["claude-agent-sdk", "claude-code"] }, + { ...runs[1], agent_names: [], frameworks: ["claude-code"] }, + { ...runs[2], frameworks: [] }, + ], + }); + renderSection(); + const [sdkRun, cliRun, plainRun] = await screen.findAllByTestId("agent-trace-row"); + const agentCell = (row: HTMLElement) => within(row).getAllByRole("cell")[1]; + + expect(agentCell(sdkRun)).toHaveTextContent(/^research-bot$/); + expect(agentCell(sdkRun)).toHaveAttribute("title", "research-bot · Claude Agent SDK"); + expect(within(sdkRun).getByRole("img", { name: "Claude Agent SDK logo", hidden: true })).toHaveAttribute( + "src", + expect.stringContaining("anthropic.svg"), + ); + expect(agentCell(cliRun)).toHaveTextContent(/^Claude Code$/); + expect(within(plainRun).queryByRole("img", { hidden: true })).not.toBeInTheDocument(); + expect(agentCell(plainRun)).toHaveTextContent((runs[2].agent_names ?? [runs[2].service]).join(", ")); + }); + + it("opens a run in a side drawer over the list and swaps runs without closing it", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + const rows = await screen.findAllByTestId("agent-trace-row"); + + fireEvent.click(rows[0]); + const drawer = screen.getByRole("complementary", { name: "Trace details" }); + expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); + expect(screen.getByTestId("runs-table")).toBeInTheDocument(); + expect(rows[0]).toHaveAttribute("aria-selected", "true"); + + fireEvent.click(rows[1]); + expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); + expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); + expect(rows[1]).toHaveAttribute("aria-selected", "true"); + expect(rows[0]).toHaveAttribute("aria-selected", "false"); + }); + + it("closes the drawer when the open row is clicked again or Escape is pressed", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + const rows = await screen.findAllByTestId("agent-trace-row"); + + fireEvent.click(rows[0]); + fireEvent.click(rows[0]); + expect(rows[0]).toHaveAttribute("aria-selected", "false"); + + fireEvent.click(rows[1]); + fireEvent.keyDown(window, { key: "Escape" }); + expect(rows[1]).toHaveAttribute("aria-selected", "false"); + }); + + it("moves to the next and previous run with j / k and the header arrows", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection(); + const rows = await screen.findAllByTestId("agent-trace-row"); + + fireEvent.click(rows[0]); + fireEvent.keyDown(window, { key: "j" }); + expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); + fireEvent.keyDown(window, { key: "k" }); + expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); + expect(screen.getByRole("button", { name: "Previous trace (K)" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Next trace (J)" })); + expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); + }); + + it("plots every loaded run on the timeline", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed(); + await screen.findAllByTestId("agent-trace-row"); + + expect(screen.getByTestId("traces-timeline")).toBeInTheDocument(); + const counts = bucketRunCounts(); + expect(counts).toHaveLength(60); + expect(counts.reduce((a, b) => a + b, 0)).toBe(runs.length); + }); + + it("zooms by dragging, resizes and pans the bracket, and clears with Esc", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed(); + await screen.findAllByTestId("agent-trace-row"); + const area = screen.getByTestId("timeline-area"); + const x = (bucket: number) => bucket * 10 + 5; + const drag = (target: HTMLElement, from: number, to: number) => { + fireEvent.pointerDown(target, { clientX: x(from), pointerId: 1 }); + fireEvent.pointerMove(area, { clientX: x(to), pointerId: 1 }); + fireEvent.pointerUp(area, { clientX: x(to), pointerId: 1 }); + }; + const rowCount = () => screen.queryAllByTestId("agent-trace-row").length; + const withRuns = bucketRunCounts().flatMap((count, i) => (count > 0 ? [i] : [])); + const first = withRuns[0]; + // The pan below moves a [0, first] bracket to the far right; it must end up clear of every run. + expect(first).toBeGreaterThan(1); + expect(first).toBeLessThan(30); + + drag(area, 0, 1); + expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); + expect(rowCount()).toBe(0); + + drag(screen.getByTestId("timeline-handle-hi"), 1, first); + expect(rowCount()).toBeGreaterThan(0); + + drag(screen.getByTestId("timeline-selection"), 1, 1 - first); + expect(rowCount()).toBeGreaterThan(0); + drag(screen.getByTestId("timeline-selection"), 0, 59); + expect(rowCount()).toBe(0); + + fireEvent.keyDown(screen.getByTestId("traces-timeline"), { key: "Escape" }); + expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); + expect(rowCount()).toBe(runs.length); + }); + + it("clears timeline zoom when refreshed", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWindowed({ rangeHours: 24, onRangeHoursChange: () => {}, onLiveChange: () => {} }); + await screen.findAllByTestId("agent-trace-row"); + const area = screen.getByTestId("timeline-area"); + const x = (bucket: number) => bucket * 10 + 5; + + fireEvent.pointerDown(area, { clientX: x(0), pointerId: 1 }); + fireEvent.pointerMove(area, { clientX: x(1), pointerId: 1 }); + fireEvent.pointerUp(area, { clientX: x(1), pointerId: 1 }); + expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); + expect(screen.queryAllByTestId("agent-trace-row")).toHaveLength(0); + + fireEvent.click(screen.getByRole("button", { name: "Refresh" })); + + expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); + expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); + }); +}); + +describe("AgentTracesPage", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(agentTraceListCall).mockReset(); + vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); + }); + + it("shows the actual range, switches presets from the popover, and toggles Live", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const trigger = screen.getByRole("button", { name: "Time range" }); + expect(trigger).toHaveTextContent(/ to /); + expect(screen.getByTestId("traces-timeline")).toHaveTextContent("Total 1d"); + + fireEvent.click(trigger); + fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last 7 days" })); + expect(await screen.findByText("Total 7d")).toBeInTheDocument(); + const last = vi.mocked(agentTraceListCall).mock.calls.at(-1)?.[0]; + expect((last?.endMs ?? 0) - (last?.startMs ?? 0)).toBeGreaterThanOrEqual(7 * 24 * 3600 * 1000 - 60_000); + + const live = screen.getByRole("button", { name: "Live" }); + expect(live).toHaveAttribute("aria-pressed", "true"); + fireEvent.click(live); + expect(live).toHaveAttribute("aria-pressed", "false"); + expect(screen.getByRole("button", { name: "Refresh" })).toBeEnabled(); + }); + + it("refreshes the trace list", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const callsBeforeRefresh = vi.mocked(agentTraceListCall).mock.calls.length; + fireEvent.click(screen.getByRole("button", { name: "Refresh" })); + + await waitFor(() => expect(vi.mocked(agentTraceListCall).mock.calls.length).toBeGreaterThan(callsBeforeRefresh)); + }); + + it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + fireEvent.click(screen.getByRole("button", { name: "Time range" })); + fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last hour" })); + + expect(await screen.findByText("No runs match these filters.")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Time range" })).toBeInTheDocument(); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + }); + + it("asks the proxy for the last 24 hours by default", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderWithProviders(); + await screen.findByTestId("runs-table"); + + const { startMs, endMs } = vi.mocked(agentTraceListCall).mock.calls[0][0]; + expect(endMs - startMs).toBeGreaterThanOrEqual(24 * 3600 * 1000 - 60_000); + expect(endMs - startMs).toBeLessThan(24 * 3600 * 1000 + 120_000); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx index e90544ca0a6..062c6d30da6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -1,330 +1,11 @@ -import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { ApiError } from "@/lib/http/client"; - -import { chooseSelectOption, renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import { describe, expect, it } from "vitest"; +import { filterRuns } from "./AgentTracesSection"; import traceList from "./__fixtures__/trace_list.json"; -import AgentTracesPage from "./AgentTracesPage"; -import { AgentTracesSection, filterRuns, type TimeControls } from "./AgentTracesSection"; import type { TracePage, TraceSummary } from "./traceTypes"; -vi.mock("../../networking", () => ({ - apiClient: { get: vi.fn(), post: vi.fn() }, - agentTraceListCall: vi.fn(), - sendOtlpTraceCall: vi.fn(), - agentTraceCall: vi.fn(), - agentTraceSpanCall: vi.fn(), - getProxyBaseUrl: () => "http://localhost:4000", -})); - -vi.mock("./TraceDrawer", () => ({ - RunView: ({ traceId, onBack }: { traceId: string; onBack: () => void }) => ( -
    - run {traceId} - -
    - ), -})); - -import { agentTraceListCall, apiClient } from "../../networking"; - const runs = (traceList as TracePage).data as TraceSummary[]; -const renderSection = () => - renderWithProviders( - , - ); - -// A UTC-pinned day around the fixture runs (2026-09-30 ~06:43 UTC), so they land in the same bucket in any timezone. -const renderWindowed = (timeControls?: TimeControls) => - renderWithProviders( - , - ); - -const bucketRunCounts = () => - screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-runs"))); - -describe("AgentTracesSection", () => { - afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - beforeEach(() => { - vi.spyOn(HTMLElement.prototype, "getBoundingClientRect").mockReturnValue({ - left: 0, - width: 600, - top: 0, - height: 56, - right: 600, - bottom: 56, - x: 0, - y: 0, - toJSON: () => ({}), - } as DOMRect); - testQueryClient.clear(); - vi.mocked(agentTraceListCall).mockReset(); - vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); - }); - - it.each([ - [401, "Your session is no longer valid. Sign out and sign in again."], - [403, "Your account does not have access to these traces."], - ])("stops live polling after HTTP %s and explains how to recover", async (status, message) => { - vi.useFakeTimers(); - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Private token details", Number(status), {})); - renderWithProviders( - , - ); - await act(async () => { - await vi.advanceTimersByTimeAsync(60_000); - }); - expect(screen.getByText(`Could not load runs: ${message}`)).toBeVisible(); - expect(screen.queryByText(/Private token details/)).not.toBeInTheDocument(); - expect(agentTraceListCall).toHaveBeenCalledOnce(); - expect(screen.getByTestId("runs-footer")).toHaveTextContent("Update failed"); - }); - - it("renders the setup snippet when the proxy answers 501", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue( - new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }), - ); - renderSection(); - - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Tracing is not enabled"); - expect(card).toHaveTextContent("type: clickhouse"); - expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); - expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); - expect(card).not.toHaveTextContent(/langsmith/i); - expect(card).toHaveTextContent("ClickHouse and proxy setup"); - expect(card).toHaveTextContent("Ask your proxy administrator"); - }); - - it("shows the waiting guide when tracing is on but no runs have arrived", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - renderSection(); - - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Connect your agent"); - expect(card).toHaveTextContent("Waiting for your first trace"); - expect(card).not.toHaveTextContent("store: clickhouse"); - }); - - it("keeps the trace list available when traces exist outside the current time window", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - vi.mocked(apiClient.get).mockResolvedValue(traceList); - renderSection(); - expect(await screen.findByText("No runs match these filters.")).toBeVisible(); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - expect(apiClient.get).toHaveBeenCalledWith("/v1/traces", { accessToken: "sk-test", query: { start_ms: 0 } }); - }); - - it("checks proxy readiness, waits for an agent, and confirms receipt using actual query results", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); - renderSection(); - const checkSetup = await screen.findByRole("button", { name: "Check setup" }); - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - fireEvent.click(checkSetup); - expect(await screen.findByRole("heading", { name: "Connect your agent" })).toBeVisible(); - expect(screen.getByText("Waiting for your first trace")).toBeVisible(); - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - fireEvent.click(screen.getByRole("button", { name: "Check for traces" })); - expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length); - expect(screen.getByText("Traces received. Select a run to inspect it.")).toBeVisible(); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - }); - - it("keeps setup visible while checking and explains when tracing is still disabled", async () => { - const failure = new ApiError("Tracing is not enabled", 501, {}); - vi.mocked(agentTraceListCall).mockRejectedValue(failure); - renderSection(); - const checkSetup = await screen.findByRole("button", { name: "Check setup" }); - let rejectCheck: (error: Error) => void = () => {}; - vi.mocked(agentTraceListCall).mockImplementationOnce( - () => - new Promise((_, reject) => { - rejectCheck = reject; - }), - ); - fireEvent.click(checkSetup); - expect(await screen.findByRole("button", { name: "Checking…" })).toBeDisabled(); - expect(screen.getByRole("heading", { name: "Enable tracing" })).toBeVisible(); - await act(async () => rejectCheck(failure)); - expect(await screen.findByText(/Tracing is still unavailable/)).toBeVisible(); - expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); - }); - - it("keeps received traces and the open drawer visible during subsequent fetches", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); - renderSection(); - const checkSetup = await screen.findByRole("button", { name: "Check setup" }); - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - fireEvent.click(checkSetup); - const rows = await screen.findAllByTestId("agent-trace-row"); - fireEvent.click(rows[0]); - const drawer = screen.getByRole("complementary", { name: "Trace details" }); - - let finishRefresh: (page: TracePage) => void = () => {}; - vi.mocked(agentTraceListCall).mockImplementationOnce( - () => - new Promise((resolve) => { - finishRefresh = resolve; - }), - ); - await act(async () => { - void testQueryClient.invalidateQueries({ queryKey: ["agentTraces"] }); - }); - expect(await screen.findByText("Updating…")).toBeVisible(); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); - expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - await act(async () => finishRefresh(traceList as TracePage)); - }); - - it("separates a failed history check from the empty list and retries that check", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - vi.mocked(apiClient.get).mockRejectedValue(new ApiError("History unavailable", 503, {})); - renderSection(); - expect(await screen.findByRole("alert")).toHaveTextContent("Could not check earlier traces. History unavailable"); - expect(screen.getByText("No runs match these filters.")).toBeVisible(); - expect(screen.queryByText(/Could not load runs/)).not.toBeInTheDocument(); - - vi.mocked(apiClient.get).mockResolvedValue(traceList); - fireEvent.click(screen.getByRole("button", { name: "Retry trace check" })); - await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); - expect(screen.getByText("No runs match these filters.")).toBeVisible(); - expect(agentTraceListCall).toHaveBeenCalledTimes(1); - }); - - it("treats a proxy without the trace routes (404) like tracing being off", async () => { - vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" })); - renderSection(); - - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Tracing is not enabled"); - expect(card).toHaveTextContent("url: os.environ/CLICKHOUSE_URL"); - }); - - it("lists every run with its input, counts and failed column", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - - const rows = await screen.findAllByTestId("agent-trace-row"); - expect(rows).toHaveLength(runs.length); - const lead = rows.find((row) => row.textContent?.includes("Should we store OTEL agent spans")); - expect(lead).toBeDefined(); - const failed = rows.find((row) => row.textContent?.includes("acme-404")) as HTMLElement; - expect(within(failed).getByLabelText("2 errors")).toBeInTheDocument(); - expect(screen.getByText(`${runs.length} runs`)).toBeInTheDocument(); - expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument(); - expect(within(failed).getByText("—")).toBeInTheDocument(); - }); - - it("shows the spend returned for a run", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ - ...(traceList as TracePage), - data: [{ ...runs[0], spend: 0.025 }], - }); - renderSection(); - - const row = await screen.findByTestId("agent-trace-row"); - expect(within(row).getByText("$0.03")).toBeInTheDocument(); - }); - - it("filters by input text and by trace id", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - await screen.findAllByTestId("agent-trace-row"); - - const search = screen.getByLabelText("Search runs"); - fireEvent.change(search, { target: { value: "acme-404" } }); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); - - const lead = runs.find((r) => r.name === "research_lead") as TraceSummary; - fireEvent.change(search, { target: { value: lead.trace_id.slice(0, 10) } }); - const rows = screen.getAllByTestId("agent-trace-row"); - expect(rows).toHaveLength(1); - expect(rows[0]).toHaveTextContent("Should we store OTEL agent spans"); - }); - - it("uses recorded agent names for the column and filter even when services are shared", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ - ...(traceList as TracePage), - data: [ - ...runs.slice(1).map((run) => ({ ...run, service: "shared-app", agent_names: ["research-agent"] })), - { ...runs[0], service: "shared-app", agent_names: ["billing-agent", "review-agent"] }, - ], - }); - const user = userEvent.setup(); - renderSection(); - await screen.findAllByTestId("agent-trace-row"); - - expect(screen.getByRole("columnheader", { name: "Agent" })).toBeInTheDocument(); - const agentFilter = screen.getByRole("combobox", { name: "Filter by agent" }); - expect(agentFilter).toHaveTextContent("All agents"); - - await chooseSelectOption(user, agentFilter, "billing-agent"); - const rows = screen.getAllByTestId("agent-trace-row"); - expect(rows).toHaveLength(1); - expect(rows[0]).toHaveTextContent("billing-agent"); - expect(rows[0]).not.toHaveTextContent("shared-app"); - - await chooseSelectOption(user, agentFilter, "review-agent"); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(1); - - await chooseSelectOption(user, agentFilter, "All agents"); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); - }); - - it("shows each run's agent name with the logo of the SDK that produced it", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue({ - ...(traceList as TracePage), - data: [ - { ...runs[0], agent_names: ["research-bot"], frameworks: ["claude-agent-sdk", "claude-code"] }, - { ...runs[1], agent_names: [], frameworks: ["claude-code"] }, - { ...runs[2], frameworks: [] }, - ], - }); - renderSection(); - const [sdkRun, cliRun, plainRun] = await screen.findAllByTestId("agent-trace-row"); - const agentCell = (row: HTMLElement) => within(row).getAllByRole("cell")[1]; - - expect(agentCell(sdkRun)).toHaveTextContent(/^research-bot$/); - expect(agentCell(sdkRun)).toHaveAttribute("title", "research-bot · Claude Agent SDK"); - expect(within(sdkRun).getByRole("img", { name: "Claude Agent SDK logo", hidden: true })).toHaveAttribute( - "src", - expect.stringContaining("anthropic.svg"), - ); - expect(agentCell(cliRun)).toHaveTextContent(/^Claude Code$/); - expect(within(plainRun).queryByRole("img", { hidden: true })).not.toBeInTheDocument(); - expect(agentCell(plainRun)).toHaveTextContent((runs[2].agent_names ?? [runs[2].service]).join(", ")); - }); - +describe("filterRuns", () => { it("status filter 'Failed' keeps only runs with errors", () => { const failed = filterRuns(runs, "", "all", "error"); expect(failed.length).toBeGreaterThan(0); @@ -333,181 +14,4 @@ describe("AgentTracesSection", () => { expect(ok.every((r) => r.error_count === 0)).toBe(true); expect(failed.length + ok.length).toBe(runs.length); }); - - it("opens a run in a side drawer over the list and swaps runs without closing it", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - const rows = await screen.findAllByTestId("agent-trace-row"); - - fireEvent.click(rows[0]); - const drawer = screen.getByRole("complementary", { name: "Trace details" }); - expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); - expect(screen.getByTestId("runs-table")).toBeInTheDocument(); - expect(rows[0]).toHaveAttribute("aria-selected", "true"); - - fireEvent.click(rows[1]); - expect(screen.getByRole("complementary", { name: "Trace details" })).toBe(drawer); - expect(within(drawer).getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); - expect(rows[1]).toHaveAttribute("aria-selected", "true"); - expect(rows[0]).toHaveAttribute("aria-selected", "false"); - }); - - it("closes the drawer when the open row is clicked again or Escape is pressed", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - const rows = await screen.findAllByTestId("agent-trace-row"); - - fireEvent.click(rows[0]); - fireEvent.click(rows[0]); - expect(rows[0]).toHaveAttribute("aria-selected", "false"); - - fireEvent.click(rows[1]); - fireEvent.keyDown(window, { key: "Escape" }); - expect(rows[1]).toHaveAttribute("aria-selected", "false"); - }); - - it("moves to the next and previous run with j / k and the header arrows", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderSection(); - const rows = await screen.findAllByTestId("agent-trace-row"); - - fireEvent.click(rows[0]); - fireEvent.keyDown(window, { key: "j" }); - expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); - fireEvent.keyDown(window, { key: "k" }); - expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[0].trace_id}`); - expect(screen.getByRole("button", { name: "Previous trace (K)" })).toBeDisabled(); - fireEvent.click(screen.getByRole("button", { name: "Next trace (J)" })); - expect(screen.getByTestId("run-view")).toHaveTextContent(`run ${runs[1].trace_id}`); - }); - - it("plots every loaded run on the timeline", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWindowed(); - await screen.findAllByTestId("agent-trace-row"); - - expect(screen.getByTestId("traces-timeline")).toBeInTheDocument(); - const counts = bucketRunCounts(); - expect(counts).toHaveLength(60); - expect(counts.reduce((a, b) => a + b, 0)).toBe(runs.length); - }); - - it("zooms by dragging, resizes and pans the bracket, and clears with Esc", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWindowed(); - await screen.findAllByTestId("agent-trace-row"); - const area = screen.getByTestId("timeline-area"); - const x = (bucket: number) => bucket * 10 + 5; - const drag = (target: HTMLElement, from: number, to: number) => { - fireEvent.pointerDown(target, { clientX: x(from), pointerId: 1 }); - fireEvent.pointerMove(area, { clientX: x(to), pointerId: 1 }); - fireEvent.pointerUp(area, { clientX: x(to), pointerId: 1 }); - }; - const rowCount = () => screen.queryAllByTestId("agent-trace-row").length; - const withRuns = bucketRunCounts().flatMap((count, i) => (count > 0 ? [i] : [])); - const first = withRuns[0]; - // The pan below moves a [0, first] bracket to the far right; it must end up clear of every run. - expect(first).toBeGreaterThan(1); - expect(first).toBeLessThan(30); - - drag(area, 0, 1); - expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); - expect(rowCount()).toBe(0); - - drag(screen.getByTestId("timeline-handle-hi"), 1, first); - expect(rowCount()).toBeGreaterThan(0); - - drag(screen.getByTestId("timeline-selection"), 1, 1 - first); - expect(rowCount()).toBeGreaterThan(0); - drag(screen.getByTestId("timeline-selection"), 0, 59); - expect(rowCount()).toBe(0); - - fireEvent.keyDown(screen.getByTestId("traces-timeline"), { key: "Escape" }); - expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); - expect(rowCount()).toBe(runs.length); - }); - - it("clears timeline zoom when refreshed", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWindowed({ rangeHours: 24, onRangeHoursChange: () => {}, onLiveChange: () => {} }); - await screen.findAllByTestId("agent-trace-row"); - const area = screen.getByTestId("timeline-area"); - const x = (bucket: number) => bucket * 10 + 5; - - fireEvent.pointerDown(area, { clientX: x(0), pointerId: 1 }); - fireEvent.pointerMove(area, { clientX: x(1), pointerId: 1 }); - fireEvent.pointerUp(area, { clientX: x(1), pointerId: 1 }); - expect(screen.getByTestId("timeline-selection")).toBeInTheDocument(); - expect(screen.queryAllByTestId("agent-trace-row")).toHaveLength(0); - - fireEvent.click(screen.getByRole("button", { name: "Refresh" })); - - expect(screen.queryByTestId("timeline-selection")).not.toBeInTheDocument(); - expect(screen.getAllByTestId("agent-trace-row")).toHaveLength(runs.length); - }); -}); - -describe("AgentTracesPage", () => { - beforeEach(() => { - testQueryClient.clear(); - vi.mocked(agentTraceListCall).mockReset(); - vi.mocked(apiClient.get).mockResolvedValue({ data: [] }); - }); - - it("shows the actual range, switches presets from the popover, and toggles Live", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - const trigger = screen.getByRole("button", { name: "Time range" }); - expect(trigger).toHaveTextContent(/ to /); - expect(screen.getByTestId("traces-timeline")).toHaveTextContent("Total 1d"); - - fireEvent.click(trigger); - fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last 7 days" })); - expect(await screen.findByText("Total 7d")).toBeInTheDocument(); - const last = vi.mocked(agentTraceListCall).mock.calls.at(-1)?.[0]; - expect((last?.endMs ?? 0) - (last?.startMs ?? 0)).toBeGreaterThanOrEqual(7 * 24 * 3600 * 1000 - 60_000); - - const live = screen.getByRole("button", { name: "Live" }); - expect(live).toHaveAttribute("aria-pressed", "true"); - fireEvent.click(live); - expect(live).toHaveAttribute("aria-pressed", "false"); - expect(screen.getByRole("button", { name: "Refresh" })).toBeEnabled(); - }); - - it("refreshes the trace list", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - const callsBeforeRefresh = vi.mocked(agentTraceListCall).mock.calls.length; - fireEvent.click(screen.getByRole("button", { name: "Refresh" })); - - await waitFor(() => expect(vi.mocked(agentTraceListCall).mock.calls.length).toBeGreaterThan(callsBeforeRefresh)); - }); - - it("keeps the time controls on an empty range the user picked, instead of showing onboarding", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - fireEvent.click(screen.getByRole("button", { name: "Time range" })); - fireEvent.click(await screen.findByRole("menuitemradio", { name: "Last hour" })); - - expect(await screen.findByText("No runs match these filters.")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Time range" })).toBeInTheDocument(); - expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); - }); - - it("asks the proxy for the last 24 hours by default", async () => { - vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); - renderWithProviders(); - await screen.findByTestId("runs-table"); - - const { startMs, endMs } = vi.mocked(agentTraceListCall).mock.calls[0][0]; - expect(endMs - startMs).toBeGreaterThanOrEqual(24 * 3600 * 1000 - 60_000); - expect(endMs - startMs).toBeLessThan(24 * 3600 * 1000 + 120_000); - }); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 1b25ca19051..45f1b223088 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import moment from "moment"; import { useMemo, useState } from "react"; @@ -61,6 +62,7 @@ interface AgentTracesSectionProps { onRunOpenChange?: (open: boolean) => void; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; } function useTracingSetup(traces: AgentTracesResult, isActive: boolean, rangeChanged: boolean) { @@ -102,7 +104,9 @@ export function AgentTracesSection({ onRunOpenChange, readOnly = false, canMintTracingKey = false, + onDemo, }: AgentTracesSectionProps) { + const demo = useLensDemo(); const [openTrace, setOpenTrace] = useState(null); const [query, setQuery] = useState(""); const [agent, setAgent] = useState(ALL_AGENTS); @@ -160,10 +164,11 @@ export function AgentTracesSection({ checking: traces.isFetching, }; - if (setup.disabledDetail != null) return ; + if (setup.disabledDetail != null) + return ; // Onboarding only on the first, default view; an empty range the user picked keeps its controls. if (checkHistory && !history.error && history.data === false) - return ; + return ; if (showSetup) { return (
    @@ -185,11 +190,7 @@ export function AgentTracesSection({ return (
    {checkHistory && } - {setup.received && ( -

    - Traces received. Select a run to inspect it. -

    - )} + - + {!demo && ( + + )} {timeControls && ( changeRange(hours, timeControls.onRangeHoursChange)} live={isLiveTail} + showLive={!demo} onLiveChange={timeControls.onLiveChange} onRefresh={() => { setZoom(null); @@ -240,6 +244,16 @@ export function AgentTracesSection({ ); } +function TracesReceived({ received }: { received: boolean }) { + const demo = useLensDemo(); + if (!received || demo) return null; + return ( +

    + Traces received. Select a run to inspect it. +

    + ); +} + function RunsFooter({ count, zoom, diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx index 86ec3007f72..da2175cf0ff 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailContent.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { useQuery, type UseQueryOptions } from "@tanstack/react-query"; import { useState } from "react"; @@ -29,9 +30,15 @@ const errorReason = (headline: string): string => /^([A-Za-z_][\w.]*)\(/.exec(he /** Shared lazy fetch of one span's full input / output / attributes. */ export function useSpanDetail(accessToken: string, traceId: string, spanId: string | null, traceRef?: string) { + const demo = useLensDemo(); const queryOptions: UseQueryOptions = { queryKey: ["agentTraceSpan", traceId, traceRef, spanId, accessToken], - queryFn: () => agentTraceSpanCall(accessToken, traceId, spanId as string, traceRef), + queryFn: () => + demo + ? demo.client.get( + `/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId as string)}`, + ) + : agentTraceSpanCall(accessToken, traceId, spanId as string, traceRef), enabled: spanId !== null, staleTime: Infinity, }; @@ -146,11 +153,17 @@ interface DetailContentProps { } function DiagnosticContent({ accessToken, traceId, traceRef, span }: DetailContentProps) { + const demo = useLensDemo(); const [opened, setOpened] = useState(false); const [cursor, setCursor] = useState(null); const queryOptions: UseQueryOptions = { queryKey: ["agentTraceSpanError", traceId, traceRef, span.span_id, accessToken, cursor], - queryFn: () => agentTraceSpanErrorCall(accessToken, traceId, span.span_id, { traceRef, cursor }), + queryFn: () => + demo + ? demo.client.get( + `/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(span.span_id)}/error`, + ) + : agentTraceSpanErrorCall(accessToken, traceId, span.span_id, { traceRef, cursor }), enabled: opened, staleTime: Infinity, gcTime: 0, diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx index 985de97cdc9..aa777b9861e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/DetailPane.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { PanelRightClose } from "lucide-react"; import { useState } from "react"; @@ -104,6 +105,7 @@ function SpanPane({ onClose: () => void; }) { const [tab, setTab] = useState("content"); + const demo = useLensDemo(); const traceId = trace.summary.trace_id; const detailQuery = useSpanDetail( accessToken, @@ -162,9 +164,13 @@ function SpanPane({
    @@ -186,6 +192,7 @@ function GroupMetric({ label, value }: { label: string; value: string }) { /** ×N group: rollup of every invocation plus the first failure's message. */ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; onClose: () => void }) { + const demo = useLensDemo(); const tokens = row.members.reduce((sum, m) => sum + m.input_tokens + m.output_tokens, 0); const firstFailure = row.members.find((m) => m.status === "error" && m.error); return ( @@ -222,7 +229,11 @@ function GroupPane({ trace, row, onClose }: { trace: Trace; row: GroupRowData; o
    diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx index 03d950a7725..c9e25c2e033 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TimeRangeControls.tsx @@ -34,6 +34,7 @@ interface TimeRangeControlsProps { rangeHours: number; onRangeHoursChange: (hours: number) => void; live: boolean; + showLive?: boolean; onLiveChange: (live: boolean) => void; onRefresh: () => void; refreshing: boolean; @@ -45,6 +46,7 @@ export function TimeRangeControls({ rangeHours, onRangeHoursChange, live, + showLive = true, onLiveChange, onRefresh, refreshing, @@ -84,18 +86,20 @@ export function TimeRangeControls({ - + {showLive && ( + + )}
    ); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx index a991c771a56..435ea51e8d2 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx @@ -1,4 +1,5 @@ "use client"; +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { useQuery } from "@tanstack/react-query"; import { ArrowLeft, Check, Copy } from "lucide-react"; @@ -74,6 +75,7 @@ const toggle = (set: ReadonlySet, id: string): Set => { }; function CopyForAgent({ traceId, traceRef }: { traceId: string; traceRef?: string }) { + const demo = useLensDemo(); const [copied, setCopied] = useState(false); useEffect(() => { if (!copied) return; @@ -86,11 +88,16 @@ function CopyForAgent({ traceId, traceRef }: { traceId: string; traceRef?: strin size="xs" className="h-7 shrink-0 gap-1.5 rounded-md text-[12px] shadow-none" onClick={async () => - setCopied(await copyToClipboard(agentHandoffText(traceId, null, traceRef), "Command copied")) + setCopied( + await copyToClipboard( + demo ? demo.copyTrace(traceId) : agentHandoffText(traceId, null, traceRef), + demo ? "Trace copied" : "Command copied", + ), + ) } > {copied ? : } - {copied ? "Command copied" : "Copy for agent"} + {copied ? "Copied" : "Copy for agent"} ); } @@ -277,9 +284,13 @@ interface RunViewProps { /** One agent run: header with totals and "Copy for agent", span tree on the left, span details on the right. */ export function RunView({ traceId, traceRef, initialSpanId, accessToken, onBack, embedded = false }: RunViewProps) { + const demo = useLensDemo(); const traceQuery = useQuery({ queryKey: ["agentTrace", traceId, traceRef, accessToken], - queryFn: () => agentTraceCall(accessToken, traceId, traceRef), + queryFn: () => + demo + ? demo.client.get(`/v1/traces/${encodeURIComponent(traceId)}`) + : agentTraceCall(accessToken, traceId, traceRef), staleTime: 30_000, }); const trace = traceQuery.data; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx deleted file mode 100644 index c4fd04602ac..00000000000 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePreview.tsx +++ /dev/null @@ -1,70 +0,0 @@ -"use client"; - -import { cn } from "@/lib/cva.config"; - -import previewTrace from "./previewTrace.json"; -import { SpanIcon } from "./SpanIcon"; -import type { SpanType } from "./traceTypes"; -import { fmtMs } from "./traceUtils"; - -interface PreviewRow { - id: string; - name: string; - type: SpanType; - model: string | null; - depth: number; - start_offset_ms: number; - duration_ms: number; - error: boolean; -} - -const preview = previewTrace as { - name: string; - input_preview: string; - span_count: number; - duration_ms: number; - rows: PreviewRow[]; -}; - -export function TracePreview() { - const total = preview.duration_ms; - return ( -
    -
    - - {preview.name} - {preview.input_preview} - - {preview.span_count} steps · {fmtMs(total)} - -
    -
    - {preview.rows.map((row) => ( -
    -
    - - {row.name} -
    -
    -
    -
    - - {fmtMs(row.duration_ms)} - -
    - ))} -
    -
    -
    - ); -} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.integration.test.tsx similarity index 63% rename from ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx rename to ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.integration.test.tsx index 65e5e9229ff..e1d128d59e9 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.integration.test.tsx @@ -12,6 +12,7 @@ import { tracingEnvSnippet, TracingSetupCard, } from "./TracingSetupCard"; +import { FRAMEWORKS } from "./tracingSetupGuides"; import type { Trace } from "./traceTypes"; vi.mock("../../networking", () => ({ @@ -31,6 +32,7 @@ const renderCard = ( onCheck?: () => void; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; } = {}, ) => { const onOpenTrace = vi.fn(); @@ -38,6 +40,7 @@ const renderCard = ( vi.clearAllMocks()); describe("TracingSetupCard", () => { - it("shows agent connection guidance and labels the example run as sample data", async () => { + it("offers the interactive demo while waiting for the first trace", async () => { const user = userEvent.setup(); - const { card } = renderCard(); + const onDemo = vi.fn(); + const { card } = renderCard({ onDemo }); expect(screen.getByRole("heading", { name: "Connect your agent" })).toBeVisible(); expect(screen.getByText("Tracing enabled")).toBeVisible(); expect(screen.getByText("Waiting for your first trace")).toBeVisible(); - expect(screen.getByTestId("trace-preview")).toBeVisible(); - expect(card).toHaveTextContent("sample data, not your runs"); + await user.click(screen.getByRole("button", { name: "Preview sample" })); + expect(onDemo).toHaveBeenCalledOnce(); + expect(sendOtlpTraceCall).not.toHaveBeenCalled(); expect(card).not.toHaveTextContent("store: clickhouse"); await user.click(screen.getByText("Set up manually")); - expect(screen.getByText(/^export OTEL_EXPORTER_OTLP_ENDPOINT=http:\/\/proxy.test/)).toBeVisible(); + expect(screen.getByText(/^export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=/)).toBeVisible(); expect(card).not.toHaveTextContent(/langsmith/i); }); - it("lists the OTEL endpoints before the framework picker, each copyable", async () => { + it("keeps connection details visible and copies the full trace endpoint", async () => { const user = userEvent.setup(); - const { card } = renderCard(); - const text = card.textContent ?? ""; - expect(text.indexOf("OpenTelemetry (OTEL) endpoints")).toBeLessThan(text.indexOf("Your agent framework")); + renderCard(); + expect(screen.getByRole("combobox", { name: "Your agent framework" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Copy http://proxy.test/v1/traces" })).toBeVisible(); await user.click(screen.getByRole("button", { name: "Copy http://proxy.test/v1/traces" })); expect(copyToClipboard).toHaveBeenLastCalledWith("http://proxy.test/v1/traces"); }); - it("hides the sample preview once traces are arriving", () => { + it("shows connection guidance for another agent without a demo", () => { renderCard({ connected: true }); expect(screen.getByRole("heading", { name: "Connect another agent" })).toBeVisible(); - expect(screen.queryByTestId("trace-preview")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); }); it("builds the coding agent command for the selected framework and keeps both manual installers", async () => { const user = userEvent.setup(); renderCard(); await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "CrewAI"); - const prompt = codingAgentPrompt("http://proxy.test", { - label: "CrewAI", - packages: "crewai openinference-instrumentation-crewai", - }); - const commandText = () => screen.getByText(/^claude |^codex /).textContent; - expect(commandText()).toBe(codingAgentCommand("Claude Code", prompt)); + const prompt = codingAgentPrompt( + "http://proxy.test", + FRAMEWORKS.find((guide) => guide.id === "crewai")!, + "openai/gpt-6-sol", + ); + expect(screen.getByText(/^claude /)).not.toBeVisible(); + await user.click(screen.getByRole("button", { name: "Copy setup command" })); + expect(copyToClipboard).toHaveBeenLastCalledWith(codingAgentCommand("Claude Code", prompt)); await user.click(screen.getByRole("tab", { name: "Codex" })); - expect(commandText()).toBe(codingAgentCommand("Codex", prompt)); + await user.click(screen.getByRole("button", { name: "Copy setup command" })); + expect(copyToClipboard).toHaveBeenLastCalledWith(codingAgentCommand("Codex", prompt)); + await user.click(screen.getByText("View command")); + expect(screen.getByText(/^codex /)).toBeVisible(); + expect(screen.getByText(/^codex /)).toHaveTextContent(codingAgentCommand("Codex", prompt), { + normalizeWhitespace: false, + }); await user.click(screen.getByText("Set up manually")); - expect(screen.getByText(/pip install -U opentelemetry-distro/)).toHaveTextContent( + expect(screen.getByText(/^pip install opentelemetry-distro/)).toHaveTextContent( "crewai openinference-instrumentation-crewai", ); await user.click(screen.getByRole("tab", { name: "uv" })); - expect(screen.getByText(/uv add opentelemetry-distro/)).toHaveTextContent( + expect(screen.getByText(/^uv add opentelemetry-distro/)).toHaveTextContent( "crewai openinference-instrumentation-crewai", ); }); - it("uses npm and a CommonJS-safe TypeScript entrypoint for the Vercel AI SDK", async () => { + it("uses the selected framework's tracing and agent name without asking for a model", async () => { const user = userEvent.setup(); const { card } = renderCard(); await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "Vercel AI SDK"); await user.click(screen.getByText("Set up manually")); - expect(card).toHaveTextContent("npm install ai @ai-sdk/openai-compatible @vercel/otel"); - expect(card).toHaveTextContent("my_agent.ts"); - expect(card).not.toHaveTextContent("opentelemetry-instrument python"); - const quickstart = screen.getByText(/registerOTel\(\{/).textContent ?? ""; - expect(quickstart).toContain("async function main()"); - expect(quickstart.split("\n").filter((line) => /^(const|let) .*= await /.test(line))).toEqual([]); + expect(screen.queryByRole("combobox", { name: "Model" })).not.toBeInTheDocument(); + expect(card).toHaveTextContent("npm install ai @ai-sdk/otel"); + expect(card).toHaveTextContent('const AGENT_NAME = "research_agent"'); + expect(card).toHaveTextContent("functionId: AGENT_NAME"); + expect(card).toHaveTextContent("Use a model configured on this proxy."); + expect(screen.getByText(/^import \{ createOpenAICompatible/)).toHaveTextContent( + 'const model = litellm("openai/gpt-6-sol")', + ); + expect(card).toHaveTextContent('baseURL: "http://proxy.test/v1"'); + }); + + it("keeps plugin model settings and uses a generated tracing key only for tracing", async () => { + const user = userEvent.setup(); + vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET }); + const { card } = renderCard(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "Hermes"); + await user.click(screen.getByText("Set up manually")); + expect(screen.queryByRole("combobox", { name: "Model" })).not.toBeInTheDocument(); + expect(card).toHaveTextContent("Keep your existing model settings"); + await user.click(screen.getByRole("button", { name: "Generate tracing key" })); + await screen.findByText("Your tracing key"); + expect(card).toHaveTextContent("gen_ai.agent.name: research_agent"); + expect(card).toHaveTextContent("endpoint: http://proxy.test/v1/traces"); + expect(card).toHaveTextContent('Authorization: "Bearer ${LITELLM_TRACING_KEY}"'); + expect(card).not.toHaveTextContent(SECRET); }); it("hides the actions a read-only viewer cannot perform", () => { @@ -121,7 +153,7 @@ describe("TracingSetupCard", () => { expect(screen.queryByRole("button", { name: "Send a test trace" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Generate tracing key" })).not.toBeInTheDocument(); expect(card).toHaveTextContent("ask a proxy admin for one"); - expect(card).toHaveTextContent("OpenTelemetry (OTEL) endpoints"); + expect(card).toHaveTextContent("Connection details"); }); it("offers a scoped tracing key only to callers allowed to set key routes", () => { @@ -135,7 +167,7 @@ describe("TracingSetupCard", () => { const user = userEvent.setup(); vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET }); const { card } = renderCard(); - + await user.click(screen.getByText("Set up manually")); await user.click(screen.getByRole("button", { name: "Generate tracing key" })); expect(await screen.findByText("Your tracing key")).toBeVisible(); @@ -193,24 +225,23 @@ describe("TracingSetupCard", () => { }); describe("setup snippets", () => { - it("uses OTLP HTTP with the proxy base URL and reads the key from the environment", () => { + it("uses the instance trace endpoint and keeps tracing and inference keys separate", () => { const env = tracingEnvSnippet("http://proxy.test"); - expect(env).toContain("OTEL_EXPORTER_OTLP_ENDPOINT=http://proxy.test\n"); - expect(env).toContain("OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"); - expect(env).not.toContain("/v1/traces"); + expect(env).toContain('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://proxy.test/v1/traces"'); + expect(env).toContain('OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf"'); expect(env).not.toContain("export LITELLM_API_KEY="); expect(env).toContain("Bearer $LITELLM_API_KEY"); const withKey = tracingEnvSnippet("http://proxy.test", SECRET); expect(withKey).toContain(`export LITELLM_TRACING_KEY=${SECRET}\n`); - expect(withKey).toContain('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_TRACING_KEY"'); - expect(withKey).not.toContain("LITELLM_API_KEY"); + expect(withKey).toContain('OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer $LITELLM_TRACING_KEY"'); + expect(withKey).not.toContain("export LITELLM_API_KEY="); - const prompt = codingAgentPrompt("http://proxy.test", { label: "LangChain", packages: "langchain" }); - expect(prompt).toContain("base_url=http://proxy.test/v1"); - expect(prompt).toContain("opentelemetry-distro opentelemetry-exporter-otlp-proto-http langchain"); - expect(prompt).toContain("OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf"); + const prompt = codingAgentPrompt("http://proxy.test", FRAMEWORKS[0], "openai/gpt-6-sol"); + expect(prompt).toContain('OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://proxy.test/v1/traces"'); + expect(prompt).toContain("Keep the existing model configuration"); + expect(prompt).toContain('AGENT_NAME = "research_agent"'); + expect(prompt).toContain("name=AGENT_NAME"); expect(prompt).toContain("Lens > Traces"); - expect(prompt).not.toMatch(/langsmith/i); }); it("builds a shell-safe command for each coding agent", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx index de776da0617..7c6155446ea 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -4,31 +4,23 @@ import { ArrowRight, ArrowUpRight, Check, Copy, KeyRound, Loader2, Send } from " import { useState } from "react"; import { cn } from "@/lib/cva.config"; +import { LensPreviewButton } from "@/components/lens/LensPreviewButton"; import { Button } from "@/components/ui/button"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { copyToClipboard } from "@/utils/dataUtils"; import anthropicLogo from "../../../../public/assets/logos/anthropic.svg"; -import crewaiLogo from "../../../../public/assets/logos/crewai-color.svg"; -import langchainLogo from "../../../../public/assets/logos/langchain.svg"; -import langgraphLogo from "../../../../public/assets/logos/langgraph-color.svg"; -import llamaindexLogo from "../../../../public/assets/logos/llamaindex-color.svg"; -import openaiAgentsLogo from "../../../../public/assets/logos/openai-agents.svg"; import openaiLogo from "../../../../public/assets/logos/openai_small.svg"; import otelLogo from "../../../../public/assets/logos/opentelemetry.svg"; -import pydanticAiLogo from "../../../../public/assets/logos/pydantic-ai-color.svg"; -import vercelLogo from "../../../../public/assets/logos/vercel.svg"; import { agentTraceCall, apiClient, getProxyBaseUrl, sendOtlpTraceCall } from "../../networking"; import { ActiveDot } from "./ActiveDot"; import { sampleTraceExport } from "./sampleTrace"; -import { TracePreview } from "./TracePreview"; +import { FRAMEWORKS, frameworkSnippet, type FrameworkGuide } from "./tracingSetupGuides"; import type { TraceSummary } from "./traceTypes"; const COPIED_RESET_MS = 1500; const DOCS_URL = "https://docs.litellm.ai/docs/proxy/lens"; -const OTEL_BASE_PACKAGES = "opentelemetry-distro opentelemetry-exporter-otlp-proto-http"; -const PY_RUN_SNIPPET = "opentelemetry-instrument python my_agent.py"; -const TS_RUN_SNIPPET = "npx tsx my_agent.ts"; +const EXAMPLE_MODEL = "openai/gpt-6-sol"; const SAMPLE_TRACE_POLL_MS = 1000; export const TRACING_KEY_REQUEST = { key_alias: "Agent tracing", @@ -41,177 +33,36 @@ type Installer = "pip" | "uv"; type CodingAgent = "Claude Code" | "Codex"; const PY_INSTALL: Record string> = { - pip: (packages) => `pip install -U ${packages}`, + pip: (packages) => `pip install ${packages}`, uv: (packages) => `uv add ${packages}`, }; -interface FrameworkGuide { - id: string; - label: string; - logo: string; - packages: string; - quickstart: string; - typescript?: boolean; -} - -const FRAMEWORKS: readonly FrameworkGuide[] = [ - { - id: "deep-agents", - label: "Deep Agents", - logo: langgraphLogo.src, - packages: "deepagents langchain-openai openinference-instrumentation-langchain", - quickstart: `from deepagents import create_deep_agent -from langchain_openai import ChatOpenAI - -llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = create_deep_agent(model=llm, tools=[], system_prompt="You are a careful researcher.") -agent.invoke({"messages": [{"role": "user", "content": "What is LiteLLM?"}]})`, - }, - { - id: "vercel-ai-sdk", - label: "Vercel AI SDK", - logo: vercelLogo.src, - typescript: true, - packages: "ai @ai-sdk/openai-compatible @vercel/otel @opentelemetry/api", - quickstart: `import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; -import { registerOTel } from "@vercel/otel"; -import { generateText } from "ai"; - -registerOTel({ serviceName: process.env.OTEL_SERVICE_NAME ?? "my-agent" }); - -const litellm = createOpenAICompatible({ name: "litellm", baseURL: "{PROXY}/v1", apiKey: process.env.LITELLM_API_KEY }); - -async function main() { - const { text } = await generateText({ - model: litellm("claude-sonnet-4-5"), - prompt: "What is LiteLLM?", - experimental_telemetry: { isEnabled: true, functionId: "my_agent" }, - }); - console.log(text); -} - -main();`, - }, - { - id: "langgraph", - label: "LangGraph", - logo: langgraphLogo.src, - packages: "langgraph langchain-openai openinference-instrumentation-langchain", - quickstart: `from langchain.agents import create_agent -from langchain_openai import ChatOpenAI - -llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = create_agent(model=llm, tools=[], name="my_agent") -agent.invoke({"messages": [{"role": "user", "content": "What is LiteLLM?"}]})`, - }, - { - id: "langchain", - label: "LangChain", - logo: langchainLogo.src, - packages: "langchain langchain-openai openinference-instrumentation-langchain", - quickstart: `from langchain_openai import ChatOpenAI - -llm = ChatOpenAI(model="claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -llm.invoke("What is LiteLLM?")`, - }, - { - id: "openai-agents", - label: "OpenAI Agents SDK", - logo: openaiAgentsLogo.src, - packages: "openai-agents openinference-instrumentation-openai-agents", - quickstart: `from agents import Agent, OpenAIChatCompletionsModel, Runner -from openai import AsyncOpenAI - -client = AsyncOpenAI(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = Agent(name="my_agent", model=OpenAIChatCompletionsModel(model="claude-sonnet-4-5", openai_client=client)) -print(Runner.run_sync(agent, "What is LiteLLM?").final_output)`, - }, - { - id: "crewai", - label: "CrewAI", - logo: crewaiLogo.src, - packages: "crewai openinference-instrumentation-crewai", - quickstart: `from crewai import LLM, Agent, Crew, Task - -llm = LLM(model="openai/claude-sonnet-4-5", base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = Agent(role="Researcher", goal="Answer questions", backstory="", llm=llm) -task = Task(description="What is LiteLLM?", expected_output="A short answer", agent=agent) -Crew(agents=[agent], tasks=[task]).kickoff()`, - }, - { - id: "pydantic-ai", - label: "Pydantic AI", - logo: pydanticAiLogo.src, - packages: "pydantic-ai openinference-instrumentation-pydantic-ai", - quickstart: `from pydantic_ai import Agent -from pydantic_ai.models.openai import OpenAIModel -from pydantic_ai.providers.openai import OpenAIProvider - -provider = OpenAIProvider(base_url="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"]) -agent = Agent(OpenAIModel("claude-sonnet-4-5", provider=provider), name="my_agent", instrument=True) -print(agent.run_sync("What is LiteLLM?").output)`, - }, - { - id: "llamaindex", - label: "LlamaIndex", - logo: llamaindexLogo.src, - packages: "llama-index llama-index-llms-openai-like openinference-instrumentation-llama-index", - quickstart: `from llama_index.llms.openai_like import OpenAILike - -llm = OpenAILike(model="claude-sonnet-4-5", api_base="{PROXY}/v1", api_key=os.environ["LITELLM_API_KEY"], is_chat_model=True) -print(llm.complete("What is LiteLLM?"))`, - }, - { - id: "otel", - label: "OpenTelemetry", - logo: otelLogo.src, - packages: "", - quickstart: `# Any OTEL SDK works. Use the gen_ai.* semantic conventions: -# gen_ai.operation.name, gen_ai.agent.name, gen_ai.response.id, gen_ai.usage.* -from opentelemetry import trace - -tracer = trace.get_tracer("my-agent") -attrs = {"gen_ai.operation.name": "invoke_agent", "gen_ai.agent.name": "my_agent"} -with tracer.start_as_current_span("my_agent", attributes=attrs): - ...`, - }, -]; - -const installPackages = (guide: Pick): string => - guide.typescript ? guide.packages : [OTEL_BASE_PACKAGES, guide.packages].filter(Boolean).join(" "); - -/** The endpoint is the proxy base URL: OTLP exporters append /v1/traces themselves. */ export const tracingEnvSnippet = (proxyUrl: string, tracingKey: string | null = null): string => [ ...(tracingKey ? [`export LITELLM_TRACING_KEY=${tracingKey}`] : []), - `export OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`, - "export OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf", - `export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $${tracingKey ? "LITELLM_TRACING_KEY" : "LITELLM_API_KEY"}"`, - "export OTEL_SERVICE_NAME=my-agent", + `export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${proxyUrl}/v1/traces"`, + `export OTEL_EXPORTER_OTLP_TRACES_HEADERS="Authorization=Bearer $${tracingKey ? "LITELLM_TRACING_KEY" : "LITELLM_API_KEY"}"`, + 'export OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf"', + 'export OTEL_METRICS_EXPORTER="none"', + 'export OTEL_LOGS_EXPORTER="none"', ].join("\n"); -export const codingAgentPrompt = ( - proxyUrl: string, - guide: Pick, -): string => +export const codingAgentPrompt = (proxyUrl: string, guide: FrameworkGuide, model: string): string => [ `Send this ${guide.label} project's OpenTelemetry traces to LiteLLM.`, - "", - `1. Add these dependencies: ${installPackages(guide)}`, - "2. Set these env vars wherever the project loads config (.env, settings, deployment manifests):", - ` OTEL_EXPORTER_OTLP_ENDPOINT=${proxyUrl}`, - " OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf", - ' OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer $LITELLM_API_KEY"', - " OTEL_SERVICE_NAME=", - guide.typescript - ? "3. Call registerOTel() from @vercel/otel at startup and pass experimental_telemetry: { isEnabled: true } to every AI SDK call." - : "3. Start the app through OTEL auto-instrumentation: opentelemetry-instrument .", - `4. Point every LLM client at LiteLLM: base_url=${proxyUrl}/v1, api key from LITELLM_API_KEY.`, - "5. Give each agent and subagent a name so runs are easy to read.", - "6. Run the agent once and confirm the run shows up in the LiteLLM UI under Lens > Traces.", - "", - "Never hardcode the key. Read it from LITELLM_API_KEY.", - ].join("\n"); + "Keep the existing model configuration, authentication, and application behavior. Never hardcode a key; read it from LITELLM_API_KEY.", + "Set the trace destination wherever this project loads environment variables:", + tracingEnvSnippet(proxyUrl), + guide.install ?? `Install and enable the ${guide.plugin?.label}: ${guide.plugin?.url}`, + guide.plugin?.instruction ?? + "Initialize OpenTelemetry before creating the agent. If the app already configures a tracer provider, keep it and point its exporter at the destination above instead.", + "Adapt this example to the existing application, replacing research_agent with the agent's name:", + frameworkSnippet(guide, proxyUrl, model), + guide.note ?? "", + "Run the agent once and confirm its named run appears in Lens > Traces.", + ] + .filter(Boolean) + .join("\n\n"); const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\\''")}'`; @@ -222,10 +73,9 @@ export const maskSecret = (secret: string): string => secret.length > 10 ? `${secret.slice(0, 5)}${"•".repeat(16)}${secret.slice(-4)}` : "•".repeat(secret.length); export const otlpEndpoints = (proxyUrl: string): readonly (readonly [string, string, boolean])[] => [ - ["Traces endpoint (POST)", `${proxyUrl}/v1/traces`, true], - ["OTEL_EXPORTER_OTLP_ENDPOINT", proxyUrl, true], + ["Traces endpoint", `${proxyUrl}/v1/traces`, true], ["Auth header", "Authorization: Bearer ", true], - ["Protocol", "OTLP/HTTP, protobuf or JSON (gRPC not supported)", false], + ["Protocol", "OTLP/HTTP (protobuf or JSON)", false], ]; export const PROXY_CONFIG_SNIPPET = [ @@ -270,8 +120,8 @@ function CodeBlock({ 
             {display}
@@ -322,7 +172,7 @@ function LineTabs({
 function Step({ title, children }: { title: React.ReactNode; children: React.ReactNode }) {
   return (
     
    
-      
    {title}
    
+      
    {title}
    
       {children}
     
    
   );
@@ -395,7 +245,7 @@ function SendTestTrace({
   const busy = state.kind === "sending" || state.kind === "waiting";
   return (
     
    
-      
-        {!connected && (
-          
    
-            Waiting for your first trace
-          
    
-        )}
       
    
       {missingAfterCheck && (
         
    
@@ -442,7 +290,7 @@ function TraceReceipt({
           logs for export errors.
         
    
       )}
-    
+
); } @@ -526,19 +374,19 @@ function EndpointValue({ value }: { value: string }) { ); } -function Endpoints({ proxyUrl }: { proxyUrl: string }) { +function Endpoints({ proxyUrl, children }: { proxyUrl: string; children?: React.ReactNode }) { return (
-

- - OpenTelemetry (OTEL) endpoints -

-

- Point any OpenTelemetry exporter here. The framework guides below set these for you. -

-
+
+

+ + Connection details +

+ {children} +
+
{otlpEndpoints(proxyUrl).map(([label, value, copyable]) => ( -
+
{label}
{copyable ? : {value}} @@ -574,7 +422,7 @@ interface ConnectAgentProps { function EnableTracing({ checked, checking, onCheck }: { checked: boolean; checking: boolean; onCheck: () => void }) { return ( <> - +

Set your ClickHouse URL, add this to config.yaml, then restart the proxy. Ask your proxy administrator if you don’t manage this deployment. @@ -594,14 +442,68 @@ function EnableTracing({ checked, checking, onCheck }: { checked: boolean; check Tracing is still unavailable. Check that the configuration was applied to this proxy and it has restarted.

)} - +
+ +
); } +function CodingAgentSetup({ proxyUrl, guide, model }: { proxyUrl: string; guide: FrameworkGuide; model: string }) { + const [codingAgent, setCodingAgent] = useState("Claude Code"); + const [copied, setCopied] = useState(null); + const command = codingAgentCommand(codingAgent, codingAgentPrompt(proxyUrl, guide, model)); + const copy = async () => { + if (await copyToClipboard(command)) { + setCopied(command); + window.setTimeout(() => setCopied(null), COPIED_RESET_MS); + } + }; + return ( +
+

+ Connect your project +

+
+
+ { + setCodingAgent(value); + setCopied(null); + }} + logos={CODING_AGENT_LOGOS} + /> +
+
+

+ Run the setup command in your agent’s project. It uses your LITELLM_API_KEY + . +

+ +
+ View command + 
+              {command}
+
+
+
+
+
+ ); +} + function ConnectAgent({ accessToken, onOpenTrace, @@ -615,27 +517,20 @@ function ConnectAgent({ const proxyUrl = getProxyBaseUrl().replace(/\/$/, ""); const [framework, setFramework] = useState(FRAMEWORKS[0].id); const [installer, setInstaller] = useState("pip"); - const [codingAgent, setCodingAgent] = useState("Claude Code"); const [tracingKey, setTracingKey] = useState(null); const guide = FRAMEWORKS.find((f) => f.id === framework) ?? FRAMEWORKS[0]; - const packages = installPackages(guide); - const install = guide.typescript ? `npm install ${packages}` : PY_INSTALL[installer](packages); - const quickstart = guide.quickstart.replace("{PROXY}", proxyUrl); + const install = guide.install?.startsWith("pip install ") + ? PY_INSTALL[installer](guide.install.slice("pip install ".length)) + : guide.install; + const model = EXAMPLE_MODEL; + const quickstart = frameworkSnippet(guide, proxyUrl, model, !!tracingKey); return ( <> - {!readOnly && ( - -

- Send a small sample run (an agent, an LLM call and a tool call) to confirm tracing works end to end. -

- - - )} - - - + + {!readOnly && } +
-
- - {canMintTracingKey && !readOnly ? ( - - ) : ( -

- Use any LiteLLM virtual key you already have, or ask a proxy admin for one. -

- )} - - - - Let Claude Code or - Codex connect it - - } - > -

- Run this in your agent’s project. It starts your coding agent with the setup task and reads the key from - LITELLM_API_KEY. -

- - } - /> - - -
- Set up manually - - npm - ) : ( - - ) - } - /> + +
+ Set up manually + + {canMintTracingKey && !readOnly ? ( + + ) : ( +

+ Use any LiteLLM virtual key you already have, or ask a proxy admin for one. +

+ )} + {install && ( + + npm + ) : ( + + ) + } + /> + + )} + {guide.plugin && ( + +

+ Install and enable the{" "} + + {guide.plugin.label} + + . {guide.plugin.instruction} +

+ + )} +

+ {tracingKey ? ( + <> + Tracing uses the key above. Set LITELLM_API_KEY to a key with model access. + + ) : ( + <> + Set LITELLM_API_KEY to your LiteLLM key. + + )} +

Shell} + wrap /> -

- Replace the example model with a model configured on your proxy. +

+ {guide.existingModel ? "Keep your existing model settings. " : "Save and run this example. "} + Replace research_agent with your agent’s name. + {!guide.existingModel && " Use a model configured on this proxy."}

- {guide.typescript ? "my_agent.ts" : "my_agent.py"}} - /> -
- Shell} /> -
+ {guide.fileName}} wrap /> + {guide.note &&

{guide.note}

} + {!guide.existingModel && ( +
+ Shell} + /> +
+ )} + + View in docs
- - {!connected && ( -
-

- Example run (sample data, not your runs) -

- -
- )} ); } @@ -755,6 +654,7 @@ export function TracingSetupCard({ checking = false, readOnly = false, canMintTracingKey = false, + onDemo, }: { detail: string | null; accessToken: string; @@ -764,6 +664,7 @@ export function TracingSetupCard({ checking?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; + onDemo?: () => void; }) { const [checked, setChecked] = useState(false); const enabled = detail === null; @@ -773,9 +674,18 @@ export function TracingSetupCard({ }; return ( -
-
+
+ {onDemo && } +

{setupTitle(enabled, connected)}

+ + {enabled ? ( + + ) : ( + -

- {enabled && } - {enabled ? "Tracing enabled" : "Tracing is not enabled"} -

{enabled ? ( = { + MODEL: JSON.stringify(model), + OPENAI_MODEL: JSON.stringify(`openai/${model}`), + BASE_URL: JSON.stringify(`${proxyUrl}/v1`), + PROXY_URL: JSON.stringify(proxyUrl), + TRACE_URL: `${proxyUrl}/v1/traces`, + }; + const code = guide.quickstart.replace( + /\{(MODEL|OPENAI_MODEL|BASE_URL|PROXY_URL|TRACE_URL)\}/g, + (_, name: string) => values[name], + ); + return tracingKey && guide.existingModel ? code.replaceAll("${LITELLM_API_KEY}", "${LITELLM_TRACING_KEY}") : code; +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts index a50d57409de..46f4b4bed71 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts @@ -1,3 +1,4 @@ +import { useLensDemo } from "@/components/lens/LensDemoContext"; import { useInfiniteQuery, useQuery, type UseQueryOptions } from "@tanstack/react-query"; import moment from "moment"; import { useMemo } from "react"; @@ -65,6 +66,7 @@ export function useAgentTraces({ isLiveTail, enabled, }: UseAgentTracesOptions): AgentTracesResult { + const demo = useLensDemo(); const fetchPage = (pageParam: unknown): Promise => { const nowMs = Date.now(); const listOptions: Parameters[0] = { @@ -73,7 +75,11 @@ export function useAgentTraces({ endMs: isCustomDate ? moment(endTime).valueOf() : nowMs, cursor: pageParam as string | null, }; - return agentTraceListCall(listOptions); + return demo + ? demo.client.get("/v1/traces", { + query: { start_ms: listOptions.startMs, end_ms: listOptions.endMs, cursor: listOptions.cursor }, + }) + : agentTraceListCall(listOptions); }; const queryOptions: Parameters>[0] = { queryKey: ["agentTraces", accessToken, startTime, endTime, isCustomDate], @@ -106,9 +112,11 @@ export function useAgentTraces({ } export function useTraceAvailability(accessToken: string, enabled: boolean) { + const demo = useLensDemo(); + const client = demo?.client ?? apiClient; const options: UseQueryOptions = { queryKey: ["trace-availability", accessToken], - queryFn: () => apiClient.get("/v1/traces", { accessToken, query: { start_ms: 0 } }), + queryFn: () => client.get("/v1/traces", { accessToken, query: { start_ms: 0 } }), select: (page: TracePage) => page.data.length > 0, enabled, retry: false,