From cc42a352cb3642f07e04cb9c8aaf66d349cf5942 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Thu, 1 Oct 2026 17:21:41 -0700 Subject: [PATCH] feat(lens): simplify setup and investigation workflow (#44089) * feat(lens): simplify investigation setup and results * fix(lens): pin worker with actionable failure diagnostics * fix(lens): focus worker success on starting an investigation * feat(lens): simplify investigation setup and worker defaults * fix(lens): remove setup repetition and label billing access * fix(lens): finish agent selection and setup readiness * fix(lens): handle unavailable setup dependencies and restore UI build --- deploy/lens/compose.yaml | 2 +- .../crates/traces/query/lens_agents.sql | 6 + .../crates/traces/query/lens_availability.sql | 8 + .../crates/traces/query/lens_sample.sql | 2 + litellm-rust/crates/traces/src/sql.rs | 6 + .../crates/traces/tests/migrations.rs | 108 +++ litellm/proxy/lens/endpoints.py | 20 +- litellm/proxy/lens/models.py | 7 +- litellm/proxy/lens/sources.py | 22 + litellm/proxy/lens/worker.py | 43 +- litellm/rust_bridge/traces.py | 6 + tests/integration/spend/test_lens_billing.py | 4 +- tests/unit/proxy/lens/test_endpoints.py | 17 +- tests/unit/proxy/lens/test_sources.py | 45 ++ tests/unit/proxy/lens/test_state.py | 6 +- tests/unit/proxy/lens/test_worker.py | 34 +- .../lens/_components/ActivityScope.tsx | 532 ++++++++------ .../AnalysisKey.integration.test.tsx | 37 +- .../lens/_components/AnalysisKey.tsx | 115 ++- .../lens/_components/AnalysisKeyDetails.tsx | 99 +++ .../lens/_components/DurationInput.tsx | 31 +- .../lens/_components/LensFinding.tsx | 146 ++++ .../lens/_components/LensOverview.tsx | 270 +++++++ .../lens/_components/LensProgress.tsx | 2 +- .../(dashboard)/lens/_components/LensRuns.tsx | 106 +-- .../LensSetup.integration.test.tsx | 347 ++++++++- .../lens/_components/LensSetup.tsx | 585 ++++++++------- .../_components/LensView.integration.test.tsx | 153 +++- .../(dashboard)/lens/_components/LensView.tsx | 687 +++++++++--------- .../lens/_components/LensWelcome.tsx | 225 +++--- .../lens/_components/TracePanel.tsx | 2 +- .../WorkerSetup.integration.test.tsx | 93 ++- .../lens/_components/WorkerSetup.tsx | 323 +++++--- .../lens/_components/lensData.test.ts | 24 +- .../(dashboard)/lens/_components/lensData.ts | 40 +- .../src/app/(dashboard)/lens/page.test.tsx | 5 +- .../src/app/(dashboard)/lens/page.tsx | 3 +- .../TraceView/AgentTracesSection.test.tsx | 26 + .../TraceView/AgentTracesSection.tsx | 60 +- .../view_logs/TraceView/AgentTracesTable.tsx | 23 +- .../view_logs/TraceView/useAgentTraces.ts | 28 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 94 ++- .../src/utils/activityTimestamp.test.ts | 26 + .../src/utils/activityTimestamp.ts | 15 + 44 files changed, 3090 insertions(+), 1343 deletions(-) create mode 100644 litellm-rust/crates/traces/query/lens_agents.sql create mode 100644 litellm-rust/crates/traces/query/lens_availability.sql create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKeyDetails.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensFinding.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensOverview.tsx create mode 100644 ui/litellm-dashboard/src/utils/activityTimestamp.test.ts create mode 100644 ui/litellm-dashboard/src/utils/activityTimestamp.ts diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml index d41cb8eb203..4d1224fd41e 100644 --- a/deploy/lens/compose.yaml +++ b/deploy/lens/compose.yaml @@ -1,6 +1,6 @@ services: lens-worker: - image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:a8e8731d954916594eea462969946b9292fb771681ff515a9fd296b53f856c77} + image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:67eba741c1b97c749975c5c38e2370a603e1105babc908d613c1b79d7b995393} environment: LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} diff --git a/litellm-rust/crates/traces/query/lens_agents.sql b/litellm-rust/crates/traces/query/lens_agents.sql new file mode 100644 index 00000000000..fbdd578f8e7 --- /dev/null +++ b/litellm-rust/crates/traces/query/lens_agents.sql @@ -0,0 +1,6 @@ +SELECT DISTINCT AgentName AS agent_name +FROM otel_traces +WHERE AgentName != '' + AND ({all_teams:UInt8}=1 OR TeamId={team:String}) + AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) +ORDER BY agent_name diff --git a/litellm-rust/crates/traces/query/lens_availability.sql b/litellm-rust/crates/traces/query/lens_availability.sql new file mode 100644 index 00000000000..8d350dd1779 --- /dev/null +++ b/litellm-rust/crates/traces/query/lens_availability.sql @@ -0,0 +1,8 @@ +SELECT + EXISTS(SELECT 1 FROM otel_traces + WHERE ({all_teams:UInt8}=1 OR TeamId={team:String}) + AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})) AS traces, + EXISTS(SELECT 1 FROM spend_logs + WHERE ({all_teams:UInt8}=1 OR team_id={team:String}) + AND ({key_hash:String}='' OR api_key={key_hash:String}) + AND NOT JSONExtractBool(metadata,'litellm_lens_internal')) AS requests diff --git a/litellm-rust/crates/traces/query/lens_sample.sql b/litellm-rust/crates/traces/query/lens_sample.sql index 1fc9c964a6f..92086c33c13 100644 --- a/litellm-rust/crates/traces/query/lens_sample.sql +++ b/litellm-rust/crates/traces/query/lens_sample.sql @@ -28,6 +28,7 @@ SELECT *, selection_key FROM ( GROUP BY TeamId,ApiKeyHash,TraceId HAVING max(EngineReceivedMs) < {end:UInt64} AND max(toUnixTimestamp64Milli(Timestamp)+toInt64(intDiv(Duration,1000000))) < {end:UInt64} + AND ({agent_name:String}='' OR countIf(AgentName={agent_name:String}) > 0) AND countIf(arrayAll((k,v) -> ResourceAttributes[k]=v OR SpanAttributes[k]=v, {filter_keys:Array(String)},{filter_values:Array(String)}) AND ({service:String}='' OR ServiceName={service:String})) > 0 @@ -48,6 +49,7 @@ SELECT *, selection_key FROM ( OR JSONExtractString(metadata,'requester_metadata',k)=v OR (k='tag' AND has(request_tags,v)), {filter_keys:Array(String)},{filter_values:Array(String)}) AND ({service:String}='' OR model_group={service:String}) + AND {agent_name:String}='' AND NOT JSONExtractBool(metadata,'litellm_lens_internal') AND ({source:String}!='both' OR (team_id,api_key,response_id) NOT IN ( SELECT TeamId,ApiKeyHash,LiteLLMRequestId FROM otel_traces diff --git a/litellm-rust/crates/traces/src/sql.rs b/litellm-rust/crates/traces/src/sql.rs index 36d6e3b4521..51632066fdd 100644 --- a/litellm-rust/crates/traces/src/sql.rs +++ b/litellm-rust/crates/traces/src/sql.rs @@ -37,6 +37,8 @@ impl ReadQuery { #[derive(Clone, Copy)] pub enum LensQuery { + Availability, + Agents, Sample, Content, Evidence, @@ -45,6 +47,8 @@ pub enum LensQuery { impl LensQuery { pub fn parse(name: &str) -> Result { match name { + "availability" => Ok(Self::Availability), + "agents" => Ok(Self::Agents), "sample" => Ok(Self::Sample), "content" => Ok(Self::Content), "evidence" => Ok(Self::Evidence), @@ -53,6 +57,8 @@ impl LensQuery { } pub fn sql(self) -> &'static str { match self { + Self::Availability => include_str!("../query/lens_availability.sql"), + Self::Agents => include_str!("../query/lens_agents.sql"), Self::Sample => include_str!("../query/lens_sample.sql"), Self::Content => include_str!("../query/lens_content.sql"), Self::Evidence => include_str!("../query/lens_evidence.sql"), diff --git a/litellm-rust/crates/traces/tests/migrations.rs b/litellm-rust/crates/traces/tests/migrations.rs index 01e8982423f..238c5e671c6 100644 --- a/litellm-rust/crates/traces/tests/migrations.rs +++ b/litellm-rust/crates/traces/tests/migrations.rs @@ -551,6 +551,7 @@ async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate( "end".into(), Parameter::Integer(timestamp / 1_000_000 + 1000), ), + ("agent_name".into(), Parameter::Text(String::new())), ("service".into(), Parameter::Text("review".into())), ( "filter_keys".into(), @@ -652,6 +653,7 @@ async fn lens_request_sample_does_not_trust_caller_tags( ("key_hash".into(), Parameter::Text(String::new())), ("start".into(), Parameter::Integer(timestamp - 1000)), ("end".into(), Parameter::Integer(timestamp + 60000)), + ("agent_name".into(), Parameter::Text(String::new())), ("service".into(), Parameter::Text(String::new())), ("filter_keys".into(), Parameter::Strings(vec![])), ("filter_values".into(), Parameter::Strings(vec![])), @@ -719,6 +721,7 @@ async fn lens_selection_pages_without_losing_or_repeating_runs( ("key_hash".into(), Parameter::Text(String::new())), ("start".into(), Parameter::Integer(0)), ("end".into(), Parameter::Integer(end)), + ("agent_name".into(), Parameter::Text(String::new())), ("service".into(), Parameter::Text(String::new())), ("filter_keys".into(), Parameter::Strings(vec![])), ("filter_values".into(), Parameter::Strings(vec![])), @@ -980,3 +983,108 @@ async fn duplicate_span_preview_matches_diagnostic( assert_eq!(diagnostic["data"][0]["message"], message); Ok(()) } + +#[rstest] +#[tokio::test] +async fn lens_agent_discovery_and_selection_preserve_scope( + #[future] database: TestResult, +) -> TestResult { + use litellm_traces::LensQuery; + let database = database.await?; + let writer = Connection::writer(&database.url)?; + ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?; + let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64; + for (team, key, trace, agent, span, parent) in [ + ("alpha", "one", "research", "research_agent", "root", ""), + ("alpha", "one", "research", "", "tool", "root"), + ("alpha", "one", "support", "support_agent", "root", ""), + ("alpha", "two", "hidden-key", "private_agent", "root", ""), + ("beta", "one", "hidden-team", "other_agent", "root", ""), + ] { + insert_rows( + &database, + "otel_traces", + vec![serde_json::from_value(serde_json::json!({ + "Timestamp": timestamp, "TraceId": trace, "SpanId": span, "ParentSpanId": parent, + "ServiceName": "shared-app", "SpanName": "run", "Input": "test", + "SpanAttributes": {"gen_ai.agent.name": agent}, + "ResourceAttributes": {"litellm.team_id": team, "litellm.api_key_hash": key} + }))?], + ) + .await?; + } + let connection = Connection::configured(&database.url, "trace_test", "default", "")?; + let scope_parameters = BTreeMap::from([ + ("all_teams".into(), Parameter::Integer(0)), + ("team".into(), Parameter::Text("alpha".into())), + ("key_hash".into(), Parameter::Text("one".into())), + ]); + let agents: serde_json::Value = serde_json::from_str( + &execute_read( + &database.client, + &connection, + LensQuery::Agents.sql(), + &scope_parameters, + ) + .await?, + )?; + assert_eq!( + agents["data"], + serde_json::json!([ + {"agent_name": "research_agent"}, {"agent_name": "support_agent"} + ]) + ); + let parameters = scope_parameters + .into_iter() + .chain([ + ("source".into(), Parameter::Text("traces".into())), + ( + "start".into(), + Parameter::Integer(timestamp / 1_000_000 - 1000), + ), + ( + "end".into(), + Parameter::Integer(timestamp / 1_000_000 + 1000), + ), + ("service".into(), Parameter::Text("shared-app".into())), + ( + "agent_name".into(), + Parameter::Text("research_agent".into()), + ), + ("filter_keys".into(), Parameter::Strings(vec![])), + ("filter_values".into(), Parameter::Strings(vec![])), + ("limit".into(), Parameter::Integer(100)), + ("offset".into(), Parameter::Integer(0)), + ("after".into(), Parameter::Text(String::new())), + ("sample_percent".into(), Parameter::Text("100".into())), + ("sample_cap".into(), Parameter::Integer(0)), + ("preview".into(), Parameter::Integer(1)), + ("selected_team".into(), Parameter::Text(String::new())), + ("execution_ids".into(), Parameter::Strings(vec![])), + ]) + .collect::>(); + let sample: serde_json::Value = serde_json::from_str( + &execute_read( + &database.client, + &connection, + LensQuery::Sample.sql(), + ¶meters, + ) + .await?, + )?; + assert_eq!(sample["data"].as_array().expect("rows").len(), 1); + assert_eq!(sample["data"][0]["trace_id"], "research"); + assert_eq!(sample["data"][0]["span_count"], 2); + let available: serde_json::Value = serde_json::from_str( + &execute_read( + &database.client, + &connection, + LensQuery::Availability.sql(), + ¶meters, + ) + .await?, + )?; + assert_eq!(available["data"][0]["traces"], 1); + assert_eq!(available["data"][0]["requests"], 0); + Ok(()) +} diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index 0349c594adf..349ecb9a353 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -2,6 +2,7 @@ import hashlib import secrets from datetime import datetime, timedelta, timezone from functools import reduce +from itertools import chain from types import MappingProxyType from typing import Annotated, Final, TypeAlias from uuid import uuid4 @@ -35,7 +36,7 @@ from litellm.proxy.lens.models import ( WorkerCreated, ) from litellm.proxy.lens.repository import LensRepository, WriterDatabase -from litellm.proxy.lens.sources import SourceReader, Storage, parse_execution +from litellm.proxy.lens.sources import ActivityAvailability, SourceReader, Storage, parse_execution from litellm.proxy.lens.state import ( can_access, claim_job, @@ -168,6 +169,18 @@ async def create_lens(settings: LensSettings, auth: Auth) -> Lens: return await repository().create(queue_job(lens, now, str(uuid4()))) +@router.get("/activity/available", response_model=ActivityAvailability) +async def activity_available(auth: Auth, storage: StorageDep) -> ActivityAvailability: + scope: Final = user_scope(auth) + return await source_reader(storage).availability(scope) if storage is not None else ActivityAvailability() + + +@router.get("/agents", response_model=tuple[str, ...]) +async def list_agents(auth: Auth, storage: StorageDep) -> tuple[str, ...]: + scope: Final = user_scope(auth) + return await source_reader(storage).agents(scope) if storage is not None else () + + @router.put("/{lens_id}", response_model=Lens) async def update_lens(lens_id: str, settings: LensSettings, auth: Auth) -> Lens: await get_lens(lens_id, user_scope(auth, write=True)) @@ -264,7 +277,7 @@ class Preview(BaseModel): as_of: AwareDatetime | None = None offset: int = Field(default=0, ge=0) settings: LensSettings - lookback_hours: int = Field(default=24, ge=1, le=720) + lookback_hours: int = Field(default=24, ge=1, le=8760) @router.post("/preview/sample", response_model=Sample) @@ -326,6 +339,9 @@ async def revoke_worker(worker_id: str, auth: Auth) -> bool: worker: Final = next((w for w in await repository().workers() if w.id == worker_id), None) if worker is None or not can_access(scope, worker.scope): raise HTTPException(404, "Worker not found") + jobs: Final = chain.from_iterable(lens.jobs for lens in await repository().lenses()) + if any(job.status == "running" and job.worker_id == worker.id for job in jobs): + raise HTTPException(409, "Wait for this worker's investigation to finish or cancel it before revoking access") await repository().revoke_worker(worker.id) return True diff --git a/litellm/proxy/lens/models.py b/litellm/proxy/lens/models.py index eb88801d065..91f0ad582bf 100644 --- a/litellm/proxy/lens/models.py +++ b/litellm/proxy/lens/models.py @@ -29,8 +29,9 @@ class LensSettings(Record): name: str = Field(min_length=1, max_length=100) context: str = Field(default="", max_length=6000) source: Literal["traces", "requests", "both"] = "traces" - lookback_hours: int = Field(default=24, ge=1, le=720) + lookback_hours: int = Field(default=24, ge=1, le=8760) service: str = Field(default="", max_length=200) + agent_name: str = Field(default="", max_length=200) filters: tuple[MetadataFilter, ...] = Field(default=(), max_length=8) checks: tuple[Check, ...] = () model: str = Field(min_length=1, max_length=200) @@ -41,7 +42,7 @@ class LensSettings(Record): concurrency: int = Field(default=8, ge=1) team_id: str = "" execution_ids: tuple[str, ...] = () - monthly_budget: float = Field(default=20, gt=0, le=100000, allow_inf_nan=False) + monthly_budget: float = Field(default=100, gt=0, le=100000, allow_inf_nan=False) @model_validator(mode="after") def unique_checks(self) -> "LensSettings": @@ -214,7 +215,7 @@ class LensList(Record): class RunRequest(Record): settings: LensSettings | None = None - lookback_hours: int | None = Field(default=None, ge=1, le=720) + lookback_hours: int | None = Field(default=None, ge=1, le=8760) class FindingUpdate(Record): diff --git a/litellm/proxy/lens/sources.py b/litellm/proxy/lens/sources.py index 12d26cd4974..17550cc4aab 100644 --- a/litellm/proxy/lens/sources.py +++ b/litellm/proxy/lens/sources.py @@ -18,7 +18,14 @@ from litellm.proxy.lens.models import ( ) +class ActivityAvailability(BaseModel): + traces: bool = False + requests: bool = False + + class Storage(Protocol): + def lens_availability(self, parameters: Mapping[str, object]) -> Awaitable[object]: ... + def lens_agents(self, parameters: Mapping[str, object]) -> Awaitable[object]: ... def lens_sample(self, parameters: Mapping[str, object]) -> Awaitable[object]: ... def lens_content(self, parameters: Mapping[str, object]) -> Awaitable[object]: ... def lens_evidence(self, parameters: Mapping[str, object]) -> Awaitable[object]: ... @@ -53,6 +60,12 @@ class CountRow(BaseModel): count: int +class AgentRow(BaseModel): + agent_name: str + + +_AVAILABILITY: Final = TypeAdapter(tuple[ActivityAvailability, ...]) +_AGENTS: Final = TypeAdapter(tuple[AgentRow, ...]) _ROWS: Final = TypeAdapter(tuple[ExecutionRow, ...]) _PARTS: Final = TypeAdapter(tuple[PartRow, ...]) _COUNTS: Final = TypeAdapter(tuple[CountRow, ...]) @@ -90,6 +103,14 @@ class SourceReader: def __init__(self, storage: Storage) -> None: self.storage: Final = storage + async def availability(self, scope: Scope) -> ActivityAvailability: + rows: Final = _AVAILABILITY.validate_python(await self.storage.lens_availability(parameters(scope, ()))) + return rows[0] if rows else ActivityAvailability() + + async def agents(self, scope: Scope) -> tuple[str, ...]: + rows: Final = _AGENTS.validate_python(await self.storage.lens_agents(parameters(scope, ()))) + return tuple(row.agent_name for row in rows) + async def sample( self, scope: Scope, @@ -108,6 +129,7 @@ class SourceReader: "start": start, "end": end, "service": settings.service, + "agent_name": settings.agent_name, "limit": page_size, "offset": offset, "after": cursor, diff --git a/litellm/proxy/lens/worker.py b/litellm/proxy/lens/worker.py index 2980f62deed..62f8295e7d3 100644 --- a/litellm/proxy/lens/worker.py +++ b/litellm/proxy/lens/worker.py @@ -15,6 +15,40 @@ from .models import Claim, Coverage, ExecutionContent, ModelRequest, ModelResult logger: Final = logging.getLogger("litellm.lens.worker") +def failure_message(error: Exception) -> str: + if isinstance(error, (OSError, sqlite3.Error)): + return "Worker temporary storage failed. Increase its capacity or reduce analysis parallelism." + if isinstance(error, httpx.TimeoutException): + return "The worker timed out waiting for the proxy. Check proxy availability and model response times." + if isinstance(error, httpx.TransportError): + return "The worker could not connect to the proxy. Check the proxy URL, network access, and TLS configuration." + if isinstance(error, httpx.HTTPStatusError): + path: Final = error.request.url.path + action: Final = ( + "Model request" + if path.endswith("/model") + else "Reading trace data" + if path.endswith(("/sample", "/content")) + else "Saving results" + if path.endswith("/result") + else "Worker request" + ) + status: Final = error.response.status_code + guidance: Final = MappingProxyType( + { + 400: "Check the configured model and whether the worker's billing key is enabled.", + 401: "Check the worker credential and its assigned billing key.", + 402: "Check the investigation's monthly limit and the worker key's remaining budget.", + 403: "Check the worker key's model permissions and access restrictions.", + 404: "Check that the proxy and worker versions match and the requested model is configured.", + 409: "This worker no longer owns the run. Check whether it was cancelled or claimed again.", + 429: "The request was rate limited. Retry later or check the worker key's rate limits.", + } + ).get(status, "Check proxy and model availability, then retry the investigation.") + return f"{action} failed (HTTP {status}). {guidance}" + return "The worker could not read an analysis response. Check structured JSON support and matching proxy/worker versions." + + class LensWorker: def __init__(self, client: httpx.AsyncClient, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep) -> None: self.client: Final = client @@ -82,14 +116,7 @@ class LensWorker: saved: Final = await self.client.post(prefix + "/result", json=result.model_dump(mode="json")) saved.raise_for_status() except (httpx.HTTPError, ValueError, OSError, sqlite3.Error) as exc: - status: Final = exc.response.status_code if isinstance(exc, httpx.HTTPStatusError) else None - message: Final = ( - "Worker temporary storage failed. Increase its capacity or reduce analysis parallelism." - if isinstance(exc, (OSError, sqlite3.Error)) - else "Monthly budget reached" - if status == 402 - else "Analysis interrupted. Check worker connectivity, model configuration, and trace storage." - ) + message: Final = failure_message(exc) logger.warning("Analysis %s interrupted (%s)", claim.job.id, type(exc).__name__) failed: Final = await self.client.post( prefix + "/result", json=Result(coverage=Coverage(), error=message).model_dump() diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index 6724db41ad3..06e006be89c 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -108,6 +108,12 @@ class ClickHouseStorage: async def lens_sample(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: return await self._lens_query("sample", parameters) + async def lens_availability(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: + return await self._lens_query("availability", parameters) + + async def lens_agents(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: + return await self._lens_query("agents", parameters) + async def lens_content(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: return await self._lens_query("content", parameters) diff --git a/tests/integration/spend/test_lens_billing.py b/tests/integration/spend/test_lens_billing.py index bedcf6c5380..d8eded62b39 100644 --- a/tests/integration/spend/test_lens_billing.py +++ b/tests/integration/spend/test_lens_billing.py @@ -124,6 +124,9 @@ def test_lens_bills_selected_key_and_rechecks_its_permissions(gateway: Gateway, seconds=70, ) assert second_rows[0]["spend"] == pytest.approx(expected) + active_revoke: Final = gateway.request("DELETE", f"/lens/workers/{worker_id}") + assert active_revoke.status_code == 409, active_revoke.text + gateway.post(f"/lens/{lens_id}/cancel", {}) revoked: Final = gateway.request("DELETE", f"/lens/workers/{worker_id}") assert revoked.status_code == 200, revoked.text denied_worker: Final = gateway.request( @@ -134,7 +137,6 @@ def test_lens_bills_selected_key_and_rechecks_its_permissions(gateway: Gateway, "PUT", f"/lens/workers/{worker_id}/billing-key", {"analysis_key_id": replacement_id} ) assert forbidden_change.status_code == 409, forbidden_change.text - gateway.post(f"/lens/{lens_id}/cancel", {}) @pytest.mark.parametrize("cancel_on_disconnect", (False, True)) diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 97bb7759a02..ca19277da08 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -4,7 +4,22 @@ import pytest from fastapi import HTTPException from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.lens.endpoints import user_scope +from litellm.proxy.lens.endpoints import list_agents, user_scope + + +@pytest.mark.parametrize("role", (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)) +@pytest.mark.asyncio +async def test_agent_discovery_without_trace_storage_is_empty(role: LitellmUserRoles) -> None: + auth: Final = UserAPIKeyAuth(user_role=role) + assert await list_agents(auth, None) == () + + +@pytest.mark.asyncio +async def test_agent_discovery_without_trace_storage_still_requires_admin_access() -> None: + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER) + with pytest.raises(HTTPException) as error: + await list_agents(auth, None) + assert error.value.status_code == 403 @pytest.mark.parametrize( diff --git a/tests/unit/proxy/lens/test_sources.py b/tests/unit/proxy/lens/test_sources.py index 5dc6e2652f0..f063f496314 100644 --- a/tests/unit/proxy/lens/test_sources.py +++ b/tests/unit/proxy/lens/test_sources.py @@ -61,3 +61,48 @@ async def test_sample_never_returns_authentication_attributes() -> None: assert sample.executions[0].metadata == (MetadataFilter(key="environment", value="production"),) assert "opaque-oauth-bearer" not in sample.model_dump_json() assert sample.eligible == 1 + + +@pytest.mark.asyncio +async def test_agents_use_the_same_team_and_key_scope_as_samples() -> None: + class AgentStorage: + async def lens_agents(self, parameters): + assert parameters["all_teams"] == 0 + assert parameters["team"] == "alpha" + assert parameters["key_hash"] == "key-hash" + return [{"agent_name": "research_agent"}, {"agent_name": "support_agent"}] + + names: Final = await SourceReader(AgentStorage()).agents(Scope(team_id="alpha", api_key_hash="key-hash")) + assert names == ("research_agent", "support_agent") + + +@pytest.mark.asyncio +async def test_request_only_storage_is_available_for_investigation() -> None: + class RequestStorage: + async def lens_availability(self, parameters): + assert parameters["team"] == "alpha" + return [{"traces": 0, "requests": 1}] + + available: Final = await SourceReader(RequestStorage()).availability(Scope(team_id="alpha")) + assert available.requests + assert not available.traces + + +@pytest.mark.asyncio +async def test_agent_filter_is_independent_of_service_and_metadata() -> None: + class SampleStorage: + async def lens_sample(self, parameters): + assert parameters["agent_name"] == "research_agent" + assert parameters["service"] == "shared-app" + assert parameters["filter_keys"] == ("enduser.id",) + assert parameters["filter_values"] == ("user-42",) + return [] + + settings: Final = lens().settings.model_copy( + update={ + "agent_name": "research_agent", + "service": "shared-app", + "filters": (MetadataFilter(key="enduser.id", value="user-42"),), + } + ) + assert not (await SourceReader(SampleStorage()).sample(Scope(all_teams=True), settings, 1, 2)).executions diff --git a/tests/unit/proxy/lens/test_state.py b/tests/unit/proxy/lens/test_state.py index ac70a22077e..0e01085fb04 100644 --- a/tests/unit/proxy/lens/test_state.py +++ b/tests/unit/proxy/lens/test_state.py @@ -86,7 +86,7 @@ def test_behavior_description_is_sufficient_without_separate_checks() -> None: @pytest.mark.parametrize( - "field,value", (("sample_percent", 0), ("sample_percent", 101), ("sample_size", 0), ("concurrency", 0)) + "field,value", (("sample_percent", 0), ("sample_percent", 101), ("sample_size", 0), ("concurrency", 0), ("lookback_hours", 0), ("lookback_hours", 8761)) ) def test_invalid_selection_and_parallelism_are_rejected(field: str, value: int) -> None: from pydantic import ValidationError @@ -145,11 +145,11 @@ def test_monthly_budget_renews_without_erasing_job_costs() -> None: assert renew_budget(spent, NOW) is spent -@pytest.mark.parametrize("hours", (24, 168, 720)) +@pytest.mark.parametrize("hours", (24, 168, 720, 4800, 8760)) def test_every_scan_uses_the_configured_lookback_window(hours: int) -> None: original: Final = lens() configured: Final = original.model_copy( - update={"settings": original.settings.model_copy(update={"lookback_hours": hours})} + update={"settings": LensSettings.model_validate({**original.settings.model_dump(), "lookback_hours": hours})} ) first: Final = queue_job(configured, NOW, "first") assert first.jobs[0].start == NOW - timedelta(hours=hours) diff --git a/tests/unit/proxy/lens/test_worker.py b/tests/unit/proxy/lens/test_worker.py index a0212e03319..dce3fc04d45 100644 --- a/tests/unit/proxy/lens/test_worker.py +++ b/tests/unit/proxy/lens/test_worker.py @@ -15,7 +15,7 @@ from litellm.proxy.lens.models import ( TracePart, ) from litellm.proxy.lens.state import queue_job -from litellm.proxy.lens.worker import LensWorker +from litellm.proxy.lens.worker import LensWorker, failure_message from tests.unit.proxy.lens.test_state import NOW, lens @@ -126,6 +126,34 @@ async def test_worker_reads_claimed_activity_and_reports_analysis_or_failure(mod assert result.coverage.screened == 1 assert result.coverage.unassessable == 0 elif model_status == 402: - assert result.error == "Monthly budget reached" + assert "HTTP 402" in result.error and "remaining budget" in result.error else: - assert result.error.startswith("Analysis interrupted.") + assert result.error.startswith("Model request failed (HTTP 503).") + + +@pytest.mark.parametrize("status", (400, 401, 402, 403, 404, 409, 429, 503)) +def test_failure_reports_action_and_status_without_private_response_content(status: int) -> None: + request: Final = httpx.Request( + "POST", "https://private-host.test/lens/worker/private-lens/private-run/model?token=secret" + ) + response: Final = httpx.Response(status, request=request, text="private trace content and key") + error: Final = httpx.HTTPStatusError("private exception details", request=request, response=response) + message: Final = failure_message(error) + assert message.startswith(f"Model request failed (HTTP {status}).") + assert "private" not in message and "secret" not in message + + +@pytest.mark.parametrize( + "route,action", (("sample", "Reading trace data"), ("content", "Reading trace data"), ("result", "Saving results")) +) +def test_failure_identifies_the_failing_worker_operation(route: str, action: str) -> None: + request: Final = httpx.Request("GET", f"https://proxy.test/lens/worker/lens/job/{route}") + response: Final = httpx.Response(503, request=request) + error: Final = httpx.HTTPStatusError("private body", request=request, response=response) + assert failure_message(error).startswith(f"{action} failed (HTTP 503).") + + +def test_connection_timeout_and_invalid_response_have_distinct_private_diagnostics() -> None: + assert "connect to the proxy" in failure_message(httpx.ConnectError("private hostname")) + assert "timed out" in failure_message(httpx.ReadTimeout("private prompt")) + assert "structured JSON" in failure_message(ValueError("private model response")) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/ActivityScope.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/ActivityScope.tsx index 188c1e6db92..a8f845b02fd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/ActivityScope.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/ActivityScope.tsx @@ -1,10 +1,18 @@ "use client"; -import { useEffect, useId, useState } from "react"; +import { useEffect, useId, useState, type ReactNode } from "react"; import { useQuery } from "@tanstack/react-query"; -import { Plus, X, ArrowUpRight } from "lucide-react"; +import { Plus, X, ChevronRight, RotateCw } from "lucide-react"; import { apiClient } from "@/components/networking"; import { Button } from "@/components/ui/button"; +import { + Combobox, + ComboboxInput, + ComboboxContent, + ComboboxList, + ComboboxItem, + ComboboxEmpty, +} from "@/components/ui/combobox"; import { Input } from "@/components/ui/input"; import { TracePanel } from "./TracePanel"; import { type Sample, type Settings, runTime, durationLabel } from "./lensData"; @@ -15,7 +23,14 @@ export type ActivitySelection = Pick & Partial< Pick< Settings, - "service" | "filters" | "lookback_hours" | "sample_percent" | "sample_size" | "team_id" | "execution_ids" + | "service" + | "agent_name" + | "filters" + | "lookback_hours" + | "sample_percent" + | "sample_size" + | "team_id" + | "execution_ids" > >; @@ -28,10 +43,8 @@ export function RunList({ executions }: { executions: Sample["executions"] }) {

{run.name}

- {runTime(run.start_time)} · {run.source === "traces" ? `${run.span_count} steps` : "LLM request"} -

-

- {run.trace_id} + {runTime(run.start_time)} ·{" "} + {run.source === "traces" ? `${run.span_count} ${run.span_count === 1 ? "step" : "steps"}` : "LLM request"}

))} @@ -43,12 +56,22 @@ export function ActivityScope({ value, onChange, accessToken, + mode = "scope", + onPreviewReady, + manualSelection = false, + nameField, }: { value: ActivitySelection; onChange: (selection: ActivitySelection) => void; accessToken: string; + mode?: "scope" | "activity"; + onPreviewReady?: (ready: boolean) => void; + manualSelection?: boolean; + nameField?: ReactNode; }) { const id = useId(); + const hasFilters = !!value.filters?.length || !!value.team_id; + const [advanced, setAdvanced] = useState(hasFilters || !!value.service || value.source !== "traces"); const [offset, setOffset] = useState(0); const [scope, setScope] = useState(value); const [trace, setTrace] = useState<{ id: string; ref?: string } | null>(null); @@ -63,7 +86,7 @@ export function ActivityScope({ return () => clearTimeout(timer); }, [serialized]); const historyHours = value.lookback_hours ?? 24; - const validWindow = Number.isInteger(historyHours) && historyHours >= 1 && historyHours <= 720; + const validWindow = Number.isInteger(historyHours) && historyHours >= 1 && historyHours <= 8760; const percent = scope.sample_percent ?? 100; const cap = scope.sample_size; const validCap = cap == null || (Number.isInteger(cap) && cap > 0); @@ -96,12 +119,19 @@ export function ActivityScope({ lookback_hours: value.lookback_hours, }; const discoveryOptions = { - queryKey: ["lens-activity-options", value.source, value.lookback_hours, accessToken], + queryKey: ["lens-activity-options", value.source, value.lookback_hours, asOf, accessToken], queryFn: () => load(discoveryScope), staleTime: 60000, enabled: validWindow, }; const discovery = useQuery(discoveryOptions); + const agentOptions = { + queryKey: ["lens-agents", accessToken, asOf], + queryFn: () => apiClient.get("/lens/agents", { accessToken }), + enabled: value.source !== "requests", + staleTime: 60000, + }; + const agents = useQuery(agentOptions); const previewOptions = { queryKey: ["lens-activity-preview", scope, offset, asOf, accessToken], queryFn: () => load(scope, offset), @@ -109,18 +139,38 @@ export function ActivityScope({ staleTime: 30000, }; const preview = useQuery(previewOptions); + const empty = preview.data?.eligible === 0; + useEffect(() => { + if (!empty || !valid) return; + const timer = window.setTimeout(() => setAsOf(new Date().toISOString()), 15000); + return () => window.clearTimeout(timer); + }, [empty, valid, asOf]); + const refreshPreview = () => { + setOffset(0); + setAsOf(new Date().toISOString()); + }; const runs = discovery.data?.executions ?? []; const services = [...new Set(runs.map((r) => r.service).filter(Boolean))].sort(); + const selectedName = value.source === "requests" ? value.service : value.agent_name; + const names = value.source === "requests" ? services : agents.data ?? []; + const selectName = (name: string) => + onChange({ ...value, [value.source === "requests" ? "service" : "agent_name"]: name, execution_ids: [] }); const attributes = runs.flatMap((r) => r.metadata ?? []); const keys = [...new Set(attributes.map((a) => a.key).filter((key) => !key.startsWith("litellm.")))].sort(); const pending = serialized !== JSON.stringify(scope) || preview.isFetching; const ready = !pending && valid; + const hasSelection = !manualSelection || !!value.execution_ids?.length; + const hasMatches = !preview.error && (preview.data?.selected ?? 0) > 0; + const canReview = ready && hasMatches && hasSelection; + useEffect(() => { + onPreviewReady?.(canReview); + }, [canReview, onPreviewReady]); const filters = value.filters ?? []; const edit = (index: number, field: "key" | "value", text: string) => onChange({ ...value, filters: filters.map((f, i) => (i === index ? { ...f, [field]: text } : f)) }); const changeSource = (source: Settings["source"]) => { - const selection = { ...value, source, service: "", filters: [], execution_ids: [] }; + const selection = { ...value, source, service: "", agent_name: "", filters: [], execution_ids: [] }; onChange(selection); }; const windowLabel = validWindow @@ -128,193 +178,201 @@ export function ActivityScope({ : "Choose a valid history window"; const previewTitle = () => { if (pending) return "Finding matching activity…"; - if (!validWindow) return "Choose a history window between 1 and 720 hours"; + if (!validWindow) return "Choose a history window between 1 hour and 365 days"; if (!valid) return "Complete your condition to preview matches"; if (!preview.data) return "Preview unavailable"; - return `${preview.data.eligible} matching ${value.source === "requests" ? "requests" : "runs"}`; + const noun = value.source === "requests" ? "request" : "run"; + return `${preview.data.eligible} matching ${noun}${preview.data.eligible === 1 ? "" : "s"}`; }; return ( -
-
- -

- {value.source === "requests" - ? "Each request is one model call, not an entire agent run." - : "An agent run contains the steps recorded under one trace ID. Separate sessions are not joined automatically."} -

- -

- { - { - requests: "The model alias configured on your LiteLLM gateway. Leave blank for all models.", - both: "Matches the application name on agent runs or the model group on requests. Leave blank to include both without a name filter.", - traces: - "The service.name recorded by your agent’s OpenTelemetry instrumentation. Leave blank for all applications.", - }[value.source ?? "traces"] - } -

-
-

- Narrow by metadata (optional) -

-

- Match a recorded tag, swarm, or environment. Every condition must match exactly. -

- {filters.map((f, index) => ( -
- edit(index, "key", e.target.value)} - /> - is - edit(index, "value", e.target.value)} - /> - - {[...new Set(attributes.filter((a) => a.key === f.key).map((a) => a.value))].sort().map((v) => ( - - -
- ))} - - {keys.map((key) => ( - - -

- Suggestions come from up to 100 recent runs. You can also type a recorded key or value. -

-
- - onChange({ ...value, lookback_hours })} - /> -

- Time window used by each scan. Activity becomes eligible two minutes after it finishes. -

-
- + {value.source !== "requests" && agents.isError && ( +

+ Could not load agents.{" "} + +

+ )} +
setAdvanced(event.currentTarget.open)} className="group"> + + Advanced filters{filters.length ? ` (${filters.length})` : ""} + +
+ {value.source !== "requests" && ( + + )} + +

+ Match any recorded metadata, such as a user ID, environment, or tag. All conditions must match. +

+ {filters.map((f, index) => ( +
+
+ edit(index, "key", e.target.value)} + /> + +
+ edit(index, "value", e.target.value)} + /> + + {[...new Set(attributes.filter((a) => a.key === f.key).map((a) => a.value))].sort().map((v) => ( + +
+ ))} + + {keys.map((key) => ( + + + +
+
+ + ) : ( + <> + onChange({ ...value, lookback_hours })} /> - - -
-

100% with no limit selects all matching activity.

- {!!value.execution_ids?.length && ( - )}
- - onChange({ - ...value, - execution_ids: checked - ? [...(value.execution_ids ?? []), runId] - : (value.execution_ids ?? []).filter((id) => id !== runId), - }) - } - selectedIds={value.execution_ids ?? []} - selectedCount={ - value.execution_ids?.length - ? Math.min( - Math.ceil((value.execution_ids.length * (value.sample_percent ?? 100)) / 100), - value.sample_size ?? Infinity, - ) - : preview.data?.selected ?? 0 - } - title={previewTitle()} - windowLabel={windowLabel} - ready={ready} - error={preview.error} - data={preview.data} - onOpen={(run) => setTrace({ id: run.trace_id, ref: run.trace_ref })} - /> + {mode === "activity" && ( + + onChange({ + ...value, + execution_ids: checked + ? [...(value.execution_ids ?? []), runId] + : (value.execution_ids ?? []).filter((id) => id !== runId), + }) + } + manualSelection={manualSelection} + selectedIds={value.execution_ids ?? []} + selectedCount={ + manualSelection + ? Math.min( + Math.ceil(((value.execution_ids?.length ?? 0) * (value.sample_percent ?? 100)) / 100), + value.sample_size ?? Infinity, + ) + : preview.data?.selected ?? 0 + } + title={previewTitle()} + windowLabel={windowLabel} + ready={ready} + error={preview.error} + data={preview.data} + onRetry={refreshPreview} + onOpen={(run) => setTrace({ id: run.trace_id, ref: run.trace_ref })} + /> + )} {trace && ( void; onSelect: (id: string, checked: boolean) => void; selectedIds: string[]; + manualSelection: boolean; selectedCount: number; title: string; windowLabel: string; ready: boolean; error: Error | null; data: Sample | undefined; + onRetry: () => void; onOpen: (run: Sample["executions"][number]) => void; }) { + const paginated = data?.next_offset != null || offset > 0; + const showSelection = selectedCount !== data?.eligible || paginated; return (
-

- {title} -

-

{windowLabel} · Preview only, no analysis cost

+
+

+ {title} +

+ +
+

{windowLabel} · No analysis cost

-
+
{ready && error && (

- {error.message} + {error.message}{" "} +

)} {ready && data?.eligible === 0 && (

- No matches. Try removing a condition or check that your agent records this metadata. Very recent runs need - two minutes to settle. + No matches. Try removing a condition or check that your agent records this metadata. Recent trace updates + need two minutes to settle.

)} {ready && data?.executions.map((run) => (
- onSelect(run.id, e.target.checked)} - /> + {manualSelection && ( + onSelect(run.id, e.target.checked)} + /> + )}
{run.source === "traces" && ( - )}
))}
- {ready && data && ( + {ready && data && showSelection && (

- {selectedCount} selected for analysis · Showing {offset + (data.executions.length ? 1 : 0)}– - {offset + data.executions.length} of {data.eligible} + {selectedCount} selected for analysis + {paginated && ( + <> + {" "} + · Showing {offset + (data.executions.length ? 1 : 0)}–{offset + data.executions.length} of{" "} + {data.eligible} + + )}

-
- - -
+ {paginated && ( +
+ + +
+ )}
)}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx index 7a0130bd929..f3174b2518d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.integration.test.tsx @@ -12,36 +12,27 @@ describe("Lens billing key", () => { testQueryClient.clear(); vi.clearAllMocks(); }); - it("creates a normal key and only passes its ID to worker settings", async () => { - const user = userEvent.setup(); - const changed = vi.fn(); - vi.mocked(apiClient.get).mockResolvedValue({ keys: [], total_pages: 0 }); - vi.mocked(apiClient.post).mockResolvedValue({ token_id: "b".repeat(64), key: "sk-secret-not-for-settings" }); - renderWithProviders(); - await user.click(screen.getByRole("button", { name: "Create worker key" })); - expect(await screen.findByRole("combobox", { name: "Charge analysis to" })).toHaveValue("Lens: Research"); - expect(apiClient.post).toHaveBeenCalledWith("/key/generate", { - accessToken: "test", - body: { key_alias: "Lens: Research", models: [], metadata: { purpose: "lens" } }, - }); - expect(changed).toHaveBeenCalledExactlyOnceWith("b".repeat(64)); - expect(screen.queryByText("sk-secret-not-for-settings")).not.toBeInTheDocument(); - }); it("pages existing keys without dropping the selected billing key", async () => { const user = userEvent.setup(); const changed = vi.fn(); - vi.mocked(apiClient.get).mockImplementation(async (_path, options) => ({ - keys: - options?.query?.page === "2" - ? [{ token: "c".repeat(64), key_alias: "Second page" }] - : [{ token: "a".repeat(64), key_alias: "First page" }], - total_pages: 2, - })); - renderWithProviders(); + vi.mocked(apiClient.get).mockImplementation(async (path, options) => + path === "/key/info" + ? { info: { models: ["restricted-model"], max_budget: 4, budget_duration: "1d" } } + : { + keys: + options?.query?.page === "2" + ? [{ token: "c".repeat(64), key_alias: "Second page" }] + : [{ token: "a".repeat(64), key_alias: "First page" }], + total_pages: 2, + }, + ); + renderWithProviders(); await user.click(screen.getByRole("combobox", { name: "Charge analysis to" })); await user.click(await screen.findByRole("option", { name: "Load more keys" })); await user.click(await screen.findByRole("option", { name: "Second page" })); expect(changed).toHaveBeenCalledExactlyOnceWith("c".repeat(64)); + expect(await screen.findByText("restricted-model")).toBeInTheDocument(); + expect(screen.getByText("$4.00 / day")).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx index c26c42f5700..4033b9634b2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKey.tsx @@ -1,10 +1,12 @@ "use client"; import { useState } from "react"; -import { useInfiniteQuery } from "@tanstack/react-query"; +import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; import { z } from "zod"; import { apiClient } from "@/components/networking"; -import { Button } from "@/components/ui/button"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { Input } from "@/components/ui/input"; +import { AnalysisKeyDetails } from "./AnalysisKeyDetails"; import { Combobox, ComboboxContent, @@ -22,17 +24,14 @@ export function AnalysisKey({ accessToken, value, onChange, - name, }: { accessToken: string; value: string | null; onChange: (key: string | null) => void; - name: string; }) { const [query, setQuery] = useState(""); const [selected, setSelected] = useState(value ? { token: value } : null); - const [creating, setCreating] = useState(false); - const [error, setError] = useState(""); + const queryOptions = { queryKey: ["lens-analysis-keys", accessToken, query], initialPageParam: 1, @@ -61,28 +60,6 @@ export function AnalysisKey({ const choice = keys.find((key) => key.token === value) ?? selected; const loading = keyPages.isFetching; - const create = async () => { - setCreating(true); - setError(""); - try { - const result = await apiClient.post("/key/generate", { - accessToken, - body: { - key_alias: `Lens: ${name}`, - models: [], - metadata: { purpose: "lens" }, - }, - }); - if (!result.token_id) throw new Error("The proxy did not return the new key's ID"); - const key = { token: result.token_id, key_alias: `Lens: ${name}` }; - setSelected(key); - onChange(key.token); - } catch (cause) { - setError(cause instanceof Error ? cause.message : "Could not create a key"); - } finally { - setCreating(false); - } - }; const changeKey = (key: Key | null, details: { cancel: () => void }) => { if (key?.token === "load-more") { details.cancel(); @@ -99,7 +76,7 @@ export function AnalysisKey({ return (

Charge analysis to

-
+
-
-

- Spend appears under this key in API Keys. Its permissions and limits apply. -

- {(error || keyPages.error) && ( + {choice && } + {keyPages.error && (

- {error || keyPages.error?.message} + {keyPages.error.message}

)}
); } + +export type AnalysisAccess = { model: string | null; budget: string }; + +export function AnalysisAccessFields({ + accessToken, + value, + onChange, +}: { + accessToken: string; + value: AnalysisAccess; + onChange: (value: AnalysisAccess) => void; +}) { + const models = useQuery({ + queryKey: ["lens-models", accessToken], + queryFn: () => apiClient.get<{ data: { id: string }[] }>("/models", { accessToken }), + }); + return ( +
+
+ + ({ label: id, value: id }))} + value={value.model} + onValueChange={(model) => onChange({ ...value, model })} + placeholder={models.isLoading ? "Loading models…" : "Select a model"} + /> +
+
+ + onChange({ ...value, budget: e.target.value })} + /> +

Shared across all investigations.

+
+ {models.error && ( +

+ {models.error.message} +

+ )} +
+ ); +} + +export async function createAnalysisKey(accessToken: string, access: AnalysisAccess): Promise { + if (!access.model || !Number.isFinite(Number(access.budget)) || Number(access.budget) <= 0) + throw new Error("Choose a model and a monthly limit greater than zero"); + const result = await apiClient.post("/key/generate", { + accessToken, + body: { + key_alias: "Lens analysis", + models: [access.model], + max_budget: Number(access.budget), + budget_duration: "1mo", + metadata: { purpose: "lens" }, + }, + }); + if (!result.token_id) throw new Error("The proxy did not return the new key's ID"); + return result.token_id; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKeyDetails.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKeyDetails.tsx new file mode 100644 index 00000000000..e94099ac6e6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/AnalysisKeyDetails.tsx @@ -0,0 +1,99 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { z } from "zod"; +import { apiClient } from "@/components/networking"; +import { Button } from "@/components/ui/button"; +import { runTime } from "./lensData"; + +const keyInfoFields = { + key_alias: z.string().nullable().optional(), + models: z.array(z.string()), + max_budget: z.number().nullable(), + budget_duration: z.string().nullable().optional(), + rpm_limit: z.number().nullable().optional(), + tpm_limit: z.number().nullable().optional(), + expires: z.string().nullable().optional(), + status: z.string().optional(), +}; +const keyInfoSchema = z.object({ info: z.object(keyInfoFields) }); + +function budgetLabel(amount: number | null, duration?: string | null): string { + if (amount === null) return "No key budget"; + const periods: Record = { + "1mo": "month", + "30d": "month", + "1d": "day", + "24h": "day", + "7d": "week", + "1h": "hour", + }; + const dollars = new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 2, + }).format(amount); + return duration ? `${dollars} / ${periods[duration] ?? duration}` : `${dollars} total`; +} + +export function useAnalysisKeyInfo(accessToken: string, keyId?: string) { + return useQuery({ + queryKey: ["lens-key-info", accessToken, keyId], + enabled: !!keyId, + queryFn: async () => + keyInfoSchema.parse(await apiClient.get("/key/info", { accessToken, query: { key: keyId } })).info, + }); +} + +export function AnalysisKeyDetails({ + accessToken, + keyId, + showName = false, +}: { + accessToken: string; + keyId: string; + showName?: boolean; +}) { + const key = useAnalysisKeyInfo(accessToken, keyId); + if (key.isLoading) return

Loading key permissions…

; + if (key.error || !key.data) + return ( +
+ Could not load key permissions + +
+ ); + const info = key.data; + return ( +
+
+ {showName && ( + <> +
Billing key
+
{info.key_alias || "Assigned virtual key"}
+ + )} +
Models
+
{info.models.length ? info.models.join(", ") : "All models"}
+
Key limit
+
{budgetLabel(info.max_budget, info.budget_duration)}
+
+ {info.status && info.status !== "active" && ( +

+ This key is {info.status}. Choose an active key. +

+ )} +
+ Other limits +
+

Requests per minute: {info.rpm_limit ?? "No key limit"}

+

Tokens per minute: {info.tpm_limit ?? "No key limit"}

+

Expires: {info.expires ? runTime(info.expires) : "No expiry"}

+

Team, organization, and model limits still apply.

+
+
+
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/DurationInput.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/DurationInput.tsx index 7e1227eac8b..ce37ee952cb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/DurationInput.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/DurationInput.tsx @@ -2,6 +2,7 @@ import { useId, useState } from "react"; import { Input } from "@/components/ui/input"; +import { ChevronDown } from "lucide-react"; export function DurationInput({ label, @@ -47,18 +48,24 @@ export function DurationInput({ value={Number.isFinite(value) ? value / scale : ""} onChange={(event) => onChange(event.target.value === "" ? NaN : Number(event.target.value) * scale)} /> - +
+ +
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensFinding.tsx b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensFinding.tsx new file mode 100644 index 00000000000..e1f0a47330b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/lens/_components/LensFinding.tsx @@ -0,0 +1,146 @@ +import { ArrowUpRight } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Textarea } from "@/components/ui/textarea"; +import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetDescription } from "@/components/ui/sheet"; +import { evidenceTarget, runTime, type Finding, type Sample } from "./lensData"; + +export function LensFinding({ + finding, + sampledRuns, + readOnly, + reason, + busy, + onClose, + onReason, + onEvidence, + changeFinding, +}: { + finding?: Finding; + sampledRuns: Sample["executions"]; + readOnly: boolean; + reason: string; + busy: boolean; + onClose: () => void; + onReason: (reason: string) => void; + onEvidence: (evidence: { id: string; span: string }) => void; + changeFinding: (status: Finding["status"]) => Promise; +}) { + const evidenceGroups = finding + ? [...new Set(finding.evidence.map((e) => e.execution_id))].map((id) => ({ + id, + run: sampledRuns.find((r) => r.id === id), + quotes: finding.evidence.filter((e) => e.execution_id === id), + })) + : []; + return ( + { + if (!open) onClose(); + }} + > + + {finding && ( + <> + + {finding.title} + + {finding.kind === "issue" ? `${finding.priority} priority` : "Pattern"} ·{" "} + {finding.occurrences?.length ?? 0} linked {finding.occurrences?.length === 1 ? "run" : "runs"} + + +
+
+

What happened

+

{finding.description}

+
+ {finding.suggestion && ( +
+

What to do next

+

{finding.suggestion}

+
+ )} + {finding.limitation && ( +
+ Evidence limits +

{finding.limitation}

+
+ )} +
+

Evidence by run

+

+ Exact quotes from the recorded activity. Counterexamples are labeled separately from supporting + evidence. +

+
+ {evidenceGroups.map((group) => ( +
+ + {group.run?.name ?? evidenceTarget(group.id)?.id.slice(0, 12) ?? "Recorded run"} + + {group.quotes.length} {group.quotes.length === 1 ? "quote" : "quotes"} + {group.run ? ` · ${runTime(group.run.start_time)}` : ""} + + +
+ {group.quotes.map((e, i) => ( +
+ {e.role === "counterexample" && ( +

Counterexample

+ )} +
+ {e.quote} +
+ +
+ ))} +
+
+ ))} +
+
+ {!readOnly && ( +
+