From 461a58c40a76d62cf402609f46921d7aa8a9998b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:39:47 -0700 Subject: [PATCH] refactor(lens): storage-independent trace reads, shared keyset pager, typed read failures (#44422) * feat(lens): own trace reads behind a cached TraceStore port Move storage-independent trace reads into litellm-traces-cache behind a TraceStore port that ClickHouse implements. One keyset pager drives the span, list span and spend reads, and a run list batch reads spend once. Trace opens, pages and list summaries share one resolved read per trace in an in-process cache with single-flight loading. Live traces and reads with unknown spend expire after 5s, quiet traces after 10 minutes, failed reads are never cached, and the accepted list page size is remembered per scope. Trace read failures map to their own status and code (400, 409, 413, 503 with Retry-After), and the trace drawer retries temporary failures while offering only a refresh for changed or oversized traces. Co-Authored-By: Claude Opus 5.5 * perf(lens): seed large profiles with server-side copies and long sessions Replay one copy through the proxy, then copy it inside ClickHouse and PostgreSQL with INSERT ... SELECT, rewriting trace, span and call IDs so every copy keeps its own spend. Add three long single-trace sessions for drawer paging and the oversized read path Co-Authored-By: Claude Opus 5.5 * chores * style(lens): float the investigation setup badge on the tab edge Co-Authored-By: Claude Opus 5.5 * refactor(lens): restructure the trace drawer and polish its layout Split the 547-line TraceDrawer into run/, tree/, span/, content/ and conversation/ modules. Step rows now sit on one line with colored span family tiles, and the per-row timing bar moved into an optional Waterfall layout with a time axis. The steps and details panes are separated by the shadcn Resizable handle, with the split remembered per orientation. Span payloads go through one pure classifier (payloadView) that picks messages, a tool result, a nested field tree or text. JSON-encoded field values unfold into a tree, prose renders as markdown, repr and tracebacks stay monospace, and every section offers a Raw view. LangChain's serialized messages now render as conversation cards. Co-Authored-By: Claude Opus 5.5 * typesafety * wip * fmt --------- Co-authored-by: Yujong Lee Co-authored-by: Claude Opus 5.5 --- Makefile | 2 +- deploy/lens/README.md | 7 +- litellm-rust/Cargo.lock | 8 +- litellm-rust/crates/python-bridge/Cargo.toml | 1 + .../crates/python-bridge/src/routes/traces.rs | 173 ++-- litellm-rust/crates/traces-cache/AGENTS.md | 6 +- litellm-rust/crates/traces-cache/Cargo.toml | 4 + litellm-rust/crates/traces-cache/src/cache.rs | 395 +++++++++ .../crates/traces-cache/src/cursor.rs | 124 +++ litellm-rust/crates/traces-cache/src/error.rs | 44 + litellm-rust/crates/traces-cache/src/lib.rs | 174 +--- litellm-rust/crates/traces-cache/src/list.rs | 197 +++++ .../crates/traces-cache/src/reader.rs | 365 +++++++++ litellm-rust/crates/traces-cache/src/spend.rs | 67 ++ litellm-rust/crates/traces-cache/src/store.rs | 62 ++ .../crates/traces-cache/tests/read.rs | 768 ++++++++++++++++++ .../crates/traces-cache/tests/snapshots.rs | 52 +- .../crates/traces-clickhouse/Cargo.toml | 3 - .../crates/traces-clickhouse/src/error.rs | 17 - .../crates/traces-clickhouse/src/insert.rs | 3 +- .../crates/traces-clickhouse/src/lib.rs | 2 +- .../crates/traces-clickhouse/src/reads.rs | 575 ++----------- .../crates/traces-clickhouse/src/schema.rs | 3 +- .../traces-clickhouse/src/span_batches.rs | 408 ++++++---- .../crates/traces-clickhouse/src/sql.rs | 6 +- .../traces-clickhouse/tests/migrations.rs | 6 +- .../crates/traces-clickhouse/tests/reads.rs | 324 ++++---- litellm-rust/crates/traces/src/query/named.rs | 18 +- litellm/constants.py | 1 + litellm/proxy/lens/endpoints.py | 9 +- litellm/proxy/lens/models.py | 19 +- litellm/proxy/lens/sources.py | 4 +- litellm/proxy/tracing_endpoints.py | 93 ++- litellm/rust_bridge/trace/errors.py | 5 + scripts/lens_dev.sh | 48 +- scripts/seed_request_logs.py | 427 ++++++++++ scripts/seed_tracing_fixtures.py | 286 +++++-- tests/test_litellm_rust/test_traces.py | 108 ++- tests/unit/proxy/lens/test_endpoints.py | 27 +- tests/unit/proxy/test_tracing_endpoints.py | 40 +- tests/unit/test_lens_dev.py | 5 +- tests/unit/test_seed_tracing_fixtures.py | 58 +- ui/litellm-dashboard/package-lock.json | 11 + ui/litellm-dashboard/package.json | 1 + ui/litellm-dashboard/src/app/globals.css | 21 +- .../src/components/lens/LensModeSwitch.tsx | 2 +- .../lens/LensPage.integration.test.tsx | 4 +- .../lens/LensSetup.integration.test.tsx | 35 +- .../lens/LensWorkspace.integration.test.tsx | 27 +- .../src/components/lens/data/LensServices.tsx | 3 +- .../src/components/lens/data/service.ts | 134 ++- .../lens/investigations/Evidence.tsx | 2 +- .../FindingDetails.integration.test.tsx | 2 +- .../InvestigationsView.integration.test.tsx | 118 ++- .../lens/onboarding/LensGettingStarted.tsx | 4 +- .../lens/onboarding/LensIntroDialog.tsx | 2 +- .../WorkerSettings.integration.test.tsx | 126 +-- .../lens/settings/worker/usePrepareWorker.ts | 3 +- .../InvestigationSetup.integration.test.tsx | 48 +- .../lens/traces/detail/AttributesDetail.tsx | 44 - .../lens/traces/detail/DetailContent.tsx | 245 ------ .../lens/traces/detail/DetailPane.tsx | 240 ------ .../lens/traces/detail/KeyValueRows.tsx | 103 --- .../lens/traces/detail/MessageCard.tsx | 221 ----- .../lens/traces/detail/RequestDetail.tsx | 79 -- .../lens/traces/detail/SpanTree.tsx | 415 ---------- .../lens/traces/detail/TraceDrawer.tsx | 497 ------------ .../lens/traces/detail/content/Block.tsx | 95 +++ .../lens/traces/detail/content/ContentTab.tsx | 105 +++ .../lens/traces/detail/content/FieldTree.tsx | 139 ++++ .../lens/traces/detail/content/Markdown.tsx | 60 ++ .../Messages.test.tsx} | 10 +- .../lens/traces/detail/content/Messages.tsx | 148 ++++ .../traces/detail/content/PayloadBody.tsx | 54 ++ .../lens/traces/detail/content/Section.tsx | 81 ++ .../lens/traces/detail/content/SpanError.tsx | 123 +++ .../traces/detail/content/payload.test.ts | 155 ++++ .../lens/traces/detail/content/payload.ts | 98 +++ .../TraceConversation.integration.test.tsx | 14 +- .../{ => conversation}/TraceConversation.tsx | 11 +- .../{ => conversation}/conversation.test.ts | 4 +- .../detail/{ => conversation}/conversation.ts | 14 +- .../lens/traces/detail/run/PagingBanner.tsx | 64 ++ .../lens/traces/detail/run/RunBody.tsx | 121 +++ .../lens/traces/detail/run/RunHeader.tsx | 117 +++ .../RunView.test.tsx} | 112 ++- .../lens/traces/detail/run/RunView.tsx | 174 ++++ .../lens/traces/detail/run/useRunTree.ts | 147 ++++ .../lens/traces/detail/span/AttributesTab.tsx | 36 + .../lens/traces/detail/span/DetailGroup.tsx | 8 + .../DetailPane.integration.test.tsx | 85 +- .../lens/traces/detail/span/DetailPane.tsx | 37 + .../lens/traces/detail/span/GroupPane.tsx | 74 ++ .../lens/traces/detail/span/PaneHeader.tsx | 55 ++ .../lens/traces/detail/span/RequestTab.tsx | 80 ++ .../lens/traces/detail/span/SpanPane.tsx | 85 ++ .../detail/{ => tree}/SpanHoverCard.tsx | 12 +- .../lens/traces/detail/tree/SpanTree.tsx | 259 ++++++ .../lens/traces/detail/tree/TreeRows.tsx | 260 ++++++ .../lens/traces/detail/tree/timeline.test.ts | 56 ++ .../lens/traces/detail/tree/timeline.ts | 34 + .../AgentTracesSection.integration.test.tsx | 2 +- .../lens/traces/list/AgentTracesSection.tsx | 2 +- .../lens/traces/list/TimeRangeControls.tsx | 4 +- .../lens/traces/list/runSearch/RunSearch.tsx | 6 +- .../traces/list/runSearch/RunsToolbar.tsx | 4 +- .../lens/traces/list/traceReadFailure.test.ts | 61 ++ .../lens/traces/list/traceReadFailure.ts | 58 ++ .../src/components/lens/traces/ui/PaneBar.tsx | 5 - .../components/lens/traces/ui/SpanIcon.tsx | 47 +- .../src/components/lens/traces/utils.test.ts | 15 + .../src/components/lens/traces/utils.ts | 35 +- .../src/components/ui/Resizable.tsx | 41 + ui/litellm-dashboard/src/lib/http/api.ts | 4 +- ui/litellm-dashboard/src/lib/http/client.ts | 17 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 43 +- .../tests/lens-test-utils.tsx | 62 ++ 117 files changed, 7190 insertions(+), 3474 deletions(-) create mode 100644 litellm-rust/crates/traces-cache/src/cache.rs create mode 100644 litellm-rust/crates/traces-cache/src/cursor.rs create mode 100644 litellm-rust/crates/traces-cache/src/list.rs create mode 100644 litellm-rust/crates/traces-cache/src/reader.rs create mode 100644 litellm-rust/crates/traces-cache/src/spend.rs create mode 100644 litellm-rust/crates/traces-cache/src/store.rs create mode 100644 litellm-rust/crates/traces-cache/tests/read.rs create mode 100644 litellm/rust_bridge/trace/errors.py create mode 100644 scripts/seed_request_logs.py delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/AttributesDetail.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/DetailContent.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/DetailPane.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/KeyValueRows.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/MessageCard.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/RequestDetail.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/SpanTree.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/TraceDrawer.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/Block.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/ContentTab.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/FieldTree.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/Markdown.tsx rename ui/litellm-dashboard/src/components/lens/traces/detail/{MessageCard.test.tsx => content/Messages.test.tsx} (93%) create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/Messages.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/PayloadBody.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/Section.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/SpanError.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/payload.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/content/payload.ts rename ui/litellm-dashboard/src/components/lens/traces/detail/{ => conversation}/TraceConversation.integration.test.tsx (96%) rename ui/litellm-dashboard/src/components/lens/traces/detail/{ => conversation}/TraceConversation.tsx (96%) rename ui/litellm-dashboard/src/components/lens/traces/detail/{ => conversation}/conversation.test.ts (98%) rename ui/litellm-dashboard/src/components/lens/traces/detail/{ => conversation}/conversation.ts (96%) create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/run/PagingBanner.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/run/RunBody.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx rename ui/litellm-dashboard/src/components/lens/traces/detail/{TraceDrawer.test.tsx => run/RunView.test.tsx} (80%) create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/run/useRunTree.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/AttributesTab.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailGroup.tsx rename ui/litellm-dashboard/src/components/lens/traces/detail/{ => span}/DetailPane.integration.test.tsx (82%) create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/GroupPane.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx rename ui/litellm-dashboard/src/components/lens/traces/detail/{ => tree}/SpanHoverCard.tsx (95%) create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/tree/SpanTree.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/tree/TreeRows.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/tree/timeline.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/tree/timeline.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/list/traceReadFailure.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/list/traceReadFailure.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/traces/ui/PaneBar.tsx create mode 100644 ui/litellm-dashboard/src/components/ui/Resizable.tsx diff --git a/Makefile b/Makefile index ad84e2c4014..7c8511a44d7 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ help: @echo " make test-unit-helm - Run helm unit tests" @echo " make test-rust-extension - Build the Rust extension and run its public Python tests" @echo " make rust-sqlx-prepare - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container" - @echo " make lens-dev - Run proxy + Lens worker + hot-reload dashboard (ARGS=\"--seed large\", LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)" + @echo " make lens-dev - Run proxy + Lens worker + hot-reload dashboard (ARGS=\"--seed large --seed-logs\", LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)" @echo "" @echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide" @echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine." diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 6f8c7d3940e..60e9c9991a4 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -178,16 +178,15 @@ Creation queues the first batch. Posting to `/lens/{id}/runs` queues another, or The default is Next.js dev with no production build (`LENS_DEV_BUILD_UI=0`). Set `LENS_DEV_BUILD_UI=1` when you also want a fresh static dashboard at `http://localhost:4000/ui/`. Build output goes to `.lens-dev/logs/ui-build.log`; a failed build stops startup. Both modes keep the live dashboard on port 3000. Startup checks the live login route before seeding and fails with the UI log path if Next.js exits. `LENS_DEV_STARTUP_TIMEOUT_SECONDS` controls startup readiness retries (default 300; `LENS_DEV_READINESS_REQUEST_TIMEOUT_SECONDS` caps each HTTP probe, default 5) -For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS="--seed large"` for 2,000 fixture copies, over one million spans and linked request logs. To seed a running stack without restarting it, use `make lens-dev ARGS="--seed-only --seed large --copies 100"`. The default profile replays one copy of every checked-in capture through authenticated `/v1/traces`, including failures, retries, streaming and multiple agent frameworks. Large seeds use the same parser and compressed ClickHouse writer in batches of four copies, and write matching request logs to PostgreSQL. The first and last batches verify linked spend totals through the proxy +For local fixture data, run `make lens-dev ARGS=--seed`. Use `make lens-dev ARGS="--seed large"` for 2,000 fixture copies spread over the last 24 hours, about 860,000 spans with linked request logs, plus three long sessions of roughly 1,150, 9,200 and 92,000 spans in a single trace for drawer paging and the oversized read path. Their trace IDs are printed at the end. To seed a running stack without restarting it, use `make lens-dev ARGS="--seed-only --seed large --copies 100"`. Every profile replays one copy of every checked-in capture through authenticated `/v1/traces`, including failures, retries, streaming and multiple agent frameworks, and verifies linked spend totals through the proxy. Large seeds then copy that first copy inside ClickHouse and PostgreSQL with `INSERT ... SELECT`, rewriting trace, span and call IDs so each copy keeps its own spend, and verify the last copy through the proxy -Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies, and `LENS_DEV_SEED_BATCH_COPIES` overrides copies per bulk insert (default 4, about 2,000 spans). Start with four or fewer on a constrained machine. Larger batches still respect the existing ClickHouse insert size limit; each capture is decoded separately within the OTLP safety budget. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key +Seeds append fresh IDs on every invocation and spread copies over recent timestamps. Restarts without `SEED` do not add data. Lens excludes activity received in the last two minutes, so wait two minutes after seeding before checking investigation previews. `LENS_DEV_SEED_COPIES` overrides total copies. Large seeds test data volume and pagination, rather than concurrent ingestion throughput or review accuracy. They can use substantial disk space; adjust `--copies` for your machine. Seeding expects the generated local tracing configuration. The old `run_tracing_proxy_local.sh --seed` command forwards to Lens dev, using its ports and saved master key Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse variables before starting the proxy and seeder so both processes use the same settings. Invalid, zero and negative values fail instead of silently falling back. Changing these limits does not require rebuilding Rust | Environment variable | Default | Controls | | --- | --- | --- | | `LENS_DEV_SEED_COPIES` | 1 default, 2000 large | Total fixture copies | -| `LENS_DEV_SEED_BATCH_COPIES` | 4 | Copies per bulk insert | | `LENS_DEV_SEED_TIMEOUT_SECONDS` | 120 | Seeder HTTP timeout | | `OTLP_MAX_BODY_BYTES` | 16777216 | HTTP body and decompressed payload bytes | | `OTLP_MAX_CONCURRENT_INGESTS` | 2 | Concurrent proxy ingestion requests | @@ -202,7 +201,7 @@ Local ingestion limits are explicit and configurable. Set OTLP and ClickHouse va | `CLICKHOUSE_TRACE_MAX_INSERT_BYTES` | 67108864 | Encoded trace or spend insert bytes | | `CLICKHOUSE_INSERT_TIMEOUT_SECONDS` | 30 | ClickHouse insert HTTP timeout | -The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits. Bulk seeding parses each capture separately, keeping the per-export limits distinct from the bulk insert limit. Use smaller batches if an insert exceeds its byte budget. For example, `LENS_DEV_SEED_COPIES=100 LENS_DEV_SEED_BATCH_COPIES=2 make lens-dev ARGS="--seed large"` +The wire parsers also enforce their library recursion limits (128 levels for JSON, 100 for protobuf). Raising the configured depth does not remove those parser limits. ## Quality evaluation diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 348beeb3813..f7b667c8ab2 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4156,6 +4156,7 @@ dependencies = [ "litellm-storage-clickhouse", "litellm-token-counter", "litellm-traces", + "litellm-traces-cache", "litellm-traces-clickhouse", "litellm-tracing", "prost", @@ -4474,13 +4475,17 @@ dependencies = [ name = "litellm-traces-cache" version = "0.1.0" dependencies = [ + "base64 0.22.1", "litellm-traces", "moka", "rstest", + "serde", "serde_json", "sha2 0.10.9", "thiserror 2.0.19", + "time", "tokio", + "tracing", ] [[package]] @@ -4488,11 +4493,9 @@ name = "litellm-traces-clickhouse" version = "0.1.0" dependencies = [ "askama", - "base64 0.22.1", "flate2", "futures-util", "hmac 0.12.1", - "itertools 0.14.0", "jsonschema", "litellm-http", "litellm-migrate", @@ -4511,7 +4514,6 @@ dependencies = [ "thiserror 2.0.19", "time", "tokio", - "tracing", "url", "wiremock", ] diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index c3a86009111..226cecd5b55 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -22,6 +22,7 @@ tiktoken = ["litellm-token-counter/tiktoken"] fancy-regex.workspace = true litellm-tracing.workspace = true litellm-traces.workspace = true +litellm-traces-cache.workspace = true litellm-traces-clickhouse.workspace = true litellm-storage-clickhouse.workspace = true litellm-host.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index dd5c6b9860f..f442d6c31de 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -1,8 +1,11 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Arc}; use litellm_http::ClientVariant; use litellm_traces::{QueryScope, ReadQuery, Tenant, query::named::ReadAccessParams}; -use litellm_traces_clickhouse::{Config, Error, InsertTable, Parameter, QueryReaders}; +use litellm_traces_cache::{ReadError, TraceReader}; +use litellm_traces_clickhouse::{ + ClickHouseTraces, Config, Error, InsertTable, Parameter, QueryReaders, +}; use prost::Message; use pyo3::{ exceptions::{PyOverflowError, PyRuntimeError, PyValueError}, @@ -10,6 +13,8 @@ use pyo3::{ types::PyBytes, }; +pyo3::import_exception!(litellm.rust_bridge.trace.errors, TraceChanged); + #[derive(Message)] struct OtlpErrorStatus { #[prost(int32, tag = "1")] @@ -35,15 +40,12 @@ fn map_error_ref(error: &Error) -> PyErr { use litellm_storage_clickhouse::Error as StorageError; match error { - Error::Decode(litellm_traces::Error::TooLarge) - | Error::InsertTooLarge - | Error::ReadTooLarge => PyOverflowError::new_err(error.to_string()), + Error::Decode(litellm_traces::Error::TooLarge) | Error::InsertTooLarge => { + PyOverflowError::new_err(error.to_string()) + } Error::InvalidRow | Error::InvalidLimit(_) | Error::InvalidTable - | Error::InvalidCursor(_) - | Error::AmbiguousTrace - | Error::TraceChanged | Error::Decode(_) | Error::InvalidSchema | Error::InvalidQuery @@ -78,6 +80,18 @@ fn map_error_ref(error: &Error) -> PyErr { } } +fn map_read_error(error: ReadError) -> PyErr { + match error { + error @ (ReadError::InvalidParameters + | ReadError::InvalidCursor(_) + | ReadError::AmbiguousTrace) => PyValueError::new_err(error.to_string()), + error @ ReadError::TraceChanged => TraceChanged::new_err(error.to_string()), + error @ ReadError::TooLarge => PyOverflowError::new_err(error.to_string()), + error @ ReadError::Encode(_) => PyRuntimeError::new_err(error.to_string()), + ReadError::Store(error) => map_error_ref(&error), + } +} + fn map_sql_error(error: Error) -> PyErr { match error { Error::Storage(litellm_storage_clickhouse::Error::QueryFailed(400 | 404)) => { @@ -112,6 +126,7 @@ impl NativeTraceConfig { pub struct NativeTraceStorage { config: Config, query_readers: QueryReaders, + reader: Arc, } #[pymethods] @@ -123,6 +138,9 @@ impl NativeTraceStorage { config.inner.storage().writer().clone(), config.inner.storage().database().to_owned(), ), + reader: Arc::new(TraceReader::new( + litellm_storage_clickhouse::READ_LIMITS.response_bytes, + )), config: config.inner.clone(), }) } @@ -218,21 +236,16 @@ impl NativeTraceStorage { ) -> PyResult> { let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; let connection = self.config.storage().reader().clone(); + let reader = Arc::clone(&self.reader); crate::execution::run_async( py, async move { - litellm_traces_clickhouse::list_traces( - &client, - &connection, - &scope, - start_ms, - end_ms, - cursor.as_deref(), - limit, - ) - .await + let store = ClickHouseTraces::new(client, connection); + reader + .list_traces(&store, &scope, start_ms, end_ms, cursor.as_deref(), limit) + .await }, - map_error, + map_read_error, ) } @@ -248,34 +261,31 @@ impl NativeTraceStorage { ) -> PyResult> { let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; let connection = self.config.storage().reader().clone(); + let reader = Arc::clone(&self.reader); crate::execution::run_async( py, async move { + let store = ClickHouseTraces::new(client, connection); if let Some(page_size) = page_size { - litellm_traces_clickhouse::get_trace_page( - &client, - &connection, - &scope, - &trace_id, - &trace_ref, - cursor.as_deref(), - page_size, - ) - .await + reader + .get_trace_page( + &store, + &scope, + &trace_id, + &trace_ref, + cursor.as_deref(), + page_size, + ) + .await } else if cursor.is_some() { - Err(Error::InvalidParameters) + Err(ReadError::InvalidParameters) } else { - litellm_traces_clickhouse::get_trace( - &client, - &connection, - &scope, - &trace_id, - &trace_ref, - ) - .await + reader + .get_trace(&store, &scope, &trace_id, &trace_ref) + .await } }, - map_error, + map_read_error, ) } @@ -289,20 +299,16 @@ impl NativeTraceStorage { ) -> PyResult> { let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; let connection = self.config.storage().reader().clone(); + let reader = Arc::clone(&self.reader); crate::execution::run_async( py, async move { - litellm_traces_clickhouse::get_span( - &client, - &connection, - &scope, - &trace_id, - &span_id, - &trace_ref, - ) - .await + let store = ClickHouseTraces::new(client, connection); + reader + .get_span(&store, &scope, &trace_id, &span_id, &trace_ref) + .await }, - map_error, + map_read_error, ) } @@ -318,21 +324,23 @@ impl NativeTraceStorage { ) -> PyResult> { let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; let connection = self.config.storage().reader().clone(); + let reader = Arc::clone(&self.reader); crate::execution::run_async( py, async move { - litellm_traces_clickhouse::get_span_error( - &client, - &connection, - &scope, - &trace_id, - &span_id, - &trace_ref, - cursor.as_deref(), - ) - .await + let store = ClickHouseTraces::new(client, connection); + reader + .get_span_error( + &store, + &scope, + &trace_id, + &span_id, + &trace_ref, + cursor.as_deref(), + ) + .await }, - map_error, + map_read_error, ) } @@ -495,11 +503,7 @@ mod tests { Error::Decode(litellm_traces::Error::InvalidLimit("OTLP_MAX_SPANS")), "ValueError" )] - #[case::cursor(Error::InvalidCursor("trace"), "ValueError")] - #[case::ambiguous(Error::AmbiguousTrace, "ValueError")] - #[case::changed_snapshot(Error::TraceChanged, "ValueError")] - #[case::read_budget(Error::ReadTooLarge, "OverflowError")] - fn trace_read_and_ingest_failures_preserve_public_exception_types( + fn trace_ingest_failures_preserve_public_exception_types( #[case] error: Error, #[case] exception_name: &str, ) { @@ -511,4 +515,43 @@ mod tests { ); }); } + + #[rstest] + #[case::invalid_parameters(ReadError::InvalidParameters, "ValueError")] + #[case::invalid_cursor(ReadError::InvalidCursor("trace"), "ValueError")] + #[case::ambiguous(ReadError::AmbiguousTrace, "ValueError")] + #[case::changed_snapshot(ReadError::TraceChanged, "TraceChanged")] + #[case::read_budget(ReadError::TooLarge, "OverflowError")] + #[case::encode( + ReadError::Encode(Arc::new(serde_json::Error::io(std::io::Error::other("invalid")))), + "RuntimeError" + )] + #[case::store(ReadError::Store(Arc::new(Error::InvalidScope)), "ValueError")] + fn trace_read_failures_preserve_public_exception_types( + #[case] error: ReadError, + #[case] exception_name: &str, + ) { + Python::initialize(); + Python::attach(|py| { + let repository = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(3) + .unwrap() + .to_str() + .unwrap(); + pyo3::types::PyModule::import(py, "sys") + .unwrap() + .getattr("path") + .unwrap() + .call_method1("insert", (0, repository)) + .unwrap(); + let message = error.to_string(); + let exception = map_read_error(error); + assert_eq!(exception.get_type(py).name().unwrap(), exception_name); + assert_eq!( + exception.value(py).str().unwrap().to_str().unwrap(), + message + ); + }); + } } diff --git a/litellm-rust/crates/traces-cache/AGENTS.md b/litellm-rust/crates/traces-cache/AGENTS.md index bfaea42d901..7e4caacd15e 100644 --- a/litellm-rust/crates/traces-cache/AGENTS.md +++ b/litellm-rust/crates/traces-cache/AGENTS.md @@ -1,5 +1,5 @@ -Own resolved-trace snapshot storage, cache identity, weighting, and expiry -Depend on trace domain types, never storage, HTTP, or Python +Own storage-independent trace reads over `TraceStore`: the in-process read cache (identity, single-flight, freshness expiry, weighting), cursor formats, paging, response splitting, spend windows and run batching +Depend on trace domain types, never storage, HTTP or Python Preserve the full source and authorization scope in every cache key Keep snapshots immutable and expose borrowed data -Keep cursor formats and database reads in their existing owners +Storage adapters implement `TraceStore`; keep SQL and row encoding there diff --git a/litellm-rust/crates/traces-cache/Cargo.toml b/litellm-rust/crates/traces-cache/Cargo.toml index 3e58ea4c965..16eb01c6c8a 100644 --- a/litellm-rust/crates/traces-cache/Cargo.toml +++ b/litellm-rust/crates/traces-cache/Cargo.toml @@ -6,11 +6,15 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true litellm-traces.workspace = true moka.workspace = true +serde.workspace = true serde_json.workspace = true sha2.workspace = true +time.workspace = true thiserror.workspace = true +tracing.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs new file mode 100644 index 00000000000..f308714146e --- /dev/null +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -0,0 +1,395 @@ +use std::{future::Future, sync::Arc, time::Duration}; + +use litellm_traces::{ + Trace, TraceSummary, + query::named::{ReadAccessParams, TraceSpansRow}, +}; +use moka::{Expiry, future::Cache}; +use serde::Serialize; +use sha2::{Digest, Sha256}; + +use crate::Error; + +pub const LIVE_TTL: Duration = Duration::from_secs(5); +pub const SETTLED_TTL: Duration = Duration::from_secs(10 * 60); +const SETTLED_AFTER_MS: u64 = 5 * 60 * 1000; +const MAX_INDEX_ENTRIES: u64 = 100_000; + +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct SnapshotKey(String); + +impl SnapshotKey { + fn digest(fields: &impl Serialize) -> Result { + Ok(Self(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(fields)?) + ))) + } + + pub fn new( + source: &str, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, + snapshot_ms: u64, + ) -> Result { + Self::digest(&(source, access, trace_id, trace_ref, snapshot_ms)) + } + + pub fn latest( + source: &str, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, + ) -> Result { + Self::digest(&(source, access, trace_id, trace_ref)) + } + + pub(crate) fn run( + source: &str, + access: &ReadAccessParams, + run: (&str, &str, &str, &str), + ) -> Result { + Self::digest(&("run", source, access, run)) + } + + pub(crate) fn scope(source: &str, access: &ReadAccessParams) -> Result { + Self::digest(&("scope", source, access)) + } +} + +/// How long a read result stays reusable: traces still receiving spans, or read with spend +/// unavailable, are re-read after `LIVE_TTL`; traces quiet for `SETTLED_AFTER_MS` are kept for +/// `SETTLED_TTL`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Freshness { + Live, + Settled, +} + +impl Freshness { + pub fn of(rows: &[TraceSpansRow], spend_known: bool, snapshot_ms: u64) -> Self { + let last_end_ms = rows + .iter() + .map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns) / 1_000_000) + .max() + .unwrap_or(i64::MAX); + let quiet_ms = i64::try_from(snapshot_ms) + .unwrap_or(i64::MAX) + .saturating_sub(last_end_ms); + if spend_known && quiet_ms >= SETTLED_AFTER_MS as i64 { + Self::Settled + } else { + Self::Live + } + } + + fn ttl(self) -> Duration { + match self { + Self::Live => LIVE_TTL, + Self::Settled => SETTLED_TTL, + } + } +} + +trait Fresh { + fn freshness(&self) -> Freshness; +} + +struct ByFreshness; + +impl Expiry for ByFreshness { + fn expire_after_create(&self, _: &K, value: &V, _: std::time::Instant) -> Option { + Some(value.freshness().ttl()) + } +} + +pub struct Snapshot { + trace: Trace, + version: String, + snapshot_ms: u64, + freshness: Freshness, + weight: u32, +} + +impl Snapshot { + pub fn trace(&self) -> &Trace { + &self.trace + } + + pub fn version(&self) -> &str { + &self.version + } + + pub fn snapshot_ms(&self) -> u64 { + self.snapshot_ms + } + + pub fn freshness(&self) -> Freshness { + self.freshness + } +} + +#[derive(Clone, Copy)] +struct Latest { + snapshot_ms: u64, + freshness: Freshness, +} + +impl Fresh for Latest { + fn freshness(&self) -> Freshness { + self.freshness + } +} + +/// Resolved trace snapshots pinned by `snapshot_ms` for paging, plus which snapshot each trace +/// currently serves so repeated opens reuse one read until its freshness expires. +pub struct SnapshotCache { + pinned: Cache>, + latest: Cache, + max_graph_bytes: usize, +} + +impl SnapshotCache { + pub fn new(max_graph_bytes: usize, idle: Duration) -> Self { + Self { + pinned: Cache::builder() + .max_capacity((max_graph_bytes as u64).saturating_mul(2)) + .weigher(|_: &SnapshotKey, snapshot: &Arc| snapshot.weight) + .time_to_idle(idle) + .build(), + latest: Cache::builder() + .max_capacity(MAX_INDEX_ENTRIES) + .expire_after(ByFreshness) + .build(), + max_graph_bytes, + } + } + + pub async fn get(&self, key: &SnapshotKey) -> Option> { + self.pinned.get(key).await + } + + /// Returns the snapshot pinned at `key`, running `load` once for all concurrent callers on a + /// miss. A failed load is not cached. + pub async fn pinned_or_load( + &self, + key: SnapshotKey, + snapshot_ms: u64, + load: F, + ) -> Result, Arc> + where + E: From + Send + Sync + 'static, + F: Future>, + { + self.pinned + .try_get_with(key, async { + let (trace, freshness) = load.await?; + Ok(Arc::new(self.snapshot(trace, snapshot_ms, freshness)?)) + }) + .await + } + + /// Returns the snapshot `latest` currently serves. On a miss, `load_at(now_ms)` runs once for + /// all concurrent callers and its snapshot is served until its freshness expires. + pub async fn latest_or_load( + &self, + latest: SnapshotKey, + now_ms: u64, + load_at: F, + ) -> Result, Arc> + where + E: Send + Sync + 'static, + F: Fn(u64) -> Fut, + Fut: Future, Arc>>, + { + let entry = self + .latest + .try_get_with(latest, async { + let snapshot = load_at(now_ms).await?; + Ok::<_, Arc>(Latest { + snapshot_ms: snapshot.snapshot_ms, + freshness: snapshot.freshness, + }) + }) + .await + .map_err(|error| Arc::clone(&*error))?; + load_at(entry.snapshot_ms).await + } + + fn snapshot( + &self, + trace: Trace, + snapshot_ms: u64, + freshness: Freshness, + ) -> Result { + let encoded = serde_json::to_vec(&trace)?; + if encoded.len() > self.max_graph_bytes { + return Err(Error::ReadTooLarge); + } + let span_ids: Vec<&str> = trace + .spans + .iter() + .map(|span| span.span_id.as_str()) + .collect(); + let version = format!("{:x}", Sha256::digest(serde_json::to_vec(&span_ids)?)); + Ok(Snapshot { + trace, + version, + snapshot_ms, + freshness, + weight: u32::try_from(encoded.len().saturating_mul(2)).unwrap_or(u32::MAX), + }) + } + + #[cfg(test)] + async fn weighted_size(&self) -> u64 { + self.pinned.run_pending_tasks().await; + self.pinned.weighted_size() + } +} + +#[derive(Clone)] +pub(crate) enum ListedRun { + Resolved(Box, Freshness), + Limited, +} + +impl Fresh for ListedRun { + fn freshness(&self) -> Freshness { + match self { + Self::Resolved(_, freshness) => *freshness, + Self::Limited => Freshness::Settled, + } + } +} + +pub(crate) struct ListCache { + pub(crate) runs: Cache, + pub(crate) limits: Cache, +} + +impl ListCache { + pub(crate) fn new() -> Self { + Self { + runs: Cache::builder() + .max_capacity(MAX_INDEX_ENTRIES) + .expire_after(ByFreshness) + .build(), + limits: Cache::builder() + .max_capacity(MAX_INDEX_ENTRIES) + .time_to_live(SETTLED_TTL) + .build(), + } + } +} + +#[cfg(test)] +mod tests { + use litellm_traces::{ + SpanStatus, + query::named::{SpendByResponseIdsRow, TraceSpansRow}, + resolve_trace, + }; + use rstest::rstest; + + use super::*; + + fn row(span_id: &str) -> TraceSpansRow { + TraceSpansRow { + trace_id: String::new(), + span_id: span_id.into(), + parent_span_id: String::new(), + name: "run".into(), + kind: litellm_traces::ObservationType::Agent, + wrapper_candidate: false, + agent: "agent".into(), + framework: String::new(), + status: SpanStatus::Ok, + status_message: String::new(), + error_truncated: false, + start_ns: 1_790_742_989_000_000_000, + duration_ns: 10_000_000, + service: "agent-demo".into(), + input_preview: format!("input of {span_id}"), + model: String::new(), + input_tokens: 0, + output_tokens: 0, + litellm_request_id: String::new(), + call_keys: Vec::new(), + call_evidence: None, + tool_call_id: String::new(), + team_id: String::new(), + api_key_hash: String::new(), + user_id: String::new(), + } + } + + fn trace(span_id: &str) -> Trace { + resolve_trace( + "trace", + "ref", + &[row(span_id)], + &[] as &[SpendByResponseIdsRow], + ) + .expect("fixture should resolve") + } + + fn key(suffix: &str) -> SnapshotKey { + SnapshotKey::new( + "source", + &ReadAccessParams { + all_teams: false, + user_id: String::new(), + team_ids: vec!["team".into()], + }, + suffix, + "ref", + 100, + ) + .unwrap() + } + + #[tokio::test] + async fn weighted_capacity_bounds_retained_snapshots() { + let limit = ["first", "second", "third"] + .iter() + .map(|span_id| serde_json::to_vec(&trace(span_id)).unwrap().len()) + .max() + .unwrap(); + let cache = SnapshotCache::new(limit, Duration::from_secs(120)); + + for (key, span_id) in [ + (key("a"), "first"), + (key("b"), "second"), + (key("c"), "third"), + ] { + cache + .pinned_or_load(key, 100, async { + Ok::<_, Error>((trace(span_id), Freshness::Settled)) + }) + .await + .unwrap(); + } + + assert!(cache.weighted_size().await <= (limit as u64) * 2); + } + + const LAST_END_MS: u64 = 1_790_742_989_010; + + #[rstest] + #[case::just_ended(LAST_END_MS, true, Freshness::Live)] + #[case::quiet_just_under(LAST_END_MS + SETTLED_AFTER_MS - 1, true, Freshness::Live)] + #[case::quiet_long_enough(LAST_END_MS + SETTLED_AFTER_MS, true, Freshness::Settled)] + #[case::spend_unknown(LAST_END_MS + SETTLED_AFTER_MS, false, Freshness::Live)] + fn freshness_settles_once_spans_stop_and_spend_is_known( + #[case] snapshot_ms: u64, + #[case] spend_known: bool, + #[case] expected: Freshness, + ) { + assert_eq!( + Freshness::of(&[row("root")], spend_known, snapshot_ms), + expected + ); + } +} diff --git a/litellm-rust/crates/traces-cache/src/cursor.rs b/litellm-rust/crates/traces-cache/src/cursor.rs new file mode 100644 index 00000000000..8858f243f09 --- /dev/null +++ b/litellm-rust/crates/traces-cache/src/cursor.rs @@ -0,0 +1,124 @@ +use base64::{Engine, engine::general_purpose::URL_SAFE}; +use serde::{Deserialize, Serialize}; + +use crate::ReadError; + +pub(super) fn encode_cursor(position: &T) -> String { + URL_SAFE.encode(serde_json::to_vec(position).unwrap_or_default()) +} + +pub(super) fn decode_cursor Deserialize<'de>, E>( + cursor: &str, + kind: &'static str, +) -> Result> { + URL_SAFE + .decode(cursor) + .ok() + .and_then(|json| serde_json::from_slice(&json).ok()) + .ok_or(ReadError::InvalidCursor(kind)) +} + +pub(super) fn trace_position(cursor: Option<&str>) -> Result<(i64, String), ReadError> { + let Some(cursor) = cursor.filter(|cursor| !cursor.is_empty()) else { + return Ok((0, String::new())); + }; + match decode_cursor::<(i64, String), E>(cursor, "trace")? { + (start_ms, trace_ref) if start_ms > 0 && !trace_ref.is_empty() => Ok((start_ms, trace_ref)), + _ => Err(ReadError::InvalidCursor("trace")), + } +} + +#[derive(Deserialize, Serialize)] +pub(super) struct ErrorPosition { + pub(super) offset: u64, + pub(super) version: String, +} + +pub(super) fn error_position( + cursor: Option<&str>, +) -> Result, ReadError> { + let Some(cursor) = cursor else { + return Ok(None); + }; + let position: ErrorPosition = decode_cursor(cursor, "diagnostic")?; + let valid_version = position.version.len() == 64 + && position + .version + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte)); + if i64::try_from(position.offset).is_err() || !valid_version { + return Err(ReadError::InvalidCursor("diagnostic")); + } + Ok(Some(position)) +} + +#[derive(Deserialize, Serialize)] +pub(super) struct SpanPosition { + pub(super) trace_ref: String, + pub(super) snapshot_ms: u64, + pub(super) offset: usize, + pub(super) version: String, +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + fn trace_cursor_round_trips_the_last_listed_run() { + let cursor = encode_cursor(&(1_790_742_989_377_i64, "4bad42b84e9de3ba46fc870185f8f023")); + assert_eq!( + trace_position::(Some(&cursor)).unwrap(), + ( + 1_790_742_989_377, + "4bad42b84e9de3ba46fc870185f8f023".to_owned() + ) + ); + assert_eq!( + trace_position::(None).unwrap(), + (0, String::new()) + ); + assert_eq!( + trace_position::(Some("")).unwrap(), + (0, String::new()) + ); + } + + #[rstest] + #[case::not_base64("abc")] + #[case::not_json("bm90LWpzb24=")] + #[case::numeric_reference("WzEsIDJd")] + #[case::zero_start("WzAsICJ0Il0=")] + fn malformed_trace_cursors_are_rejected(#[case] cursor: &str) { + let result: Result<(i64, String), ReadError> = trace_position(Some(cursor)); + assert!(matches!(result, Err(ReadError::InvalidCursor("trace")))); + } + + #[rstest] + #[case::not_base64("garbage")] + #[case::missing_fields("e30=")] + #[case::not_an_object("WzEsMl0=")] + fn malformed_diagnostic_cursors_are_rejected(#[case] cursor: &str) { + let result: Result, ReadError> = + error_position(Some(cursor)); + assert!(matches!( + result, + Err(ReadError::InvalidCursor("diagnostic")) + )); + } + + #[rstest] + #[case::lowercase_version("a".repeat(64))] + #[case::short_version("A".repeat(63))] + fn diagnostic_cursor_requires_a_content_version(#[case] version: String) { + let cursor = encode_cursor(&ErrorPosition { offset: 1, version }); + let result: Result, ReadError> = + error_position(Some(&cursor)); + assert!(matches!( + result, + Err(ReadError::InvalidCursor("diagnostic")) + )); + } +} diff --git a/litellm-rust/crates/traces-cache/src/error.rs b/litellm-rust/crates/traces-cache/src/error.rs index ac5f19369fe..c7aaa3fd332 100644 --- a/litellm-rust/crates/traces-cache/src/error.rs +++ b/litellm-rust/crates/traces-cache/src/error.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error("trace snapshot serialization failed")] @@ -5,3 +7,45 @@ pub enum Error { #[error("trace snapshot exceeds the size limit")] ReadTooLarge, } + +/// Cheap to clone so one failed single-flight read can be returned to every waiting caller. +#[derive(Debug, thiserror::Error)] +pub enum ReadError { + #[error("invalid trace read parameters")] + InvalidParameters, + #[error("Invalid {0} cursor")] + InvalidCursor(&'static str), + #[error("Multiple traces have this ID; provide trace_ref")] + AmbiguousTrace, + #[error("Trace changed while paging; refresh the trace to continue")] + TraceChanged, + #[error("Trace exceeds the interactive read budget; use a filtered trace query")] + TooLarge, + #[error("trace could not be encoded")] + Encode(#[source] Arc), + #[error(transparent)] + Store(Arc), +} + +impl Clone for ReadError { + fn clone(&self) -> Self { + match self { + Self::InvalidParameters => Self::InvalidParameters, + Self::InvalidCursor(kind) => Self::InvalidCursor(kind), + Self::AmbiguousTrace => Self::AmbiguousTrace, + Self::TraceChanged => Self::TraceChanged, + Self::TooLarge => Self::TooLarge, + Self::Encode(error) => Self::Encode(Arc::clone(error)), + Self::Store(error) => Self::Store(Arc::clone(error)), + } + } +} + +impl From for ReadError { + fn from(error: Error) -> Self { + match error { + Error::ReadTooLarge => Self::TooLarge, + Error::Serialization(error) => Self::Encode(Arc::new(error)), + } + } +} diff --git a/litellm-rust/crates/traces-cache/src/lib.rs b/litellm-rust/crates/traces-cache/src/lib.rs index 7dfd3bf32a1..8ed79c1a547 100644 --- a/litellm-rust/crates/traces-cache/src/lib.rs +++ b/litellm-rust/crates/traces-cache/src/lib.rs @@ -1,166 +1,12 @@ -use std::{sync::Arc, time::Duration}; - -use litellm_traces::{Trace, query::named::ReadAccessParams}; -use moka::future::Cache; -use sha2::{Digest, Sha256}; - +mod cache; +mod cursor; mod error; +mod list; +mod reader; +mod spend; +mod store; -pub use error::Error; - -#[derive(Clone, Eq, Hash, PartialEq)] -pub struct SnapshotKey(String); - -impl SnapshotKey { - pub fn new( - source: &str, - access: &ReadAccessParams, - trace_id: &str, - trace_ref: &str, - snapshot_ms: u64, - ) -> Result { - let encoded = serde_json::to_vec(&(source, access, trace_id, trace_ref, snapshot_ms))?; - Ok(Self(format!("{:x}", Sha256::digest(encoded)))) - } -} - -pub struct Snapshot { - trace: Trace, - version: String, - weight: u32, -} - -impl Snapshot { - pub fn trace(&self) -> &Trace { - &self.trace - } - - pub fn version(&self) -> &str { - &self.version - } -} - -pub struct SnapshotCache { - entries: Cache>, - max_graph_bytes: usize, -} - -impl SnapshotCache { - pub fn new(max_graph_bytes: usize, ttl: Duration) -> Self { - Self { - entries: Cache::builder() - .max_capacity((max_graph_bytes as u64).saturating_mul(2)) - .weigher(|_: &SnapshotKey, snapshot: &Arc| snapshot.weight) - .time_to_live(ttl) - .build(), - max_graph_bytes, - } - } - - pub async fn get(&self, key: &SnapshotKey) -> Option> { - self.entries.get(key).await - } - - pub async fn insert(&self, key: SnapshotKey, trace: Trace) -> Result, Error> { - let encoded = serde_json::to_vec(&trace)?; - if encoded.len() > self.max_graph_bytes { - return Err(Error::ReadTooLarge); - } - - let span_ids: Vec<&str> = trace - .spans - .iter() - .map(|span| span.span_id.as_str()) - .collect(); - - let version = format!("{:x}", Sha256::digest(serde_json::to_vec(&span_ids)?)); - - let snapshot = Arc::new(Snapshot { - trace, - version, - weight: u32::try_from(encoded.len().saturating_mul(2)).unwrap_or(u32::MAX), - }); - - self.entries.insert(key, Arc::clone(&snapshot)).await; - Ok(snapshot) - } -} - -#[cfg(test)] -mod tests { - use litellm_traces::{ - SpanStatus, - query::named::{SpendByResponseIdsRow, TraceSpansRow}, - resolve_trace, - }; - - use super::*; - - fn trace(span_id: &str) -> Trace { - let rows = [TraceSpansRow { - trace_id: String::new(), - span_id: span_id.into(), - parent_span_id: String::new(), - name: "run".into(), - kind: litellm_traces::ObservationType::Agent, - wrapper_candidate: false, - agent: "agent".into(), - framework: String::new(), - status: SpanStatus::Ok, - status_message: String::new(), - error_truncated: false, - start_ns: 1_790_742_989_000_000_000, - duration_ns: 10_000_000, - service: "agent-demo".into(), - input_preview: format!("input of {span_id}"), - model: String::new(), - input_tokens: 0, - output_tokens: 0, - litellm_request_id: String::new(), - call_keys: Vec::new(), - call_evidence: None, - tool_call_id: String::new(), - team_id: String::new(), - api_key_hash: String::new(), - user_id: String::new(), - }]; - resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow]) - .expect("fixture should resolve") - } - - fn key(suffix: &str) -> SnapshotKey { - SnapshotKey::new( - "source", - &ReadAccessParams { - all_teams: false, - user_id: String::new(), - team_ids: vec!["team".into()], - }, - suffix, - "ref", - 100, - ) - .unwrap() - } - - #[tokio::test] - async fn weighted_capacity_bounds_retained_snapshots() { - let limit = ["first", "second", "third"] - .iter() - .map(|span_id| serde_json::to_vec(&trace(span_id)).unwrap().len()) - .max() - .unwrap(); - let cache = SnapshotCache::new(limit, Duration::from_secs(120)); - - for (key, span_id) in [ - (key("a"), "first"), - (key("b"), "second"), - (key("c"), "third"), - ] { - cache.insert(key, trace(span_id)).await.unwrap(); - } - - cache.entries.run_pending_tasks().await; - assert!(cache.entries.weighted_size() <= (limit as u64) * 2); - } -} +pub use cache::{Freshness, LIVE_TTL, SETTLED_TTL, Snapshot, SnapshotCache, SnapshotKey}; +pub use error::{Error, ReadError}; +pub use reader::{MAX_GRAPH_BYTES, MAX_GRAPH_SPANS, TraceReader}; +pub use store::{StoreError, TraceStore}; diff --git a/litellm-rust/crates/traces-cache/src/list.rs b/litellm-rust/crates/traces-cache/src/list.rs new file mode 100644 index 00000000000..1b1dcbbefc3 --- /dev/null +++ b/litellm-rust/crates/traces-cache/src/list.rs @@ -0,0 +1,197 @@ +use std::collections::HashMap; + +use crate::{ + ReadError, SnapshotKey, TraceReader, TraceStore, + cache::{Freshness, ListedRun}, + reader::{map_store_error, now_ms}, + spend::{spend, spend_window, spend_within}, + store::StoreError, +}; +use litellm_traces::{ + TraceSummary, listed_summary, + query::named::{ListTracesRow, ReadAccessParams, TracePageSpansParams, TraceSpansRow}, + resolve_trace, +}; + +const RUNS_PER_SPAN_READ: usize = 16; + +fn run_key(team_id: &str, api_key_hash: &str, trace_id: &str) -> (String, String, String) { + ( + team_id.to_owned(), + api_key_hash.to_owned(), + trace_id.to_owned(), + ) +} + +fn cache_key( + source: &str, + access: &ReadAccessParams, + row: &ListTracesRow, +) -> Result> { + Ok(SnapshotKey::run( + source, + access, + ( + &row.team_id, + &row.api_key_hash, + &row.trace_id, + &row.trace_ref, + ), + )?) +} + +fn summary(row: &ListTracesRow, listed: Option<&ListedRun>) -> TraceSummary { + match listed { + Some(ListedRun::Resolved(summary, _)) => (**summary).clone(), + Some(ListedRun::Limited) | None => listed_summary(row), + } +} + +/// Summaries for one batch of listed runs. Runs resolved within their freshness window come +/// from the cache; only the rest are read from storage, with one span and one spend read. +pub(super) async fn list_summaries( + reader: &TraceReader, + store: &S, + access: &ReadAccessParams, + runs: &[ListTracesRow], +) -> Result, ReadError> { + let mut keys = Vec::with_capacity(runs.len()); + let mut listed = Vec::with_capacity(runs.len()); + for row in runs { + let key = cache_key(store.source(), access, row)?; + listed.push(reader.lists.runs.get(&key).await); + keys.push(key); + } + let misses: Vec<&ListTracesRow> = runs + .iter() + .zip(&listed) + .filter_map(|(row, listed)| listed.is_none().then_some(row)) + .collect(); + let mut resolved = resolve_runs(reader, store, access, &misses) + .await? + .into_iter(); + let mut summaries = Vec::with_capacity(runs.len()); + for ((row, key), cached) in runs.iter().zip(keys).zip(listed) { + let listed = match cached { + Some(listed) => Some(listed), + None => { + let listed = resolved.next().flatten(); + if let Some(listed) = &listed { + reader.lists.runs.insert(key, listed.clone()).await; + } + listed + } + }; + summaries.push(summary(row, listed.as_ref())); + } + Ok(summaries) +} + +/// One entry per run; `None` means the run could not be resolved and keeps its listed summary +/// without being cached. +async fn resolve_runs( + reader: &TraceReader, + store: &S, + access: &ReadAccessParams, + runs: &[&ListTracesRow], +) -> Result>, ReadError> { + let (Some(start_ms), Some(end_ms)) = ( + runs.iter().map(|row| row.start_ms).min(), + runs.iter() + .map(|row| row.start_ms.saturating_add(row.duration_ms)) + .max(), + ) else { + return Ok(Vec::new()); + }; + let params = TracePageSpansParams { + access: access.clone(), + trace_refs: runs.iter().map(|row| row.trace_ref.clone()).collect(), + start_ms, + end_ms: end_ms.saturating_add(1), + }; + let snapshot_ms = now_ms(); + let spans = match store.run_spans(¶ms, snapshot_ms).await { + Ok(spans) => spans, + Err(StoreError::TooLarge) => { + let mut resolved = Vec::with_capacity(runs.len()); + for row in runs { + resolved.push(resolve_run(reader, store, access, row).await?); + } + return Ok(resolved); + } + Err(error) => return Err(map_store_error(error)), + }; + let Some(spend_rows) = spend(store, access, &spans).await else { + // The batch's combined spend read failed; a run's own narrower window may still + // resolve, so fall back per run instead of leaving every run in the batch costless. + let mut resolved = Vec::with_capacity(runs.len()); + for row in runs { + resolved.push(resolve_run(reader, store, access, row).await?); + } + return Ok(resolved); + }; + let mut spans = spans; + spans.sort_by(|left, right| { + run_key(&left.team_id, &left.api_key_hash, &left.trace_id) + .cmp(&run_key( + &right.team_id, + &right.api_key_hash, + &right.trace_id, + )) + .then(left.start_ns.cmp(&right.start_ns)) + }); + let by_run: HashMap<_, &[TraceSpansRow]> = spans + .chunk_by(|left, right| { + (&left.team_id, &left.api_key_hash, &left.trace_id) + == (&right.team_id, &right.api_key_hash, &right.trace_id) + }) + .map(|run| { + ( + run_key(&run[0].team_id, &run[0].api_key_hash, &run[0].trace_id), + run, + ) + }) + .collect(); + Ok(runs + .iter() + .map(|row| { + let spans = by_run + .get(&run_key(&row.team_id, &row.api_key_hash, &row.trace_id)) + .copied() + .unwrap_or_default(); + let spend = + spend_window(spans).map_or(&[][..], |window| spend_within(&spend_rows, window)); + resolve_trace(&row.trace_id, &row.trace_ref, spans, spend).map(|trace| { + ListedRun::Resolved( + Box::new(trace.summary), + Freshness::of(spans, true, snapshot_ms), + ) + }) + }) + .collect()) +} + +async fn resolve_run( + reader: &TraceReader, + store: &S, + access: &ReadAccessParams, + row: &ListTracesRow, +) -> Result, ReadError> { + match reader + .current(store, access, &row.trace_id, &row.trace_ref) + .await + { + Ok(snapshot) => Ok(snapshot.map(|snapshot| { + ListedRun::Resolved( + Box::new(snapshot.trace().summary.clone()), + snapshot.freshness(), + ) + })), + Err(ReadError::TooLarge) => Ok(Some(ListedRun::Limited)), + Err(error) => Err(error), + } +} + +pub(super) fn run_batches(runs: &[T]) -> impl Iterator + '_ { + runs.chunks(RUNS_PER_SPAN_READ) +} diff --git a/litellm-rust/crates/traces-cache/src/reader.rs b/litellm-rust/crates/traces-cache/src/reader.rs new file mode 100644 index 00000000000..5c68d6e58bd --- /dev/null +++ b/litellm-rust/crates/traces-cache/src/reader.rs @@ -0,0 +1,365 @@ +use std::{sync::Arc, time::Duration}; + +use crate::{ + ReadError, Snapshot, SnapshotCache, SnapshotKey, StoreError, TraceStore, + cache::{Freshness, ListCache}, + cursor::{ + ErrorPosition, SpanPosition, decode_cursor, encode_cursor, error_position, trace_position, + }, + list::{list_summaries, run_batches}, + spend::spend, +}; +use litellm_traces::{ + SpanDetail, SpanErrorPage, Trace, TracePage, + query::named::{ + ListTracesParams, ReadAccessParams, SpanDetailParams, SpanErrorParams, TraceIdentityParams, + TraceSpansParams, + }, + resolve_trace, to_ui_content, +}; + +pub const MAX_GRAPH_BYTES: usize = 64 * 1024 * 1024; +pub const MAX_GRAPH_SPANS: usize = 100_000; + +const SNAPSHOT_IDLE: Duration = Duration::from_secs(120); + +/// A read that found no trace, kept apart from failures so single-flight waiters share it +/// without it being cached. +pub(super) enum Miss { + Absent, + Read(ReadError), +} + +impl From for Miss { + fn from(error: crate::Error) -> Self { + Self::Read(error.into()) + } +} + +fn settle(result: Result>>) -> Result, ReadError> { + match result { + Ok(value) => Ok(Some(value)), + Err(miss) => match &*miss { + Miss::Absent => Ok(None), + Miss::Read(error) => Err(error.clone()), + }, + } +} + +pub struct TraceReader { + snapshots: SnapshotCache, + pub(super) lists: ListCache, + response_bytes: usize, +} + +impl TraceReader { + pub fn new(response_bytes: usize) -> Self { + Self { + snapshots: SnapshotCache::new(MAX_GRAPH_BYTES, SNAPSHOT_IDLE), + lists: ListCache::new(), + response_bytes, + } + } + + pub async fn list_traces( + &self, + store: &S, + access: &ReadAccessParams, + start_ms: i64, + end_ms: i64, + cursor: Option<&str>, + limit: u32, + ) -> Result> { + if limit == 0 { + return Err(ReadError::InvalidParameters); + } + let (cursor_ms, cursor_trace_id) = trace_position(cursor)?; + let scope = SnapshotKey::scope(store.source(), access)?; + let accepted = self.lists.limits.get(&scope).await.unwrap_or(u32::MAX); + let mut params = ListTracesParams { + access: access.clone(), + start_ms, + end_ms, + cursor_ms, + cursor_trace_id, + limit: limit.min(500).min(accepted), + }; + let page = loop { + match store.list_runs(¶ms).await { + Err(StoreError::TooLarge) if params.limit > 1 => { + params.limit /= 2; + self.lists.limits.insert(scope.clone(), params.limit).await; + } + Err(StoreError::TooLarge) => return Err(ReadError::TooLarge), + result => break result.map_err(map_store_error)?, + } + }; + let next_cursor = page + .last() + .filter(|_| page.len() == params.limit as usize) + .map(|last| encode_cursor(&(last.start_ms, &last.trace_ref))); + let data = { + let mut summaries = Vec::with_capacity(page.len()); + for batch in run_batches(&page) { + summaries.extend(list_summaries(self, store, access, batch).await?); + } + summaries + }; + Ok(TracePage { data, next_cursor }) + } + + pub async fn get_trace( + &self, + store: &S, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, + ) -> Result, ReadError> { + let Some(trace_ref) = reference(store, access, trace_id, trace_ref).await? else { + return Ok(None); + }; + Ok(self + .current(store, access, trace_id, &trace_ref) + .await? + .map(|snapshot| snapshot.trace().clone())) + } + + pub async fn get_trace_page( + &self, + store: &S, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, + cursor: Option<&str>, + page_size: u32, + ) -> Result, ReadError> { + if !(1..=500).contains(&page_size) { + return Err(ReadError::InvalidParameters); + } + let Some(trace_ref) = reference(store, access, trace_id, trace_ref).await? else { + return Ok(None); + }; + let Some(cursor) = cursor else { + let Some(snapshot) = self.current(store, access, trace_id, &trace_ref).await? else { + return Ok(None); + }; + let position = SpanPosition { + trace_ref, + snapshot_ms: snapshot.snapshot_ms(), + offset: 0, + version: snapshot.version().to_owned(), + }; + return page(&snapshot, &position, page_size, self.response_bytes).map(Some); + }; + let position: SpanPosition = decode_cursor(cursor, "span")?; + if position.trace_ref != trace_ref || position.snapshot_ms == 0 { + return Err(ReadError::InvalidCursor("span")); + } + let Some(snapshot) = settle( + self.pinned(store, access, trace_id, &trace_ref, position.snapshot_ms) + .await, + )? + else { + return Ok(None); + }; + if position.version != snapshot.version() { + return Err(ReadError::TraceChanged); + } + if position.offset > snapshot.trace().spans.len() { + return Err(ReadError::InvalidCursor("span")); + } + page(&snapshot, &position, page_size, self.response_bytes).map(Some) + } + + pub(super) async fn current( + &self, + store: &S, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, + ) -> Result>, ReadError> { + let latest = SnapshotKey::latest(store.source(), access, trace_id, trace_ref)?; + settle( + self.snapshots + .latest_or_load(latest, now_ms(), |snapshot_ms| { + self.pinned(store, access, trace_id, trace_ref, snapshot_ms) + }) + .await, + ) + } + + async fn pinned( + &self, + store: &S, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, + snapshot_ms: u64, + ) -> Result, Arc>> { + let key = SnapshotKey::new(store.source(), access, trace_id, trace_ref, snapshot_ms) + .map_err(|error| Arc::new(error.into()))?; + self.snapshots + .pinned_or_load(key, snapshot_ms, async { + let params = TraceSpansParams { + access: access.clone(), + trace_id: trace_id.to_owned(), + trace_ref: trace_ref.to_owned(), + }; + let rows = store + .trace_spans(¶ms, snapshot_ms) + .await + .map_err(|error| Miss::Read(map_store_error(error)))?; + let spend_rows = spend(store, access, &rows).await; + let freshness = Freshness::of(&rows, spend_rows.is_some(), snapshot_ms); + resolve_trace( + trace_id, + trace_ref, + &rows, + spend_rows.as_deref().unwrap_or_default(), + ) + .map(|trace| (trace, freshness)) + .ok_or(Miss::Absent) + }) + .await + } + + pub async fn get_span( + &self, + store: &S, + access: &ReadAccessParams, + trace_id: &str, + span_id: &str, + trace_ref: &str, + ) -> Result, ReadError> { + let Some(trace_ref) = reference(store, access, trace_id, trace_ref).await? else { + return Ok(None); + }; + let params = SpanDetailParams { + access: access.clone(), + trace_id: trace_id.to_owned(), + trace_ref, + span_id: span_id.to_owned(), + }; + let row = store.span_detail(¶ms).await.map_err(map_store_error)?; + Ok(row.map(|row| SpanDetail { + input_ui: to_ui_content(&row.input), + output_ui: to_ui_content(&row.output), + span_id: row.span_id, + input: row.input, + output: row.output, + attributes: row.attributes, + })) + } + + pub async fn get_span_error( + &self, + store: &S, + access: &ReadAccessParams, + trace_id: &str, + span_id: &str, + trace_ref: &str, + cursor: Option<&str>, + ) -> Result, ReadError> { + let position = error_position(cursor)?; + let Some(trace_ref) = reference(store, access, trace_id, trace_ref).await? else { + return Ok(None); + }; + let offset = position.as_ref().map_or(0, |position| position.offset); + let params = SpanErrorParams { + access: access.clone(), + trace_id: trace_id.to_owned(), + trace_ref, + span_id: span_id.to_owned(), + error_offset: offset, + error_version: position + .map(|position| position.version) + .unwrap_or_default(), + }; + let Some(row) = store.span_error(¶ms).await.map_err(map_store_error)? else { + return Ok(None); + }; + let next_offset = offset + row.message.chars().count() as u64; + let next_cursor = (next_offset < row.total_chars).then(|| { + encode_cursor(&ErrorPosition { + offset: next_offset, + version: row.version, + }) + }); + Ok(Some(SpanErrorPage { + span_id: row.span_id, + message: row.message, + total_chars: row.total_chars, + next_cursor, + })) + } +} + +async fn reference( + store: &S, + access: &ReadAccessParams, + trace_id: &str, + trace_ref: &str, +) -> Result, ReadError> { + if !trace_ref.is_empty() { + return Ok(Some(trace_ref.to_owned())); + } + let params = TraceIdentityParams { + access: access.clone(), + trace_id: trace_id.to_owned(), + }; + let identities = store.trace_refs(¶ms).await.map_err(map_store_error)?; + if identities.len() > 1 { + return Err(ReadError::AmbiguousTrace); + } + Ok(identities.into_iter().next()) +} + +fn page( + snapshot: &Snapshot, + position: &SpanPosition, + page_size: u32, + response_bytes: usize, +) -> Result> { + let spans = &snapshot.trace().spans; + let create_page = |count: usize| { + let end = position.offset.saturating_add(count).min(spans.len()); + Trace { + summary: snapshot.trace().summary.clone(), + agents: snapshot.trace().agents.clone(), + spans: spans[position.offset..end].to_vec(), + next_cursor: (end < spans.len()).then(|| { + encode_cursor(&SpanPosition { + trace_ref: position.trace_ref.clone(), + snapshot_ms: position.snapshot_ms, + offset: end, + version: snapshot.version().to_owned(), + }) + }), + } + }; + let mut trace = create_page(page_size as usize); + loop { + if serde_json::to_vec(&trace) + .map_err(|error| ReadError::Encode(Arc::new(error)))? + .len() + <= response_bytes + { + return Ok(trace); + } + if trace.spans.len() <= 1 { + return Err(ReadError::TooLarge); + } + trace = create_page(trace.spans.len() / 2); + } +} + +pub(super) fn now_ms() -> u64 { + (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as u64 +} + +pub(super) fn map_store_error(error: StoreError) -> ReadError { + match error { + StoreError::TooLarge => ReadError::TooLarge, + StoreError::Failed(error) => ReadError::Store(Arc::new(error)), + } +} diff --git a/litellm-rust/crates/traces-cache/src/spend.rs b/litellm-rust/crates/traces-cache/src/spend.rs new file mode 100644 index 00000000000..1cfcde841d0 --- /dev/null +++ b/litellm-rust/crates/traces-cache/src/spend.rs @@ -0,0 +1,67 @@ +use std::ops::Range; + +use crate::TraceStore; +use litellm_traces::{ + SpendLookup, + query::named::{ + ReadAccessParams, SpendByResponseIdsParams, SpendByResponseIdsRow, TraceSpansRow, + }, +}; + +const NANOS_PER_MS: i64 = 1_000_000; +const SPEND_WINDOW_MS: i64 = 30 * 60 * 1000; + +pub(super) fn spend_window(rows: &[TraceSpansRow]) -> Option> { + let start_ns = rows.iter().map(|row| row.start_ns).min()?; + let end_ns = rows + .iter() + .map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns)) + .max()?; + Some( + start_ns.div_euclid(NANOS_PER_MS) - SPEND_WINDOW_MS + ..end_ns.div_euclid(NANOS_PER_MS) + SPEND_WINDOW_MS, + ) +} + +pub(super) fn spend_within( + spend: &[SpendByResponseIdsRow], + window: Range, +) -> &[SpendByResponseIdsRow] { + let first = spend.partition_point(|row| row.start_ms < window.start); + let end = spend.partition_point(|row| row.start_ms < window.end); + &spend[first..end.max(first)] +} + +/// Spend rows sorted by `start_ms`, or `None` when the lookup failed and spend is unknown. +pub(super) async fn spend( + store: &S, + access: &ReadAccessParams, + rows: &[TraceSpansRow], +) -> Option> { + let lookup = SpendLookup::new(rows); + let Some(window) = spend_window(rows) else { + return Some(Vec::new()); + }; + if lookup.is_empty() { + return Some(Vec::new()); + } + let params = SpendByResponseIdsParams { + access: access.clone(), + response_ids: lookup.response_ids, + request_ids: lookup.request_ids, + trace_ids: lookup.trace_ids, + start_ms: window.start, + end_ms: window.end, + }; + match store.spend(¶ms).await { + Ok(rows) => { + let mut rows = rows; + rows.sort_by_key(|row| row.start_ms); + Some(rows) + } + Err(error) => { + tracing::warn!(%error, "trace spend lookup unavailable"); + None + } + } +} diff --git a/litellm-rust/crates/traces-cache/src/store.rs b/litellm-rust/crates/traces-cache/src/store.rs new file mode 100644 index 00000000000..3f2a536d4cf --- /dev/null +++ b/litellm-rust/crates/traces-cache/src/store.rs @@ -0,0 +1,62 @@ +use std::future::Future; + +use litellm_traces::query::named::{ + ListTracesParams, ListTracesRow, SpanDetailParams, SpanDetailRow, SpanErrorParams, + SpanErrorRow, SpendByResponseIdsParams, SpendByResponseIdsRow, TraceIdentityParams, + TracePageSpansParams, TraceSpansParams, TraceSpansRow, +}; + +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("trace read exceeds the storage read budget")] + TooLarge, + #[error(transparent)] + Failed(E), +} + +pub trait TraceStore: Sync { + type Error: std::error::Error + Send + Sync + 'static; + + /// Identifies the backing storage for snapshot cache keys. + fn source(&self) -> &str; + + fn trace_refs( + &self, + params: &TraceIdentityParams, + ) -> impl Future, StoreError>> + Send; + + /// Returns `TooLarge` when the response exceeds the storage limit so the reader can halve `limit`. + fn list_runs( + &self, + params: &ListTracesParams, + ) -> impl Future, StoreError>> + Send; + + /// Returns spans visible at `snapshot_ms`, sorted by `start_ns`, or `TooLarge` past `MAX_GRAPH_BYTES`/`MAX_GRAPH_SPANS`. + fn trace_spans( + &self, + params: &TraceSpansParams, + snapshot_ms: u64, + ) -> impl Future, StoreError>> + Send; + + /// Returns spans visible at `snapshot_ms`, sorted by `start_ns`, or `TooLarge` past `MAX_GRAPH_BYTES`/`MAX_GRAPH_SPANS`. + fn run_spans( + &self, + params: &TracePageSpansParams, + snapshot_ms: u64, + ) -> impl Future, StoreError>> + Send; + + fn spend( + &self, + params: &SpendByResponseIdsParams, + ) -> impl Future, StoreError>> + Send; + + fn span_detail( + &self, + params: &SpanDetailParams, + ) -> impl Future, StoreError>> + Send; + + fn span_error( + &self, + params: &SpanErrorParams, + ) -> impl Future, StoreError>> + Send; +} diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs new file mode 100644 index 00000000000..d714d7c9ed4 --- /dev/null +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -0,0 +1,768 @@ +use std::{ + collections::{HashMap, HashSet}, + sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use litellm_traces::{ + CallEvidenceKind, CallKey, ObservationType, SpanStatus, + query::named::{ + ListTracesParams, ListTracesRow, ReadAccessParams, SpanDetailParams, SpanDetailRow, + SpanErrorParams, SpanErrorRow, SpendByResponseIdsParams, SpendByResponseIdsRow, + TraceIdentityParams, TracePageSpansParams, TraceSpansParams, TraceSpansRow, + }, +}; +use litellm_traces_cache::{LIVE_TTL, ReadError, StoreError, TraceReader, TraceStore}; +use rstest::rstest; + +const START_NS: i64 = 1_790_742_989_000_000_000; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +enum Operation { + TraceRefs, + ListRuns, + TraceSpans, + RunSpans, + Spend, + SpanDetail, + SpanError, +} + +#[derive(Clone, Copy)] +enum Failure { + TooLarge, + Failed, +} + +#[derive(Debug, thiserror::Error)] +#[error("fake trace store failed")] +struct FakeError; + +#[derive(Default)] +struct State { + failures: HashMap, + trace_refs: Vec, + list_runs: Vec, + trace_spans: HashMap>, + run_spans: Vec, + spend: Vec, + span_detail: Option, + span_error: Option, + list_runs_too_large_above: Option, + trace_too_large_refs: HashSet, + spend_fails_above_response_ids: Option, +} + +#[derive(Default)] +struct Calls { + trace_refs: AtomicUsize, + list_runs: AtomicUsize, + trace_spans: AtomicUsize, + run_spans: AtomicUsize, + spend: AtomicUsize, + span_detail: AtomicUsize, + span_error: AtomicUsize, +} + +#[derive(Default)] +struct FakeStore { + state: Mutex, + calls: Calls, +} + +impl FakeStore { + fn with_spans(trace_ref: &str, spans: Vec) -> Self { + Self { + state: Mutex::new(State { + trace_spans: HashMap::from([(trace_ref.to_owned(), spans)]), + ..State::default() + }), + calls: Calls::default(), + } + } + + fn set_failure(&self, operation: Operation, failure: Failure) { + self.state + .lock() + .unwrap() + .failures + .insert(operation, failure); + } + + fn set_trace_refs(&self, trace_refs: Vec) { + self.state.lock().unwrap().trace_refs = trace_refs; + } + + fn set_list_runs(&self, rows: Vec) { + self.state.lock().unwrap().list_runs = rows; + } + + fn set_list_runs_too_large_above(&self, limit: u32) { + self.state.lock().unwrap().list_runs_too_large_above = Some(limit); + } + + fn set_run_spans(&self, rows: Vec) { + self.state.lock().unwrap().run_spans = rows; + } + + fn set_trace_spans_too_large(&self, trace_ref: &str) { + self.state + .lock() + .unwrap() + .trace_too_large_refs + .insert(trace_ref.to_owned()); + } + + /// Fails `spend` only when the lookup covers more than `limit` response ids, so a batch + /// covering several runs fails while each run's own narrower lookup still succeeds. + fn set_spend_fails_above_response_ids(&self, limit: usize) { + self.state.lock().unwrap().spend_fails_above_response_ids = Some(limit); + } + + fn calls(&self, operation: Operation) -> usize { + match operation { + Operation::TraceRefs => self.calls.trace_refs.load(Ordering::SeqCst), + Operation::ListRuns => self.calls.list_runs.load(Ordering::SeqCst), + Operation::TraceSpans => self.calls.trace_spans.load(Ordering::SeqCst), + Operation::RunSpans => self.calls.run_spans.load(Ordering::SeqCst), + Operation::Spend => self.calls.spend.load(Ordering::SeqCst), + Operation::SpanDetail => self.calls.span_detail.load(Ordering::SeqCst), + Operation::SpanError => self.calls.span_error.load(Ordering::SeqCst), + } + } + + fn failure(state: &State, operation: Operation) -> Result<(), StoreError> { + match state.failures.get(&operation) { + Some(Failure::TooLarge) => Err(StoreError::TooLarge), + Some(Failure::Failed) => Err(StoreError::Failed(FakeError)), + None => Ok(()), + } + } +} + +impl TraceStore for FakeStore { + type Error = FakeError; + + fn source(&self) -> &str { + "fake" + } + + async fn trace_refs( + &self, + _: &TraceIdentityParams, + ) -> Result, StoreError> { + self.calls.trace_refs.fetch_add(1, Ordering::SeqCst); + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::TraceRefs)?; + Ok(state.trace_refs.clone()) + } + + async fn list_runs( + &self, + params: &ListTracesParams, + ) -> Result, StoreError> { + self.calls.list_runs.fetch_add(1, Ordering::SeqCst); + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::ListRuns)?; + if state + .list_runs_too_large_above + .is_some_and(|limit| params.limit > limit) + { + return Err(StoreError::TooLarge); + } + Ok(state + .list_runs + .iter() + .take(params.limit as usize) + .cloned() + .collect()) + } + + async fn trace_spans( + &self, + params: &TraceSpansParams, + _: u64, + ) -> Result, StoreError> { + self.calls.trace_spans.fetch_add(1, Ordering::SeqCst); + tokio::task::yield_now().await; + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::TraceSpans)?; + if state.trace_too_large_refs.contains(¶ms.trace_ref) { + return Err(StoreError::TooLarge); + } + Ok(state + .trace_spans + .get(¶ms.trace_ref) + .cloned() + .unwrap_or_default()) + } + + async fn run_spans( + &self, + _: &TracePageSpansParams, + _: u64, + ) -> Result, StoreError> { + self.calls.run_spans.fetch_add(1, Ordering::SeqCst); + tokio::task::yield_now().await; + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::RunSpans)?; + Ok(state.run_spans.clone()) + } + + async fn spend( + &self, + params: &SpendByResponseIdsParams, + ) -> Result, StoreError> { + self.calls.spend.fetch_add(1, Ordering::SeqCst); + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::Spend)?; + if state + .spend_fails_above_response_ids + .is_some_and(|limit| params.response_ids.len() > limit) + { + return Err(StoreError::Failed(FakeError)); + } + Ok(state.spend.clone()) + } + + async fn span_detail( + &self, + _: &SpanDetailParams, + ) -> Result, StoreError> { + self.calls.span_detail.fetch_add(1, Ordering::SeqCst); + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::SpanDetail)?; + Ok(state.span_detail.clone()) + } + + async fn span_error( + &self, + _: &SpanErrorParams, + ) -> Result, StoreError> { + self.calls.span_error.fetch_add(1, Ordering::SeqCst); + let state = self.state.lock().unwrap(); + Self::failure(&state, Operation::SpanError)?; + Ok(state.span_error.clone()) + } +} + +fn access() -> ReadAccessParams { + ReadAccessParams { + all_teams: true, + user_id: String::new(), + team_ids: Vec::new(), + } +} + +fn span(index: usize) -> TraceSpansRow { + TraceSpansRow { + trace_id: "trace".into(), + span_id: format!("span-{index}"), + parent_span_id: if index == 0 { + String::new() + } else { + "span-0".into() + }, + name: "agent".into(), + kind: ObservationType::Agent, + wrapper_candidate: false, + agent: "agent".into(), + framework: String::new(), + status: SpanStatus::Ok, + status_message: String::new(), + error_truncated: false, + start_ns: START_NS + index as i64 * 1_000_000, + duration_ns: 10_000_000, + service: "test".into(), + input_preview: format!("span input {index}"), + model: String::new(), + input_tokens: 0, + output_tokens: 0, + litellm_request_id: String::new(), + call_keys: Vec::new(), + call_evidence: None, + tool_call_id: String::new(), + team_id: "team".into(), + api_key_hash: "key".into(), + user_id: "user".into(), + } +} + +fn run(trace_id: &str, trace_ref: &str) -> ListTracesRow { + ListTracesRow { + trace_id: trace_id.into(), + trace_ref: trace_ref.into(), + team_id: "team".into(), + api_key_hash: "key".into(), + user_id: "user".into(), + name: "listed".into(), + service: "test".into(), + input_preview: String::new(), + status: SpanStatus::Ok, + start_ms: 1_790_742_989_000, + duration_ms: 10, + span_count: 1, + agent_count: 1, + agent_invocations: 1, + agent_names: vec!["agent".into()], + frameworks: Vec::new(), + llm_calls: 0, + tool_calls: 0, + input_tokens: 0, + output_tokens: 0, + models: Vec::new(), + error_count: 0, + request_ids: Vec::new(), + } +} + +#[rstest] +#[tokio::test] +async fn pages_reuse_one_trace_snapshot_and_concatenate_in_order() { + let store = FakeStore::with_spans("ref", (0..5).map(span).collect()); + let reader = TraceReader::new(usize::MAX); + let access = access(); + let first = reader + .get_trace_page(&store, &access, "trace", "ref", None, 2) + .await + .unwrap() + .unwrap(); + let second = reader + .get_trace_page( + &store, + &access, + "trace", + "ref", + first.next_cursor.as_deref(), + 2, + ) + .await + .unwrap() + .unwrap(); + let third = reader + .get_trace_page( + &store, + &access, + "trace", + "ref", + second.next_cursor.as_deref(), + 2, + ) + .await + .unwrap() + .unwrap(); + let ids: Vec<_> = first + .spans + .iter() + .chain(&second.spans) + .chain(&third.spans) + .map(|span| span.span_id.as_str()) + .collect(); + assert_eq!(ids, ["span-0", "span-1", "span-2", "span-3", "span-4"]); + assert!(third.next_cursor.is_none()); + assert_eq!(store.calls(Operation::TraceSpans), 1); +} + +#[rstest] +#[tokio::test] +async fn snapshot_versions_are_stable_across_readers_and_detect_changes() { + let access = access(); + let original = FakeStore::with_spans("ref", vec![span(0), span(1)]); + let reader_a = TraceReader::new(usize::MAX); + let first = reader_a + .get_trace_page(&original, &access, "trace", "ref", None, 1) + .await + .unwrap() + .unwrap(); + let cursor = first.next_cursor.unwrap(); + + let changed = FakeStore::with_spans("ref", vec![span(0), span(1), span(2)]); + let reader_b = TraceReader::new(usize::MAX); + let result = reader_b + .get_trace_page(&changed, &access, "trace", "ref", Some(&cursor), 1) + .await; + assert!(matches!(result, Err(ReadError::TraceChanged))); + + let unchanged = FakeStore::with_spans("ref", vec![span(0), span(1)]); + let reader_c = TraceReader::new(usize::MAX); + let next = reader_c + .get_trace_page(&unchanged, &access, "trace", "ref", Some(&cursor), 1) + .await + .unwrap() + .unwrap(); + assert_eq!(next.spans[0].span_id, "span-1"); +} + +#[rstest] +#[tokio::test] +async fn response_size_splits_pages_and_rejects_a_single_oversized_span() { + let spans: Vec<_> = (0..4) + .map(|index| { + let mut row = span(index); + row.input_preview = "x".repeat(256); + row + }) + .collect(); + let access = access(); + let full_budget_reader = TraceReader::new(usize::MAX); + let one_span = full_budget_reader + .get_trace_page( + &FakeStore::with_spans("ref", spans.clone()), + &access, + "trace", + "ref", + None, + 1, + ) + .await + .unwrap() + .unwrap(); + let response_bytes = serde_json::to_vec(&one_span).unwrap().len() + 128; + let reader = TraceReader::new(response_bytes); + let store = FakeStore::with_spans("ref", spans.clone()); + let page = reader + .get_trace_page(&store, &access, "trace", "ref", None, 4) + .await + .unwrap() + .unwrap(); + assert!(!page.spans.is_empty()); + assert!(page.spans.len() < 4); + assert!(page.next_cursor.is_some()); + let continued = reader + .get_trace_page( + &store, + &access, + "trace", + "ref", + page.next_cursor.as_deref(), + 4, + ) + .await + .unwrap() + .unwrap(); + assert!(!continued.spans.is_empty()); + assert!(matches!( + TraceReader::new(1) + .get_trace_page(&store, &access, "trace", "ref", None, 1) + .await, + Err(ReadError::TooLarge) + )); +} + +#[rstest] +#[tokio::test] +async fn list_run_budget_halves_the_limit_and_cursor_requires_a_full_page() { + let store = FakeStore::default(); + store.set_list_runs( + (0..3) + .map(|index| run(&format!("trace-{index}"), &format!("ref-{index}"))) + .collect(), + ); + store.set_list_runs_too_large_above(2); + let reader = TraceReader::new(usize::MAX); + let access = access(); + let page = reader + .list_traces(&store, &access, 0, i64::MAX, None, 8) + .await + .unwrap(); + assert_eq!(page.data.len(), 2); + assert!(page.next_cursor.is_some()); + assert_eq!(store.calls(Operation::ListRuns), 3); + + let shorter = FakeStore::default(); + shorter.set_list_runs(vec![run("only", "ref-only")]); + shorter.set_list_runs_too_large_above(2); + let page = reader + .list_traces(&shorter, &access, 0, i64::MAX, None, 8) + .await + .unwrap(); + assert_eq!(page.data.len(), 1); + assert!(page.next_cursor.is_none()); + assert_eq!(shorter.calls(Operation::ListRuns), 1); +} + +#[rstest] +#[tokio::test] +async fn oversized_run_batch_falls_back_to_each_run_and_keeps_listed_summaries() { + let store = FakeStore::with_spans("ref-good", vec![span(0)]); + store.set_list_runs(vec![ + run("trace-large", "ref-large"), + run("trace-good", "ref-good"), + ]); + store.set_trace_spans_too_large("ref-large"); + store.set_run_spans(Vec::new()); + store.set_failure(Operation::RunSpans, Failure::TooLarge); + let reader = TraceReader::new(usize::MAX); + let page = reader + .list_traces(&store, &access(), 0, i64::MAX, None, 2) + .await + .unwrap(); + assert_eq!(page.data.len(), 2); + assert!(page.data[0].resolution_limited); + assert_eq!(page.data[0].trace_ref, "ref-large"); + assert!(!page.data[1].resolution_limited); + assert_eq!(page.data[1].trace_ref, "ref-good"); + assert_eq!(store.calls(Operation::RunSpans), 1); + assert_eq!(store.calls(Operation::TraceSpans), 2); + + let again = reader + .list_traces(&store, &access(), 0, i64::MAX, None, 2) + .await + .unwrap(); + assert_eq!(again.data, page.data); + assert_eq!(store.calls(Operation::RunSpans), 1); + assert_eq!(store.calls(Operation::TraceSpans), 2); +} + +fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + request_id: response_id.into(), + litellm_call_id: String::new(), + response_id: response_id.into(), + upstream_response_id: String::new(), + trace_id: String::new(), + span_id: String::new(), + team_id: "team".into(), + api_key: "key".into(), + user: "user".into(), + spend: Some(cost), + start_ms: START_NS / 1_000_000, + } +} + +#[rstest] +#[tokio::test] +async fn failed_batch_spend_lookup_falls_back_to_each_run_instead_of_losing_every_cost() { + let mut first = span(0); + first.trace_id = "trace-a".into(); + first.kind = ObservationType::Llm; + first.litellm_request_id = "response-a".into(); + first.call_keys = vec![CallKey::ProviderResponse("response-a".into())]; + first.call_evidence = Some(CallEvidenceKind::Complete); + let mut second = span(0); + second.trace_id = "trace-b".into(); + second.kind = ObservationType::Llm; + second.litellm_request_id = "response-b".into(); + second.call_keys = vec![CallKey::ProviderResponse("response-b".into())]; + second.call_evidence = Some(CallEvidenceKind::Complete); + + let store = FakeStore::default(); + store.set_list_runs(vec![run("trace-a", "ref-a"), run("trace-b", "ref-b")]); + store.set_run_spans(vec![first.clone(), second.clone()]); + { + let mut state = store.state.lock().unwrap(); + state.trace_spans.insert("ref-a".to_owned(), vec![first]); + state.trace_spans.insert("ref-b".to_owned(), vec![second]); + state.spend = vec![spend_row("response-a", 1.5), spend_row("response-b", 2.5)]; + } + // The batch covers both runs' response ids (2); each run resolved on its own only ever + // asks for its own (1), so this fails only the combined read, not the per-run fallback. + store.set_spend_fails_above_response_ids(1); + + let page = TraceReader::new(usize::MAX) + .list_traces(&store, &access(), 0, i64::MAX, None, 8) + .await + .unwrap(); + + assert_eq!(page.data.len(), 2); + let by_ref: HashMap<&str, f64> = page + .data + .iter() + .map(|run| { + ( + run.trace_ref.as_str(), + run.spend + .expect("run's own spend read should have succeeded"), + ) + }) + .collect(); + assert_eq!(by_ref["ref-a"], 1.5); + assert_eq!(by_ref["ref-b"], 2.5); + assert_eq!(store.calls(Operation::RunSpans), 1); + assert_eq!(store.calls(Operation::TraceSpans), 2); +} + +#[rstest] +#[tokio::test] +async fn failed_spend_lookup_preserves_the_trace_with_unknown_spend() { + let mut row = span(0); + row.litellm_request_id = "response".into(); + row.call_keys = vec![CallKey::ProviderResponse("response".into())]; + row.call_evidence = Some(CallEvidenceKind::Complete); + let store = FakeStore::with_spans("ref", vec![row]); + store.set_failure(Operation::Spend, Failure::Failed); + let trace = TraceReader::new(usize::MAX) + .get_trace(&store, &access(), "trace", "ref") + .await + .unwrap() + .unwrap(); + assert_eq!(trace.summary.spend, None); + assert_eq!(trace.spans[0].spend, None); + assert_eq!(store.calls(Operation::Spend), 1); +} + +#[rstest] +#[tokio::test] +async fn ambiguous_trace_references_fail_and_a_single_reference_is_resolved() { + let reader = TraceReader::new(usize::MAX); + let access = access(); + let ambiguous = FakeStore::default(); + ambiguous.set_trace_refs(vec!["ref-a".into(), "ref-b".into()]); + assert!(matches!( + reader.get_trace(&ambiguous, &access, "trace", "").await, + Err(ReadError::AmbiguousTrace) + )); + + let unique = FakeStore::with_spans("ref-only", vec![span(0)]); + unique.set_trace_refs(vec!["ref-only".into()]); + let trace = reader + .get_trace(&unique, &access, "trace", "") + .await + .unwrap() + .unwrap(); + assert_eq!(trace.summary.trace_ref, "ref-only"); +} + +#[rstest] +#[case::zero(0)] +#[case::above_max(501)] +#[tokio::test] +async fn invalid_page_sizes_are_rejected(#[case] page_size: u32) { + let store = FakeStore::default(); + let reader = TraceReader::new(usize::MAX); + let access = access(); + assert!(matches!( + reader + .get_trace_page(&store, &access, "trace", "ref", None, page_size) + .await, + Err(ReadError::InvalidParameters) + )); +} + +#[rstest] +#[tokio::test] +async fn zero_list_limit_is_rejected() { + let store = FakeStore::default(); + let reader = TraceReader::new(usize::MAX); + let access = access(); + assert!(matches!( + reader + .list_traces(&store, &access, 0, i64::MAX, None, 0) + .await, + Err(ReadError::InvalidParameters) + )); +} + +fn now_ns() -> i64 { + time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64 +} + +#[rstest] +#[tokio::test] +async fn concurrent_and_repeated_opens_share_one_storage_read() { + let store = FakeStore::with_spans("ref", (0..3).map(span).collect()); + let reader = TraceReader::new(usize::MAX); + let access = access(); + let (first, second) = tokio::join!( + reader.get_trace(&store, &access, "trace", "ref"), + reader.get_trace_page(&store, &access, "trace", "ref", None, 2), + ); + let first = first.unwrap().unwrap(); + let second = second.unwrap().unwrap(); + let reopened = reader + .get_trace_page(&store, &access, "trace", "ref", None, 2) + .await + .unwrap() + .unwrap(); + assert_eq!(first.spans.len(), 3); + assert_eq!(second.spans, first.spans[..2]); + assert_eq!(reopened.next_cursor, second.next_cursor); + assert_eq!(store.calls(Operation::TraceSpans), 1); +} + +#[rstest] +#[tokio::test] +async fn failed_reads_are_not_cached() { + let store = FakeStore::with_spans("ref", vec![span(0)]); + store.set_failure(Operation::TraceSpans, Failure::Failed); + let reader = TraceReader::new(usize::MAX); + let access = access(); + assert!(matches!( + reader.get_trace(&store, &access, "trace", "ref").await, + Err(ReadError::Store(_)) + )); + store.state.lock().unwrap().failures.clear(); + let trace = reader + .get_trace(&store, &access, "trace", "ref") + .await + .unwrap() + .unwrap(); + assert_eq!(trace.spans.len(), 1); + assert_eq!(store.calls(Operation::TraceSpans), 2); +} + +#[rstest] +#[tokio::test] +async fn listed_runs_are_read_once_until_a_live_run_expires() { + let live = TraceSpansRow { + trace_id: "trace-live".into(), + start_ns: now_ns(), + ..span(0) + }; + let settled = TraceSpansRow { + trace_id: "trace-settled".into(), + ..span(0) + }; + let store = FakeStore::default(); + store.set_list_runs(vec![ + run("trace-live", "ref-live"), + run("trace-settled", "ref-settled"), + ]); + store.set_run_spans(vec![live, settled]); + let reader = TraceReader::new(usize::MAX); + let access = access(); + let list = || reader.list_traces(&store, &access, 0, i64::MAX, None, 2); + + let first = list().await.unwrap(); + assert!(first.data.iter().all(|summary| summary.name == "agent")); + list().await.unwrap(); + assert_eq!(store.calls(Operation::RunSpans), 1); + + store.set_run_spans(Vec::new()); + tokio::time::sleep(LIVE_TTL + Duration::from_millis(200)).await; + let after = list().await.unwrap(); + assert_eq!(store.calls(Operation::RunSpans), 2); + assert_eq!(after.data[0].name, "listed"); + assert_eq!(after.data[1], first.data[1]); +} + +#[rstest] +#[tokio::test] +async fn concurrent_pages_of_an_evicted_snapshot_share_one_storage_read() { + let access = access(); + let first = TraceReader::new(usize::MAX) + .get_trace_page( + &FakeStore::with_spans("ref", (0..3).map(span).collect()), + &access, + "trace", + "ref", + None, + 1, + ) + .await + .unwrap() + .unwrap(); + let cursor = first.next_cursor.as_deref(); + let store = FakeStore::with_spans("ref", (0..3).map(span).collect()); + let reader = TraceReader::new(usize::MAX); + let (left, right) = tokio::join!( + reader.get_trace_page(&store, &access, "trace", "ref", cursor, 1), + reader.get_trace_page(&store, &access, "trace", "ref", cursor, 1), + ); + assert_eq!(left.unwrap().unwrap().spans[0].span_id, "span-1"); + assert_eq!(right.unwrap().unwrap().spans[0].span_id, "span-1"); + assert_eq!(store.calls(Operation::TraceSpans), 1); +} diff --git a/litellm-rust/crates/traces-cache/tests/snapshots.rs b/litellm-rust/crates/traces-cache/tests/snapshots.rs index de199ae0301..42085d3e95b 100644 --- a/litellm-rust/crates/traces-cache/tests/snapshots.rs +++ b/litellm-rust/crates/traces-cache/tests/snapshots.rs @@ -5,7 +5,9 @@ use litellm_traces::{ query::named::{ReadAccessParams, SpendByResponseIdsRow, TraceSpansRow}, resolve_trace, }; -use litellm_traces_cache::{Error, SnapshotCache, SnapshotKey}; +use std::sync::Arc; + +use litellm_traces_cache::{Error, Freshness, Snapshot, SnapshotCache, SnapshotKey}; use rstest::{fixture, rstest}; const T0: i64 = 1_790_742_989_000_000_000; @@ -60,6 +62,18 @@ fn key( SnapshotKey::new(source, access, trace_id, trace_ref, ms).unwrap() } +async fn insert( + cache: &SnapshotCache, + key: SnapshotKey, + trace: Trace, +) -> Result, Arc> { + cache + .pinned_or_load(key, 100, async { + Ok::<_, Error>((trace, Freshness::Settled)) + }) + .await +} + #[fixture] fn trace() -> Trace { resolve_trace( @@ -85,7 +99,7 @@ async fn cached_trace_is_isolated_by_access_scope( let cache = SnapshotCache::new(1024 * 1024, TTL); let stored = key("source", &access(), "trace", "ref", 100); - cache.insert(stored.clone(), trace.clone()).await.unwrap(); + insert(&cache, stored.clone(), trace.clone()).await.unwrap(); let other_access = ReadAccessParams { all_teams, @@ -115,7 +129,7 @@ async fn cached_trace_is_isolated_by_key_fields( let cache = SnapshotCache::new(1024 * 1024, TTL); let stored = key("source", &access(), "trace", "ref", 100); - cache.insert(stored.clone(), trace.clone()).await.unwrap(); + insert(&cache, stored.clone(), trace.clone()).await.unwrap(); let other = key(source, &access(), trace_id, trace_ref, snapshot_ms); assert!(cache.get(&other).await.is_none()); @@ -129,7 +143,7 @@ async fn snapshot_at_the_size_limit_is_accepted(trace: Trace) { let cache = SnapshotCache::new(size, TTL); let stored = key("source", &access(), "trace", "ref", 100); - cache.insert(stored.clone(), trace).await.unwrap(); + insert(&cache, stored.clone(), trace).await.unwrap(); assert!(cache.get(&stored).await.is_some()); } @@ -141,8 +155,8 @@ async fn snapshot_one_byte_over_the_size_limit_is_rejected(trace: Trace) { let stored = key("source", &access(), "trace", "ref", 100); assert!(matches!( - cache.insert(stored.clone(), trace).await, - Err(Error::ReadTooLarge) + insert(&cache, stored.clone(), trace).await, + Err(error) if matches!(*error, Error::ReadTooLarge) )); assert!(cache.get(&stored).await.is_none()); } @@ -166,25 +180,31 @@ async fn snapshot_version_tracks_the_ordered_span_ids( }; let cache = SnapshotCache::new(1024 * 1024, TTL); - let first = cache - .insert(key("source", &access(), "a", "ref", 100), build(first_ids)) - .await - .unwrap(); - let second = cache - .insert(key("source", &access(), "b", "ref", 100), build(second_ids)) - .await - .unwrap(); + let first = insert( + &cache, + key("source", &access(), "a", "ref", 100), + build(first_ids), + ) + .await + .unwrap(); + let second = insert( + &cache, + key("source", &access(), "b", "ref", 100), + build(second_ids), + ) + .await + .unwrap(); assert_eq!(first.version() == second.version(), equal); } #[rstest] #[tokio::test] -async fn snapshots_expire_after_the_ttl(trace: Trace) { +async fn snapshots_expire_when_idle(trace: Trace) { let cache = SnapshotCache::new(1024 * 1024, Duration::from_millis(50)); let stored = key("source", &access(), "trace", "ref", 100); - cache.insert(stored.clone(), trace).await.unwrap(); + insert(&cache, stored.clone(), trace).await.unwrap(); tokio::time::sleep(Duration::from_millis(200)).await; assert!(cache.get(&stored).await.is_none()); diff --git a/litellm-rust/crates/traces-clickhouse/Cargo.toml b/litellm-rust/crates/traces-clickhouse/Cargo.toml index 6b95eb149d3..b90ad8a7bf8 100644 --- a/litellm-rust/crates/traces-clickhouse/Cargo.toml +++ b/litellm-rust/crates/traces-clickhouse/Cargo.toml @@ -12,11 +12,9 @@ schema = ["dep:schemars", "litellm-traces/schema"] macro_rules_attribute.workspace = true schemars = { workspace = true, optional = true } askama.workspace = true -base64.workspace = true flate2.workspace = true futures-util.workspace = true hmac = "0.12.1" -itertools = "0.14.0" litellm-http.workspace = true litellm-migrate.workspace = true litellm-storage-clickhouse.workspace = true @@ -30,7 +28,6 @@ strum.workspace = true thiserror.workspace = true time = { workspace = true, features = ["formatting"] } tokio.workspace = true -tracing.workspace = true url.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/traces-clickhouse/src/error.rs b/litellm-rust/crates/traces-clickhouse/src/error.rs index 78df9a2121c..1d15c556316 100644 --- a/litellm-rust/crates/traces-clickhouse/src/error.rs +++ b/litellm-rust/crates/traces-clickhouse/src/error.rs @@ -16,8 +16,6 @@ pub enum Error { InvalidResponse, #[error("ClickHouse insert exceeds the encoded size limit")] InsertTooLarge, - #[error("Trace exceeds the interactive read budget; use a filtered trace query")] - ReadTooLarge, #[error("ClickHouse schema setup failed with HTTP status {0}")] SchemaFailed(u16), #[error("ClickHouse schema setup transport failed")] @@ -34,12 +32,6 @@ pub enum Error { ProvisionFailed(u16), #[error("ClickHouse reader provisioning transport failed")] ProvisionTransport, - #[error("Invalid {0} cursor")] - InvalidCursor(&'static str), - #[error("Multiple traces have this ID; provide trace_ref")] - AmbiguousTrace, - #[error("Trace changed while paging; refresh the trace to continue")] - TraceChanged, #[error(transparent)] Decode(#[from] litellm_traces::Error), #[error("trace ingestion task failed")] @@ -49,12 +41,3 @@ pub enum Error { #[error(transparent)] Cached(#[from] std::sync::Arc), } - -impl From for Error { - fn from(error: litellm_traces_cache::Error) -> Self { - match error { - litellm_traces_cache::Error::Serialization(_) => Self::InvalidResponse, - litellm_traces_cache::Error::ReadTooLarge => Self::ReadTooLarge, - } - } -} diff --git a/litellm-rust/crates/traces-clickhouse/src/insert.rs b/litellm-rust/crates/traces-clickhouse/src/insert.rs index 9e6452844f4..d45cc8d53b8 100644 --- a/litellm-rust/crates/traces-clickhouse/src/insert.rs +++ b/litellm-rust/crates/traces-clickhouse/src/insert.rs @@ -239,8 +239,7 @@ mod tests { use rstest::rstest; use serde_json::json; - use super::Error; - use super::{shared_rows, write_rows}; + use super::{Error, shared_rows, write_rows}; #[rstest] fn encoded_limit_counts_utf8_bytes_across_rows() { diff --git a/litellm-rust/crates/traces-clickhouse/src/lib.rs b/litellm-rust/crates/traces-clickhouse/src/lib.rs index d83708d27f1..43ff0b8bf33 100644 --- a/litellm-rust/crates/traces-clickhouse/src/lib.rs +++ b/litellm-rust/crates/traces-clickhouse/src/lib.rs @@ -31,7 +31,7 @@ pub use litellm_storage_clickhouse::{Connection, Parameter}; pub use litellm_traces::{QueryScope, ReadQuery}; pub use query::{QueryHelp, execute_read, query_help, query_sql}; pub use query_access::QueryReaders; -pub use reads::{get_span, get_span_error, get_trace, get_trace_page, list_traces}; +pub use reads::ClickHouseTraces; pub use schema::{ NORMALIZED_FIELD_DEFINITIONS, NormalizedFieldDefinition, ensure_schema, schema_statements, }; diff --git a/litellm-rust/crates/traces-clickhouse/src/reads.rs b/litellm-rust/crates/traces-clickhouse/src/reads.rs index 6938aab06b3..eaaeb4fef82 100644 --- a/litellm-rust/crates/traces-clickhouse/src/reads.rs +++ b/litellm-rust/crates/traces-clickhouse/src/reads.rs @@ -1,26 +1,13 @@ -//! Scoped trace reads: the trace list, one trace resolved with its spend, and span payloads. - -use std::sync::LazyLock; -use std::time::Duration; - -use base64::{Engine, engine::general_purpose::URL_SAFE}; -use futures_util::{StreamExt, TryStreamExt, stream}; -use itertools::Itertools; use litellm_http::Client; -use litellm_storage_clickhouse::{Query, fetch}; -use litellm_traces::{ - SpanDetail, SpanErrorPage, SpendLookup, Trace, TracePage, listed_summary, - query::named as contracts, resolve_trace, to_ui_content, -}; -use litellm_traces_cache::{SnapshotCache, SnapshotKey}; -use serde::{Deserialize, Serialize}; +use litellm_storage_clickhouse::{Error as StorageError, Query, fetch}; +use litellm_traces::query::named as contracts; +use litellm_traces_cache::{StoreError, TraceStore}; use crate::{ Connection, Error, query::named::{ - ListTracesParams, ListTracesRow, ReadAccessParams, SpanDetail as SpanDetailQuery, - SpanDetailParams, SpanError, SpanErrorParams, SpendByResponseIdsParams, TraceIdentity, - TraceIdentityParams, TracePageSpansParams, TraceSpansParams, + ListTracesParams, ListTracesRow, SpanDetail as SpanDetailQuery, SpanError, SpanErrorParams, + SpendByResponseIdsParams, TraceIdentity, TracePageSpansParams, }, }; @@ -36,501 +23,103 @@ impl Query for RunCandidates { ); } -// Cursor pages share a bounded snapshot so advancing does not resolve the whole graph again. -static TRACE_SNAPSHOTS: LazyLock = LazyLock::new(|| { - SnapshotCache::new( - crate::span_batches::MAX_GRAPH_BYTES, - Duration::from_secs(120), - ) -}); - -const NANOS_PER_MS: i64 = 1_000_000; -const SPEND_WINDOW_MS: i64 = 30 * 60 * 1000; -const SPEND_CONCURRENCY: usize = 4; - -fn encode_cursor(position: &T) -> String { - URL_SAFE.encode(serde_json::to_vec(position).unwrap_or_default()) +pub struct ClickHouseTraces { + client: Client, + connection: Connection, } -fn decode_cursor Deserialize<'de>>( - cursor: &str, - kind: &'static str, -) -> Result { - URL_SAFE - .decode(cursor) - .ok() - .and_then(|json| serde_json::from_slice(&json).ok()) - .ok_or(Error::InvalidCursor(kind)) -} - -fn trace_position(cursor: Option<&str>) -> Result<(i64, String), Error> { - let Some(cursor) = cursor.filter(|cursor| !cursor.is_empty()) else { - return Ok((0, String::new())); - }; - match decode_cursor::<(i64, String)>(cursor, "trace")? { - (start_ms, trace_ref) if start_ms > 0 && !trace_ref.is_empty() => Ok((start_ms, trace_ref)), - _ => Err(Error::InvalidCursor("trace")), +impl ClickHouseTraces { + pub fn new(client: Client, connection: Connection) -> Self { + Self { client, connection } } } -#[derive(Deserialize, Serialize)] -struct ErrorPosition { - offset: u64, - version: String, -} +impl TraceStore for ClickHouseTraces { + type Error = Error; -fn error_position(cursor: Option<&str>) -> Result, Error> { - let Some(cursor) = cursor else { - return Ok(None); - }; - let position = decode_cursor::(cursor, "diagnostic")?; - let valid_version = position.version.len() == 64 - && position - .version - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte)); - if i64::try_from(position.offset).is_err() || !valid_version { - return Err(Error::InvalidCursor("diagnostic")); + fn source(&self) -> &str { + self.connection.url().as_str() } - Ok(Some(position)) -} -/// The stored run a trace id names for this caller; ids can repeat across tenants and runs. -async fn reference( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - trace_id: &str, - trace_ref: &str, -) -> Result, Error> { - if !trace_ref.is_empty() { - return Ok(Some(trace_ref.to_owned())); + async fn trace_refs( + &self, + params: &contracts::TraceIdentityParams, + ) -> Result, StoreError> { + fetch::(&self.client, &self.connection, params) + .await + .map(|rows| rows.into_iter().map(|row| row.trace_ref).collect()) + .map_err(failed) } - let params = TraceIdentityParams { - access: access.clone(), - trace_id: trace_id.to_owned(), - }; - let mut identities = fetch::(client, connection, ¶ms).await?; - if identities.len() > 1 { - return Err(Error::AmbiguousTrace); - } - Ok(identities.pop().map(|identity| identity.trace_ref)) -} -/// Spend records behind the spans' calls. A failed lookup leaves cost unknown instead of failing -/// the read. -async fn spend( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - rows: &[contracts::TraceSpansRow], -) -> Vec { - let lookup = SpendLookup::new(rows); - let (Some(start_ns), Some(end_ns)) = ( - rows.iter().map(|row| row.start_ns).min(), - rows.iter() - .map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns)) - .max(), - ) else { - return Vec::new(); - }; - if lookup.is_empty() { - return Vec::new(); - } - let params = SpendByResponseIdsParams::from(contracts::SpendByResponseIdsParams { - access: access.clone(), - response_ids: lookup.response_ids, - request_ids: lookup.request_ids, - trace_ids: lookup.trace_ids, - start_ms: start_ns.div_euclid(NANOS_PER_MS) - SPEND_WINDOW_MS, - end_ms: end_ns.div_euclid(NANOS_PER_MS) + SPEND_WINDOW_MS, - }); - match crate::span_batches::read_spend(client, connection, params).await { - Ok(rows) => rows, - Err(error) => { - tracing::warn!(%error, "trace spend lookup unavailable"); - Vec::new() + async fn list_runs( + &self, + params: &contracts::ListTracesParams, + ) -> Result, StoreError> { + let storage_params = ListTracesParams::from(params.clone()); + match fetch::(&self.client, &self.connection, &storage_params).await { + Ok(rows) => Ok(rows.into_iter().map(|row| row.0).collect()), + Err(StorageError::ResponseTooLarge) => Err(StoreError::TooLarge), + Err(error) => Err(failed(error)), } } -} -pub async fn list_traces( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - start_ms: i64, - end_ms: i64, - cursor: Option<&str>, - limit: u32, -) -> Result { - if limit == 0 { - return Err(Error::InvalidParameters); + async fn trace_spans( + &self, + params: &contracts::TraceSpansParams, + snapshot_ms: u64, + ) -> Result, StoreError> { + crate::span_batches::read_spans(&self.client, &self.connection, params.clone(), snapshot_ms) + .await } - let (cursor_ms, cursor_trace_id) = trace_position(cursor)?; - let mut params = ListTracesParams::from(contracts::ListTracesParams { - access: access.clone(), - start_ms, - end_ms, - cursor_ms, - cursor_trace_id, - limit: limit.min(500), - }); - let page: Vec = loop { - match fetch::(client, connection, ¶ms).await { - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) if params.0.limit > 1 => { - params.0.limit /= 2; - } - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => { - return Err(Error::ReadTooLarge); - } - result => break result?.into_iter().map(|row| row.0).collect(), - } - }; - let next_cursor = page - .last() - .filter(|_| page.len() == params.0.limit as usize) - .map(|last| encode_cursor(&(last.start_ms, &last.trace_ref))); - let data = stream::iter(page.chunks(16)) - .then(|batch| list_summaries(client, connection, access, batch)) - .try_collect::>() - .await? - .into_iter() - .flatten() - .collect(); - Ok(TracePage { data, next_cursor }) -} -async fn list_summaries( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - runs: &[contracts::ListTracesRow], -) -> Result, Error> { - let (Some(start_ms), Some(end_ms)) = ( - runs.iter().map(|row| row.start_ms).min(), - runs.iter() - .map(|row| row.start_ms.saturating_add(row.duration_ms)) - .max(), - ) else { - return Ok(Vec::new()); - }; - let params = TracePageSpansParams::from(contracts::TracePageSpansParams { - access: access.clone(), - trace_refs: runs.iter().map(|row| row.trace_ref.clone()).collect(), - start_ms, - end_ms: end_ms.saturating_add(1), - }); - let spans = match crate::span_batches::read_list_spans(client, connection, params).await { - Ok(spans) => spans, - Err(Error::ReadTooLarge) => { - return stream::iter(runs) - .then(|row| async move { - match get_trace(client, connection, access, &row.trace_id, &row.trace_ref).await - { - Ok(trace) => { - Ok(trace.map_or_else(|| listed_summary(row), |trace| trace.summary)) - } - Err(Error::ReadTooLarge) => Ok(listed_summary(row)), - Err(error) => Err(error), - } - }) - .try_collect() - .await; - } - Err(error) => return Err(error), - }; - let by_trace = spans.into_iter().into_group_map_by(|span| { - ( - span.team_id.clone(), - span.api_key_hash.clone(), - span.trace_id.clone(), + async fn run_spans( + &self, + params: &contracts::TracePageSpansParams, + snapshot_ms: u64, + ) -> Result, StoreError> { + crate::span_batches::read_list_spans( + &self.client, + &self.connection, + TracePageSpansParams::from(params.clone()), + snapshot_ms, ) - }); - let summaries = runs - .iter() - .map(|row| { - let spans = by_trace - .get(&( - row.team_id.clone(), - row.api_key_hash.clone(), - row.trace_id.clone(), - )) - .map(Vec::as_slice) - .unwrap_or_default(); - async move { - let spend_rows = spend(client, connection, access, spans).await; - resolve_trace(&row.trace_id, &row.trace_ref, spans, &spend_rows) - .map_or_else(|| listed_summary(row), |trace| trace.summary) - } - }) - .collect::>(); - Ok(stream::iter(summaries) - .buffered(SPEND_CONCURRENCY) - .collect() - .await) -} - -pub async fn get_trace( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - trace_id: &str, - trace_ref: &str, -) -> Result, Error> { - let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else { - return Ok(None); - }; - let params = TraceSpansParams { - access: access.clone(), - trace_id: trace_id.to_owned(), - trace_ref: trace_ref.clone(), - }; - let rows = crate::span_batches::read_spans(client, connection, params, u64::MAX).await?; - if rows.is_empty() { - return Ok(None); + .await } - let spend_rows = spend(client, connection, access, &rows).await; - Ok(resolve_trace(trace_id, &trace_ref, &rows, &spend_rows)) -} -#[derive(Deserialize, Serialize)] -struct SpanPosition { - trace_ref: String, - snapshot_ms: u64, - offset: usize, - version: String, -} - -pub async fn get_trace_page( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - trace_id: &str, - trace_ref: &str, - cursor: Option<&str>, - page_size: u32, -) -> Result, Error> { - if !(1..=500).contains(&page_size) { - return Err(Error::InvalidParameters); + async fn spend( + &self, + params: &contracts::SpendByResponseIdsParams, + ) -> Result, StoreError> { + crate::span_batches::read_spend( + &self.client, + &self.connection, + SpendByResponseIdsParams::from(params.clone()), + ) + .await } - let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else { - return Ok(None); - }; - let position = match cursor { - Some(cursor) => { - let position: SpanPosition = decode_cursor(cursor, "span")?; - if position.trace_ref != trace_ref || position.snapshot_ms == 0 { - return Err(Error::InvalidCursor("span")); - } - position + + async fn span_detail( + &self, + params: &contracts::SpanDetailParams, + ) -> Result, StoreError> { + match fetch::(&self.client, &self.connection, params).await { + Ok(rows) => Ok(rows.into_iter().next()), + Err(error) => Err(failed(error)), } - None => SpanPosition { - trace_ref: trace_ref.clone(), - snapshot_ms: (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) - as u64, - offset: 0, - version: String::new(), - }, - }; - let key = SnapshotKey::new( - connection.url().as_str(), - access, - trace_id, - &trace_ref, - position.snapshot_ms, - ) - .map_err(|_| Error::InvalidParameters)?; - let snapshot = match TRACE_SNAPSHOTS.get(&key).await { - Some(snapshot) => snapshot, - None => { - let params = TraceSpansParams { - access: access.clone(), - trace_id: trace_id.to_owned(), - trace_ref: trace_ref.clone(), - }; - let rows = - crate::span_batches::read_spans(client, connection, params, position.snapshot_ms) - .await?; - let spend_rows = spend(client, connection, access, &rows).await; - let Some(trace) = resolve_trace(trace_id, &trace_ref, &rows, &spend_rows) else { - return Ok(None); - }; - TRACE_SNAPSHOTS.insert(key, trace).await? + } + + async fn span_error( + &self, + params: &contracts::SpanErrorParams, + ) -> Result, StoreError> { + let storage_params = SpanErrorParams::from(params.clone()); + match fetch::(&self.client, &self.connection, &storage_params).await { + Ok(rows) => Ok(rows.into_iter().next().map(|row| row.0)), + Err(error) => Err(failed(error)), } - }; - let spans = &snapshot.trace().spans; - if cursor.is_some() && position.version != snapshot.version() { - return Err(Error::TraceChanged); - } - let mut trace = Trace { - summary: snapshot.trace().summary.clone(), - agents: snapshot.trace().agents.clone(), - spans: Vec::new(), - next_cursor: None, - }; - if position.offset > spans.len() { - return Err(Error::InvalidCursor("span")); - } - let end = position - .offset - .saturating_add(page_size as usize) - .min(spans.len()); - trace.next_cursor = (end < spans.len()).then(|| { - encode_cursor(&SpanPosition { - offset: end, - version: snapshot.version().to_owned(), - ..position - }) - }); - trace.spans = spans[position.offset..end].to_vec(); - while serde_json::to_vec(&trace) - .map_err(|_| Error::InvalidResponse)? - .len() - > litellm_storage_clickhouse::READ_LIMITS.response_bytes - { - if trace.spans.len() <= 1 { - return Err(Error::ReadTooLarge); - } - trace.spans.truncate(trace.spans.len() / 2); - trace.next_cursor = Some(encode_cursor(&SpanPosition { - trace_ref: trace_ref.clone(), - snapshot_ms: position.snapshot_ms, - offset: position.offset + trace.spans.len(), - version: snapshot.version().to_owned(), - })); - } - Ok(Some(trace)) -} - -pub async fn get_span( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - trace_id: &str, - span_id: &str, - trace_ref: &str, -) -> Result, Error> { - let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else { - return Ok(None); - }; - let params = SpanDetailParams { - access: access.clone(), - trace_id: trace_id.to_owned(), - trace_ref, - span_id: span_id.to_owned(), - }; - let row = fetch::(client, connection, ¶ms) - .await? - .into_iter() - .next(); - Ok(row.map(|row| SpanDetail { - input_ui: to_ui_content(&row.input), - output_ui: to_ui_content(&row.output), - span_id: row.span_id, - input: row.input, - output: row.output, - attributes: row.attributes, - })) -} - -pub async fn get_span_error( - client: &Client, - connection: &Connection, - access: &ReadAccessParams, - trace_id: &str, - span_id: &str, - trace_ref: &str, - cursor: Option<&str>, -) -> Result, Error> { - let position = error_position(cursor)?; - let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else { - return Ok(None); - }; - let offset = position.as_ref().map_or(0, |position| position.offset); - let params = SpanErrorParams::from(contracts::SpanErrorParams { - access: access.clone(), - trace_id: trace_id.to_owned(), - trace_ref, - span_id: span_id.to_owned(), - error_offset: offset, - error_version: position - .map(|position| position.version) - .unwrap_or_default(), - }); - let Some(row) = fetch::(client, connection, ¶ms) - .await? - .into_iter() - .next() - else { - return Ok(None); - }; - let row = row.0; - let next_offset = offset + row.message.chars().count() as u64; - let next_cursor = (next_offset < row.total_chars).then(|| { - encode_cursor(&ErrorPosition { - offset: next_offset, - version: row.version, - }) - }); - Ok(Some(SpanErrorPage { - span_id: row.span_id, - message: row.message, - total_chars: row.total_chars, - next_cursor, - })) -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::*; - - #[rstest] - fn trace_cursor_round_trips_the_last_listed_run() { - let cursor = encode_cursor(&(1_790_742_989_377_i64, "4bad42b84e9de3ba46fc870185f8f023")); - assert_eq!( - trace_position(Some(&cursor)).unwrap(), - ( - 1_790_742_989_377, - "4bad42b84e9de3ba46fc870185f8f023".to_owned() - ) - ); - assert_eq!(trace_position(None).unwrap(), (0, String::new())); - assert_eq!(trace_position(Some("")).unwrap(), (0, String::new())); - } - - #[rstest] - #[case::not_base64("abc")] - #[case::not_json("bm90LWpzb24=")] - #[case::numeric_reference("WzEsIDJd")] - #[case::zero_start("WzAsICJ0Il0=")] - fn malformed_trace_cursors_are_rejected(#[case] cursor: &str) { - assert!(matches!( - trace_position(Some(cursor)), - Err(Error::InvalidCursor("trace")) - )); - } - - #[rstest] - #[case::not_base64("garbage")] - #[case::missing_fields("e30=")] - #[case::not_an_object("WzEsMl0=")] - fn malformed_diagnostic_cursors_are_rejected(#[case] cursor: &str) { - assert!(matches!( - error_position(Some(cursor)), - Err(Error::InvalidCursor("diagnostic")) - )); - } - - #[rstest] - #[case::lowercase_version("a".repeat(64))] - #[case::short_version("A".repeat(63))] - fn diagnostic_cursor_requires_a_content_version(#[case] version: String) { - let cursor = encode_cursor(&ErrorPosition { offset: 1, version }); - assert!(matches!( - error_position(Some(&cursor)), - Err(Error::InvalidCursor("diagnostic")) - )); } } + +fn failed(error: StorageError) -> StoreError { + StoreError::Failed(Error::Storage(error)) +} diff --git a/litellm-rust/crates/traces-clickhouse/src/schema.rs b/litellm-rust/crates/traces-clickhouse/src/schema.rs index 07590b59338..6a39bd24041 100644 --- a/litellm-rust/crates/traces-clickhouse/src/schema.rs +++ b/litellm-rust/crates/traces-clickhouse/src/schema.rs @@ -3,8 +3,7 @@ use litellm_migrate::Migration; use serde::Serialize; use std::time::Duration; -use super::Connection; -use super::Error; +use super::{Connection, Error}; const SCHEMA_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); diff --git a/litellm-rust/crates/traces-clickhouse/src/span_batches.rs b/litellm-rust/crates/traces-clickhouse/src/span_batches.rs index d66ad9506da..5e406d67cfa 100644 --- a/litellm-rust/crates/traces-clickhouse/src/span_batches.rs +++ b/litellm-rust/crates/traces-clickhouse/src/span_batches.rs @@ -1,15 +1,20 @@ -use futures_util::{TryStreamExt, stream}; -use itertools::Itertools; +//! Keyset-paged reads that shrink their page when ClickHouse rejects a response as too large and +//! stop accumulating once a graph exceeds the interactive budget. + +use std::{future::Future, marker::PhantomData}; + use litellm_http::Client; use litellm_storage_clickhouse::{Query, fetch}; use litellm_traces::query::named as contracts; -use serde::Serialize; +use litellm_traces_cache::{MAX_GRAPH_BYTES, MAX_GRAPH_SPANS, StoreError}; +use serde::{Serialize, de::DeserializeOwned}; -use crate::{Connection, Error, query::named::TraceSpansRow}; +use crate::{ + Connection, Error, + query::named::{SpendByResponseIdsParams, SpendByResponseIdsRow, TraceSpansRow}, +}; const PAGE_SIZE: u32 = 256; -pub(crate) const MAX_GRAPH_BYTES: usize = 64 * 1024 * 1024; -const MAX_GRAPH_SPANS: usize = 100_000; #[derive(Default)] struct ReadBudget { @@ -18,40 +23,137 @@ struct ReadBudget { } impl ReadBudget { - fn checked_add(&self, bytes: usize, rows: usize) -> Result { - let next = Self { - bytes: self.bytes.saturating_add(bytes), - rows: self.rows.saturating_add(rows), - }; - if next.bytes > MAX_GRAPH_BYTES || next.rows > MAX_GRAPH_SPANS { - return Err(Error::ReadTooLarge); + fn reserve(&mut self, bytes: usize) -> Result<(), StoreError> { + self.bytes = self.bytes.saturating_add(bytes); + if self.bytes > MAX_GRAPH_BYTES || self.rows == MAX_GRAPH_SPANS { + return Err(StoreError::TooLarge); } - Ok(next) - } - - fn record(&mut self, row: &impl Serialize) -> Result<(), Error> { - let bytes = serde_json::to_vec(row).map_err(|_| Error::InvalidResponse)?; - *self = self.checked_add(bytes.len(), 1)?; + self.rows += 1; Ok(()) } + + fn record(&mut self, row: &impl Serialize) -> Result<(), StoreError> { + let bytes = + serde_json::to_vec(row).map_err(|_| StoreError::Failed(Error::InvalidResponse))?; + self.reserve(bytes.len()) + } +} + +/// One keyset position in a paged query: the SQL reads the cursor fields of `Self` plus the +/// `page_size` that [`Batch`] adds. +trait Keyset: Serialize + Sized + Send + Sync { + type Row: Serialize + DeserializeOwned + Send; + const SQL: &'static str; + + fn after(self, last: &Self::Row) -> Self; } #[derive(Serialize)] -struct Parameters { +struct Batch { + #[serde(flatten)] + keyset: K, + page_size: u32, +} + +trait PageSource { + fn page( + &self, + batch: &Batch, + ) -> impl Future, litellm_storage_clickhouse::Error>> + Send; +} + +struct Paged(PhantomData); + +impl Query for Paged { + type Params = Batch; + type Row = K::Row; + const SQL: &'static str = K::SQL; +} + +/// Reads every row after `keyset`. A page ClickHouse rejects as too large is retried at half the +/// size, and the smaller page is kept for the rest of the read because row sizes within one graph +/// rarely shrink again. Halving a one-row page means a single row exceeds the response limit. +async fn read_all>( + source: &S, + keyset: K, +) -> Result, StoreError> { + let mut batch = Batch { + keyset, + page_size: PAGE_SIZE, + }; + let mut rows = Vec::new(); + let mut budget = ReadBudget::default(); + loop { + let page = match source.page(&batch).await { + Err(litellm_storage_clickhouse::Error::ResponseTooLarge) if batch.page_size > 1 => { + batch.page_size /= 2; + continue; + } + Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => { + return Err(StoreError::TooLarge); + } + result => result.map_err(|error| StoreError::Failed(Error::Storage(error)))?, + }; + let complete = page.len() < batch.page_size as usize; + for row in &page { + budget.record(row)?; + } + if let Some(last) = page.last() { + batch.keyset = batch.keyset.after(last); + } + rows.extend(page); + if complete { + return Ok(rows); + } + } +} + +struct ClickHouse<'a> { + client: &'a Client, + connection: &'a Connection, +} + +impl PageSource for ClickHouse<'_> { + fn page( + &self, + batch: &Batch, + ) -> impl Future, litellm_storage_clickhouse::Error>> + Send { + fetch::>(self.client, self.connection, batch) + } +} + +async fn read_paged( + client: &Client, + connection: &Connection, + keyset: K, +) -> Result, StoreError> { + let source = ClickHouse { client, connection }; + read_all(&source, keyset).await +} + +fn by_start(mut rows: Vec) -> Vec { + rows.sort_by_key(|row| row.start_ns); + rows +} + +#[derive(Serialize)] +struct SpanKeyset { #[serde(flatten)] trace: contracts::TraceSpansParams, after_span_id: String, - page_size: u32, snapshot_ms: u64, } -struct SpanBatch; - -impl Query for SpanBatch { - type Params = Parameters; +impl Keyset for SpanKeyset { type Row = TraceSpansRow; - const SQL: &'static str = include_str!("../query/trace_span_batch.sql"); + + fn after(self, last: &TraceSpansRow) -> Self { + Self { + after_span_id: last.0.span_id.clone(), + ..self + } + } } pub(crate) async fn read_spans( @@ -59,199 +161,109 @@ pub(crate) async fn read_spans( connection: &Connection, trace: contracts::TraceSpansParams, snapshot_ms: u64, -) -> Result, Error> { - let mut parameters = Parameters { +) -> Result, StoreError> { + let keyset = SpanKeyset { trace, after_span_id: String::new(), - page_size: PAGE_SIZE, snapshot_ms, }; - let mut spans = Vec::new(); - let mut budget = ReadBudget::default(); - loop { - let page = match fetch::(client, connection, ¶meters).await { - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) - if parameters.page_size > 1 => - { - parameters.page_size /= 2; - continue; - } - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => { - return Err(Error::ReadTooLarge); - } - result => result?, - }; - let complete = page.len() < parameters.page_size as usize; - if let Some(last) = page.last() { - parameters.after_span_id.clone_from(&last.0.span_id); - } - for row in page { - budget.record(&row)?; - spans.push(row.0); - } - if complete { - spans.sort_by_key(|row| row.start_ns); - return Ok(spans); - } - parameters.page_size = (parameters.page_size * 2).min(PAGE_SIZE); - } + let rows = read_paged(client, connection, keyset).await?; + Ok(by_start(rows.into_iter().map(|row| row.0).collect())) } #[derive(Serialize)] -struct ListParameters { +struct ListSpanKeyset { #[serde(flatten)] runs: crate::query::named::TracePageSpansParams, after_team: String, after_key: String, after_trace: String, after_span: String, - page_size: u32, snapshot_ms: u64, } -struct ListSpanBatch; - -impl Query for ListSpanBatch { - type Params = ListParameters; +impl Keyset for ListSpanKeyset { type Row = TraceSpansRow; - const SQL: &'static str = include_str!("../query/trace_list_span_batch.sql"); + + fn after(self, last: &TraceSpansRow) -> Self { + Self { + after_team: last.0.team_id.clone(), + after_key: last.0.api_key_hash.clone(), + after_trace: last.0.trace_id.clone(), + after_span: last.0.span_id.clone(), + ..self + } + } } pub(crate) async fn read_list_spans( client: &Client, connection: &Connection, runs: crate::query::named::TracePageSpansParams, -) -> Result, Error> { - let parameters = ListParameters { + snapshot_ms: u64, +) -> Result, StoreError> { + let keyset = ListSpanKeyset { runs, after_team: String::new(), after_key: String::new(), after_trace: String::new(), after_span: String::new(), - page_size: PAGE_SIZE, - snapshot_ms: (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as u64, + snapshot_ms, }; - let pages = stream::try_unfold( - (Some(parameters), ReadBudget::default()), - |(parameters, budget)| async move { - let Some(parameters) = parameters else { - return Ok(None); - }; - let page = match fetch::(client, connection, ¶meters).await { - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) - if parameters.page_size > 1 => - { - let retry = ListParameters { - page_size: parameters.page_size / 2, - ..parameters - }; - return Ok(Some((Vec::new(), (Some(retry), budget)))); - } - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => { - return Err(Error::ReadTooLarge); - } - result => result?, - }; - let next = page - .last() - .filter(|_| page.len() == parameters.page_size as usize) - .map(|last| ListParameters { - after_team: last.0.team_id.clone(), - after_key: last.0.api_key_hash.clone(), - after_trace: last.0.trace_id.clone(), - after_span: last.0.span_id.clone(), - page_size: (parameters.page_size * 2).min(PAGE_SIZE), - ..parameters - }); - let next_budget = page.iter().try_fold(budget, |budget, row| { - let bytes = serde_json::to_vec(row).map_err(|_| Error::InvalidResponse)?; - budget.checked_add(bytes.len(), 1) - })?; - Ok(Some((page, (next, next_budget)))) - }, - ) - .try_collect::>() - .await?; - Ok(pages - .into_iter() - .flatten() - .map(|row| row.0) - .sorted_by_key(|row| row.start_ns) - .collect()) + let rows = read_paged(client, connection, keyset).await?; + Ok(by_start(rows.into_iter().map(|row| row.0).collect())) } #[derive(Serialize)] -struct SpendParameters { +struct SpendKeyset { #[serde(flatten)] - lookup: crate::query::named::SpendByResponseIdsParams, + lookup: SpendByResponseIdsParams, has_cursor: u8, after_team: String, after_ms: i64, after_id: String, - page_size: u32, } -struct SpendBatch; - -impl Query for SpendBatch { - type Params = SpendParameters; - type Row = crate::query::named::SpendByResponseIdsRow; - +impl Keyset for SpendKeyset { + type Row = SpendByResponseIdsRow; const SQL: &'static str = include_str!("../query/spend_batch.sql"); + + fn after(self, last: &SpendByResponseIdsRow) -> Self { + Self { + has_cursor: 1, + after_team: last.0.team_id.clone(), + after_ms: last.0.start_ms, + after_id: last.0.request_id.clone(), + ..self + } + } } pub(crate) async fn read_spend( client: &Client, connection: &Connection, - lookup: crate::query::named::SpendByResponseIdsParams, -) -> Result, Error> { - let mut parameters = SpendParameters { + lookup: SpendByResponseIdsParams, +) -> Result, StoreError> { + let keyset = SpendKeyset { lookup, has_cursor: 0, after_team: String::new(), after_ms: 0, after_id: String::new(), - page_size: PAGE_SIZE, }; - let mut rows = Vec::new(); - let mut budget = ReadBudget::default(); - loop { - let page = match fetch::(client, connection, ¶meters).await { - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) - if parameters.page_size > 1 => - { - parameters.page_size /= 2; - continue; - } - Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => { - return Err(Error::ReadTooLarge); - } - result => result?, - }; - let complete = page.len() < parameters.page_size as usize; - if let Some(last) = page.last() { - parameters.has_cursor = 1; - parameters.after_team.clone_from(&last.0.team_id); - parameters.after_ms = last.0.start_ms; - parameters.after_id.clone_from(&last.0.request_id); - } - for row in page { - budget.record(&row)?; - rows.push(row.0); - } - if complete { - return Ok(rows); - } - parameters.page_size = (parameters.page_size * 2).min(PAGE_SIZE); - } + let rows = read_paged(client, connection, keyset).await?; + Ok(rows.into_iter().map(|row| row.0).collect()) } #[cfg(test)] mod tests { - use super::*; + use std::sync::Mutex; + use rstest::rstest; + use super::*; + #[rstest] #[case::byte_boundary(MAX_GRAPH_BYTES - 1, 0, 1, false)] #[case::byte_overflow(MAX_GRAPH_BYTES - 1, 0, 2, true)] @@ -264,7 +276,77 @@ mod tests { #[case] next: usize, #[case] rejected: bool, ) { - let budget = ReadBudget { bytes, rows }; - assert_eq!(budget.checked_add(next, 1).is_err(), rejected); + let mut budget = ReadBudget { bytes, rows }; + assert_eq!(budget.reserve(next).is_err(), rejected); + } + + #[derive(Serialize)] + struct Numbers { + after: u32, + } + + impl Keyset for Numbers { + type Row = u32; + const SQL: &'static str = ""; + + fn after(self, last: &u32) -> Self { + Self { after: *last } + } + } + + /// A table of `total` rows whose transport rejects any page larger than `largest_page`. + struct Table { + total: u32, + largest_page: u32, + requests: Mutex>, + } + + impl PageSource for Table { + async fn page( + &self, + batch: &Batch, + ) -> Result, litellm_storage_clickhouse::Error> { + self.requests.lock().unwrap().push(batch.page_size); + if batch.page_size > self.largest_page { + return Err(litellm_storage_clickhouse::Error::ResponseTooLarge); + } + let end = (batch.keyset.after + batch.page_size).min(self.total); + Ok((batch.keyset.after + 1..=end).collect()) + } + } + + #[rstest] + #[case::fits(1000, PAGE_SIZE, &[256, 256, 256, 256])] + #[case::uniform_large_rows(1000, 100, &[256, 128, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64])] + #[tokio::test] + async fn a_rejected_page_size_is_not_retried( + #[case] total: u32, + #[case] largest_page: u32, + #[case] requests: &[u32], + ) { + let table = Table { + total, + largest_page, + requests: Mutex::new(Vec::new()), + }; + let rows = read_all(&table, Numbers { after: 0 }).await.unwrap(); + assert_eq!(rows, (1..=total).collect::>()); + assert_eq!(table.requests.lock().unwrap().as_slice(), requests); + } + + #[rstest] + #[tokio::test] + async fn a_single_oversized_row_fails_the_read() { + let table = Table { + total: 10, + largest_page: 0, + requests: Mutex::new(Vec::new()), + }; + let result = read_all(&table, Numbers { after: 0 }).await; + assert!(matches!(result, Err(StoreError::TooLarge)), "{result:?}"); + assert_eq!( + table.requests.lock().unwrap().as_slice(), + &[256, 128, 64, 32, 16, 8, 4, 2, 1] + ); } } diff --git a/litellm-rust/crates/traces-clickhouse/src/sql.rs b/litellm-rust/crates/traces-clickhouse/src/sql.rs index 80b3ec88534..dfa0618773a 100644 --- a/litellm-rust/crates/traces-clickhouse/src/sql.rs +++ b/litellm-rust/crates/traces-clickhouse/src/sql.rs @@ -3,8 +3,10 @@ use std::collections::BTreeMap; use litellm_http::Client; use litellm_traces::ReadQuery; -use super::query::{lens::*, named::*}; -use super::{Connection, Error, Parameter}; +use super::{ + Connection, Error, Parameter, + query::{lens::*, named::*}, +}; use litellm_storage_clickhouse::{Query, fetch_json}; pub async fn execute_named_read( diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index cd23484566f..09fd3e78da8 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -1887,10 +1887,10 @@ async fn named_and_sql_readers_share_request_log_visibility( #[case] legacy_key: Option<&str>, #[case] expected: Vec<&str>, ) -> TestResult { - use litellm_traces_clickhouse::query::named::{ - ReadAccessParams, SpendByResponseIds, SpendByResponseIdsParams, + use litellm_traces_clickhouse::{ + QueryReaders, QueryScope, + query::named::{ReadAccessParams, SpendByResponseIds, SpendByResponseIdsParams}, }; - use litellm_traces_clickhouse::{QueryReaders, QueryScope}; let database = database?; let writer = Connection::writer(&database.url)?; diff --git a/litellm-rust/crates/traces-clickhouse/tests/reads.rs b/litellm-rust/crates/traces-clickhouse/tests/reads.rs index 2abb308f9c1..e1996d6af04 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/reads.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/reads.rs @@ -1,8 +1,10 @@ use std::collections::BTreeMap; +use litellm_http::Client; use litellm_traces::query::named::ReadAccessParams; +use litellm_traces_cache::{ReadError, TraceReader}; use litellm_traces_clickhouse::{ - Connection, InsertTable, QueryScope, get_trace, get_trace_page, insert_rows, list_traces, + ClickHouseTraces, Connection, InsertTable, QueryScope, insert_rows, }; use rstest::rstest; use serde_json::json; @@ -14,6 +16,13 @@ mod support; use fixtures::{DATABASE, SeededDatabase, migrated_database, seeded_database}; use support::TestResult; +fn make_reader(client: &Client, connection: Connection) -> (TraceReader, ClickHouseTraces) { + ( + TraceReader::new(litellm_storage_clickhouse::READ_LIMITS.response_bytes), + ClickHouseTraces::new(client.clone(), connection), + ) +} + #[rstest] #[case::api_key("key-a", "")] #[case::user("", "user-a")] @@ -74,16 +83,19 @@ async fn list_costs_match_each_run_when_response_ids_are_reused( .collect(), ) .await?; - let reader = fixture + let connection = fixture .readers .connection(client, &QueryScope::All, "fixture-secret") .await?; + let (reader, store) = make_reader(client, connection); let access = ReadAccessParams { all_teams: false, user_id: user_id.into(), team_ids: vec!["team-a".into()], }; - let page = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 50).await?; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; assert_eq!(page.data.len(), runs.len()); for (trace_id, _, cost) in runs { let summary = page @@ -91,7 +103,8 @@ async fn list_costs_match_each_run_when_response_ids_are_reused( .iter() .find(|summary| summary.trace_id == trace_id) .ok_or("missing run")?; - let detail = get_trace(client, &reader, &access, trace_id, &summary.trace_ref) + let detail = reader + .get_trace(&store, &access, trace_id, &summary.trace_ref) .await? .ok_or("missing trace")?; assert_eq!(detail.summary.spend, Some(cost)); @@ -199,16 +212,19 @@ async fn large_runs_remain_complete_under_default_reader_limits( .collect::>(); insert_rows(client, &writer, DATABASE, InsertTable::SpendLogs, costs).await?; } - let reader = fixture + let connection = fixture .readers .connection(client, &QueryScope::All, "fixture-secret") .await?; + let (reader, store) = make_reader(client, connection); let access = ReadAccessParams { all_teams: false, user_id: String::new(), team_ids: vec!["team-a".into()], }; - let page = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 500).await?; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 500) + .await?; assert_eq!(page.data.len(), runs); assert!( page.data @@ -222,42 +238,17 @@ async fn large_runs_remain_complete_under_default_reader_limits( .send() .await? .error_for_status()?; - let read_queries = client.post(writer.url().clone()).body(format!( - "SELECT count() FROM system.query_log WHERE type = 'QueryFinish' AND current_database = '{DATABASE}' AND query LIKE '%FROM otel_traces AS o%' AND query NOT LIKE '%system.query_log%'" - )).send().await?.error_for_status()?.text().await?; - let read_queries = read_queries.trim().parse::()?; - assert!( - read_queries > 0 && read_queries < runs, - "{read_queries} span queries for {runs} runs" - ); - if costed { - let overlapping = client - .post(writer.url().clone()) - .body(format!( - "WITH spend_reads AS ( - SELECT query_start_time_microseconds AS started, event_time_microseconds AS finished - FROM system.query_log - WHERE type = 'QueryFinish' AND current_database = '{DATABASE}' - AND query LIKE '%FROM spend_logs FINAL%' AND query NOT LIKE '%system.query_log%' - ), events AS ( - SELECT started AS at, 1 AS delta FROM spend_reads - UNION ALL SELECT finished AS at, -1 AS delta FROM spend_reads - ) - SELECT max(active) FROM ( - SELECT sum(delta) OVER (ORDER BY at, delta ROWS UNBOUNDED PRECEDING) AS active - FROM events - )" - )) - .send() - .await? - .error_for_status()? - .text() - .await? - .trim() - .parse::()?; + for table in ["otel_traces AS o", "spend_logs FINAL"] + .into_iter() + .take(if costed { 2 } else { 1 }) + { + let read_queries = client.post(writer.url().clone()).body(format!( + "SELECT count() FROM system.query_log WHERE type = 'QueryFinish' AND current_database = '{DATABASE}' AND query LIKE '%FROM {table}%' AND query NOT LIKE '%system.query_log%'" + )).send().await?.error_for_status()?.text().await?; + let read_queries = read_queries.trim().parse::()?; assert!( - (2..=4).contains(&overlapping), - "{overlapping} simultaneous spend reads for {runs} runs" + read_queries > 0 && read_queries < runs, + "{read_queries} {table} queries for {runs} runs" ); } } @@ -281,7 +272,8 @@ async fn large_runs_remain_complete_under_default_reader_limits( .find(|run| run.trace_id == "trace-0000") .ok_or("missing run")? .trace_ref; - let detail = get_trace(client, &reader, &access, "trace-0000", trace_ref) + let detail = reader + .get_trace(&store, &access, "trace-0000", trace_ref) .await? .ok_or("missing trace")?; assert_eq!(detail.spans.len(), steps); @@ -303,24 +295,25 @@ async fn large_runs_remain_complete_under_default_reader_limits( ..access.clone() }; assert!( - get_trace(client, &reader, &denied, "trace-0000", trace_ref) + reader + .get_trace(&store, &denied, "trace-0000", trace_ref) .await? .is_none() ); let mut cursor = None; let mut ids = Vec::new(); loop { - let page = get_trace_page( - client, - &reader, - &access, - "trace-0000", - trace_ref, - cursor.as_deref(), - 200, - ) - .await? - .ok_or("missing page")?; + let page = reader + .get_trace_page( + &store, + &access, + "trace-0000", + trace_ref, + cursor.as_deref(), + 200, + ) + .await? + .ok_or("missing page")?; assert_eq!(page.summary, detail.summary); assert!(page.spans.len() <= 200); assert!( @@ -329,17 +322,17 @@ async fn large_runs_remain_complete_under_default_reader_limits( ); if ids.is_empty() { assert!( - get_trace_page( - client, - &reader, - &denied, - "trace-0000", - trace_ref, - page.next_cursor.as_deref(), - 200, - ) - .await? - .is_none() + reader + .get_trace_page( + &store, + &denied, + "trace-0000", + trace_ref, + page.next_cursor.as_deref(), + 200, + ) + .await? + .is_none() ); client .post(writer.url().clone()) @@ -372,45 +365,43 @@ async fn cursor_pages_keep_a_tenant_scoped_snapshot_when_more_spans_arrive( ) -> TestResult { let fixture = seeded_database?; let client = &fixture.database.client; - let reader = fixture + let connection = fixture .readers .connection(client, &QueryScope::All, "fixture-secret") .await?; + let (reader, store) = make_reader(client, connection.clone()); let access = ReadAccessParams { all_teams: true, user_id: String::new(), team_ids: Vec::new(), }; - let listed = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 10).await?; + let listed = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 10) + .await?; let summary = listed .data .iter() .find(|summary| summary.span_count == 3) .ok_or("missing fixture")?; - let first = get_trace_page( - client, - &reader, - &access, - &summary.trace_id, - &summary.trace_ref, - None, - 1, - ) - .await? - .ok_or("missing first page")?; - let original_ids = get_trace( - client, - &reader, - &access, - &summary.trace_id, - &summary.trace_ref, - ) - .await? - .ok_or("missing trace")? - .spans - .into_iter() - .map(|span| span.span_id) - .collect::>(); + let first = reader + .get_trace_page( + &store, + &access, + &summary.trace_id, + &summary.trace_ref, + None, + 1, + ) + .await? + .ok_or("missing first page")?; + let original_ids = reader + .get_trace(&store, &access, &summary.trace_id, &summary.trace_ref) + .await? + .ok_or("missing trace")? + .spans + .into_iter() + .map(|span| span.span_id) + .collect::>(); let writer = Connection::writer(&fixture.database.url)?; insert_rows( client, @@ -434,17 +425,17 @@ async fn cursor_pages_keep_a_tenant_scoped_snapshot_when_more_spans_arrive( team_ids: vec!["not-this-team".into()], }; assert!( - get_trace_page( - client, - &reader, - &denied, - &summary.trace_id, - &summary.trace_ref, - first.next_cursor.as_deref(), - 1 - ) - .await? - .is_none() + reader + .get_trace_page( + &store, + &denied, + &summary.trace_id, + &summary.trace_ref, + first.next_cursor.as_deref(), + 1 + ) + .await? + .is_none() ); let first_cursor = first.next_cursor.clone(); let mut cursor = first.next_cursor; @@ -454,44 +445,45 @@ async fn cursor_pages_keep_a_tenant_scoped_snapshot_when_more_spans_arrive( .map(|span| span.span_id) .collect::>(); while let Some(current) = cursor { - let next = get_trace_page( - client, - &reader, - &access, - &summary.trace_id, - &summary.trace_ref, - Some(¤t), - 1, - ) - .await? - .ok_or("missing next page")?; + let next = reader + .get_trace_page( + &store, + &access, + &summary.trace_id, + &summary.trace_ref, + Some(¤t), + 1, + ) + .await? + .ok_or("missing next page")?; assert_eq!(next.summary.span_count, 3); ids.extend(next.spans.into_iter().map(|span| span.span_id)); cursor = next.next_cursor; } assert_eq!(ids, original_ids); - let refreshed = get_trace( - client, - &reader, - &access, - &summary.trace_id, - &summary.trace_ref, - ) - .await? - .ok_or("missing refreshed trace")?; + let cached = reader + .get_trace(&store, &access, &summary.trace_id, &summary.trace_ref) + .await? + .ok_or("missing cached trace")?; + assert_eq!(cached.spans.len(), 3); + let (fresh_reader, fresh_store) = make_reader(client, connection); + let refreshed = fresh_reader + .get_trace(&fresh_store, &access, &summary.trace_id, &summary.trace_ref) + .await? + .ok_or("missing refreshed trace")?; assert_eq!(refreshed.spans.len(), 4); assert!(matches!( - get_trace_page( - client, - &reader, - &access, - &summary.trace_id, - &summary.trace_ref, - Some("invalid"), - 1 - ) - .await, - Err(litellm_traces_clickhouse::Error::InvalidCursor("span")) + reader + .get_trace_page( + &store, + &access, + &summary.trace_id, + &summary.trace_ref, + Some("invalid"), + 1 + ) + .await, + Err(ReadError::InvalidCursor("span")) )); let backdated = json!({ "Timestamp": "2026-09-01 00:00:00.000000000", @@ -509,20 +501,21 @@ async fn cursor_pages_keep_a_tenant_scoped_snapshot_when_more_spans_arrive( .send() .await? .error_for_status()?; - let uncached_reader = + let uncached_connection = Connection::reader(&format!("{}?max_threads=1", fixture.database.url), DATABASE)?; - let changed = get_trace_page( - client, - &uncached_reader, - &access, - &summary.trace_id, - &summary.trace_ref, - first_cursor.as_deref(), - 1, - ) - .await; + let (uncached_reader, uncached_store) = make_reader(client, uncached_connection); + let changed = uncached_reader + .get_trace_page( + &uncached_store, + &access, + &summary.trace_id, + &summary.trace_ref, + first_cursor.as_deref(), + 1, + ) + .await; assert!( - matches!(changed, Err(litellm_traces_clickhouse::Error::TraceChanged)), + matches!(changed, Err(ReadError::TraceChanged)), "{changed:?}" ); Ok(()) @@ -535,16 +528,19 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( ) -> TestResult { let fixture = seeded_database?; let client = &fixture.database.client; - let reader = fixture + let connection = fixture .readers .connection(client, &QueryScope::All, "fixture-secret") .await?; + let (reader, store) = make_reader(client, connection.clone()); let access = ReadAccessParams { all_teams: true, user_id: String::new(), team_ids: Vec::new(), }; - let before = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 50).await?; + let before = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; let run = before .data .iter() @@ -571,7 +567,14 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( ])], ) .await?; - let after = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 50).await?; + let cached = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; + assert_eq!(cached.data, before.data); + let (reader, store) = make_reader(client, connection); + let after = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; assert_eq!(after.data.len(), before.data.len()); let limited = after .data @@ -588,17 +591,10 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( .all(|item| !item.resolution_limited) ); assert!(matches!( - get_trace_page( - client, - &reader, - &access, - &run.trace_id, - &run.trace_ref, - None, - 200 - ) - .await, - Err(litellm_traces_clickhouse::Error::ReadTooLarge) + reader + .get_trace_page(&store, &access, &run.trace_id, &run.trace_ref, None, 200) + .await, + Err(ReadError::TooLarge) )); Ok(()) } @@ -685,16 +681,19 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback .collect(), ) .await?; - let reader = fixture + let connection = fixture .readers .connection(client, &QueryScope::All, "fixture-secret") .await?; + let (reader, store) = make_reader(client, connection); let access = ReadAccessParams { all_teams: false, user_id: String::new(), team_ids: vec!["team-a".into()], }; - let page = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 50).await?; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; assert_eq!(page.data.len(), cases.len()); for (id, _, _, _, _, expected) in cases { let summary = page @@ -702,7 +701,8 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback .iter() .find(|summary| summary.trace_id == id) .ok_or("missing run")?; - let detail = get_trace(client, &reader, &access, id, &summary.trace_ref) + let detail = reader + .get_trace(&store, &access, id, &summary.trace_ref) .await? .ok_or("missing trace")?; assert_eq!(detail.summary.spend, expected, "{id}"); diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index 5765efaa62e..a981bc46471 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -15,7 +15,7 @@ pub struct ReadAccessParams { pub team_ids: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct ListTracesParams { #[serde(flatten)] pub access: ReadAccessParams, @@ -26,7 +26,7 @@ pub struct ListTracesParams { pub limit: u32, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct ListTracesRow { pub trace_id: String, pub trace_ref: String, @@ -56,7 +56,7 @@ pub struct ListTracesRow { pub request_ids: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct TraceSpansParams { #[serde(flatten)] pub access: ReadAccessParams, @@ -113,7 +113,7 @@ pub struct TraceSpansRow { pub user_id: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct TracePageSpansParams { #[serde(flatten)] pub access: ReadAccessParams, @@ -131,7 +131,7 @@ pub struct SpanDetailParams { pub span_id: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SpanDetailRow { pub span_id: String, pub input: String, @@ -139,7 +139,7 @@ pub struct SpanDetailRow { pub attributes: BTreeMap, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SpanErrorParams { #[serde(flatten)] pub access: ReadAccessParams, @@ -150,7 +150,7 @@ pub struct SpanErrorParams { pub error_version: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SpanErrorRow { pub span_id: String, pub message: String, @@ -158,7 +158,7 @@ pub struct SpanErrorRow { pub version: String, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SpendByResponseIdsParams { #[serde(flatten)] pub access: ReadAccessParams, @@ -169,7 +169,7 @@ pub struct SpendByResponseIdsParams { pub end_ms: i64, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct SpendByResponseIdsRow { pub request_id: String, pub litellm_call_id: String, diff --git a/litellm/constants.py b/litellm/constants.py index d58fc8a6318..69ff3cf7a5e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -55,6 +55,7 @@ DEFAULT_AGENT_TRACING_RETENTION_DAYS: Final = 14 OTLP_MAX_BODY_BYTES: Final = get_env_int("OTLP_MAX_BODY_BYTES", 16 * 1024 * 1024) OTLP_MAX_ATTRIBUTE_VALUE_BYTES: Final = get_env_int("OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 64 * 1024) OTLP_RETRY_AFTER_SECONDS: Final = get_env_int("OTLP_RETRY_AFTER_SECONDS", 2) +TRACE_READ_RETRY_AFTER_SECONDS: Final = get_env_int("TRACE_READ_RETRY_AFTER_SECONDS", 2) OTLP_MAX_CONCURRENT_INGESTS: Final = get_env_int("OTLP_MAX_CONCURRENT_INGESTS", 2) AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240) AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50) diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index 3ae9db0a600..12119f07962 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -20,6 +20,7 @@ from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper from litellm.proxy.lens.billing import validate_key from litellm.proxy.lens.inference import Deployment, deployment_prices from litellm.proxy.lens.models import ( + ActivitySelection, Claim, Execution, ExecutionContent, @@ -129,7 +130,7 @@ def required(lens: Lens | None) -> Lens: return lens -def validate_selection(settings: LensSettings) -> None: +def validate_selection(settings: ActivitySelection) -> None: for identity in settings.execution_ids: try: source, _, _, _ = parse_execution(identity) @@ -391,13 +392,13 @@ async def update_finding(lens_id: str, finding_id: str, body: FindingUpdate, aut class Preview(BaseModel): as_of: AwareDatetime | None = None offset: int = Field(default=0, ge=0) - settings: LensSettings + selection: ActivitySelection lookback_hours: LookbackHours = 24 @router.post("/preview/sample", response_model=Sample) async def preview_sample(body: Preview, auth: Auth, storage: StorageDep) -> Sample: - validate_selection(body.settings) + validate_selection(body.selection) now: Final = min(body.as_of or datetime.now(timezone.utc), datetime.now(timezone.utc)) try: start: Final = int((now - timedelta(hours=body.lookback_hours)).timestamp() * 1000) @@ -406,7 +407,7 @@ async def preview_sample(body: Preview, auth: Auth, storage: StorageDep) -> Samp raise HTTPException(422, "Preview window exceeds the supported calendar range") from error return await source_reader(storage).sample( user_scope(auth), - body.settings, + body.selection, start, end, offset=body.offset, diff --git a/litellm/proxy/lens/models.py b/litellm/proxy/lens/models.py index e3a08103c8d..90c92cc7acd 100644 --- a/litellm/proxy/lens/models.py +++ b/litellm/proxy/lens/models.py @@ -45,23 +45,26 @@ class Check(Record): enabled: bool = True -class LensSettings(Record): - name: str = Field(min_length=1) - context: str = Field(default="") +class ActivitySelection(Record): source: Literal["traces", "requests", "both"] = "traces" - lookback_hours: LookbackHours = 24 service: str = Field(default="") agent_name: str = Field(default="") filters: tuple[MetadataFilter, ...] = Field(default=()) + sample_size: int | None = Field(default=None, ge=1) + sample_percent: float = Field(default=100, gt=0, le=100, allow_inf_nan=False) + team_id: str = "" + execution_ids: tuple[str, ...] = () + + +class LensSettings(ActivitySelection): + name: str = Field(min_length=1) + context: str = Field(default="") + lookback_hours: LookbackHours = 24 checks: tuple[Check, ...] = () model: str = Field(min_length=1) enabled: bool = True interval_minutes: IntervalMinutes = 15 - sample_size: int | None = Field(default=None, ge=1) - sample_percent: float = Field(default=100, gt=0, le=100, allow_inf_nan=False) concurrency: int = Field(default=8, ge=1) - team_id: str = "" - execution_ids: tuple[str, ...] = () monthly_budget: float = Field(default=100, gt=0, allow_inf_nan=False) @model_validator(mode="after") diff --git a/litellm/proxy/lens/sources.py b/litellm/proxy/lens/sources.py index 9af0f3679b6..e36653aa091 100644 --- a/litellm/proxy/lens/sources.py +++ b/litellm/proxy/lens/sources.py @@ -6,10 +6,10 @@ from typing import Final, Protocol, TypeAlias from pydantic import TypeAdapter from litellm.proxy.lens.models import ( + ActivitySelection, Evidence, Execution, ExecutionContent, - LensSettings, MetadataFilter, Sample, Scope, @@ -73,7 +73,7 @@ class SourceReader: async def sample( self, scope: Scope, - settings: LensSettings, + settings: ActivitySelection, start: int, end: int, offset: int = 0, diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py index 1563e4b5b55..0d3723c45b8 100644 --- a/litellm/proxy/tracing_endpoints.py +++ b/litellm/proxy/tracing_endpoints.py @@ -13,19 +13,21 @@ from dataclasses import dataclass from functools import partial from http.client import responses from types import MappingProxyType -from typing import Annotated, Final +from typing import Annotated, Final, Literal from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from pydantic import BaseModel, ConfigDict +from typing_extensions import assert_never from litellm._logging import verbose_proxy_logger -from litellm.constants import OTLP_RETRY_AFTER_SECONDS +from litellm.constants import OTLP_RETRY_AFTER_SECONDS, TRACE_READ_RETRY_AFTER_SECONDS from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.authorization import AllRows, ReadScope, resolve_trace_read_scope from litellm.proxy.auth.authorization_dependencies import LogTeamLookupDependency from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.http_parsing_utils import is_otlp_trace_request from litellm.proxy.tracing_runtime import provide_receiver, require_receiver +from litellm.rust_bridge.trace.errors import TraceChanged from litellm.rust_bridge.trace.generated.models import TraceQueryHelp from litellm.rust_bridge.trace.generated.types import ( AllQueryScope, @@ -135,6 +137,49 @@ async def ingest_otlp_traces( return Response(content=body, media_type=media_type) +class TraceReadFailure(BaseModel): + """The body of every failed trace read. Clients branch on `code`, never on `message`.""" + + model_config = ConfigDict(frozen=True) + + code: Literal["invalid_request", "trace_changed", "too_large", "unavailable"] + message: str + + +def read_failure(error: TraceChanged | ValueError | OverflowError | RuntimeError) -> HTTPException: + """One status per failure kind, so a client can tell a bad cursor (400, fix the request) from a + traversal it must restart (409), a result it cannot page through (413), and an outage it should + retry after `Retry-After` (503).""" + match error: + case TraceChanged(): + return HTTPException( + status_code=409, + detail=TraceReadFailure(code="trace_changed", message=str(error)).model_dump(), + ) + case ValueError(): + return HTTPException( + status_code=400, detail=TraceReadFailure(code="invalid_request", message=str(error)).model_dump() + ) + case OverflowError(): + return HTTPException( + status_code=413, + detail=TraceReadFailure( + code="too_large", message="Trace is too large for this view. Use a filtered trace query." + ).model_dump(), + ) + case RuntimeError(): + verbose_proxy_logger.warning("Trace read unavailable: %s", error) + return HTTPException( + status_code=503, + detail=TraceReadFailure( + code="unavailable", message="Traces are temporarily unavailable. Please try again." + ).model_dump(), + headers={"Retry-After": str(TRACE_READ_RETRY_AFTER_SECONDS)}, + ) + case _: + return assert_never(error) + + @router.get("/v1/traces", response_model=TracePage) async def list_agent_traces( context: Annotated[TraceAccessContext, Depends(provide_trace_access)], @@ -151,15 +196,8 @@ async def list_agent_traces( end_ms=end_ms if end_ms is not None else now_ms, cursor=cursor, ) - except ValueError as error: - raise HTTPException(status_code=400, detail=str(error)) from error - except OverflowError as error: - raise HTTPException( - status_code=413, detail="Trace is too large for this view. Use a filtered trace query." - ) from error - except RuntimeError as error: - verbose_proxy_logger.warning("Trace read unavailable: %s", error) - raise HTTPException(status_code=503, detail="Traces are temporarily unavailable. Please try again.") from error + except (TraceChanged, ValueError, OverflowError, RuntimeError) as error: + raise read_failure(error) from error class TraceQueryRequest(BaseModel): @@ -241,15 +279,8 @@ async def get_agent_trace( tracing, scope = context.reader() try: trace: Final = await tracing.get_trace(trace_id, scope, trace_ref, cursor, page_size) - except ValueError as error: - raise HTTPException(status_code=400, detail=str(error)) from error - except OverflowError as error: - raise HTTPException( - status_code=413, detail="Trace is too large for this view. Use a filtered trace query." - ) from error - except RuntimeError as error: - verbose_proxy_logger.warning("Trace read unavailable: %s", error) - raise HTTPException(status_code=503, detail="Traces are temporarily unavailable. Please try again.") from error + except (TraceChanged, ValueError, OverflowError, RuntimeError) as error: + raise read_failure(error) from error if trace is None: raise HTTPException(status_code=404, detail=f"Trace {trace_id} not found") return trace @@ -265,15 +296,8 @@ async def get_agent_trace_span( tracing, scope = context.reader() try: span: Final = await tracing.get_span(trace_id, span_id, scope, trace_ref) - except ValueError as error: - raise HTTPException(status_code=400, detail=str(error)) from error - except OverflowError as error: - raise HTTPException( - status_code=413, detail="Trace is too large for this view. Use a filtered trace query." - ) from error - except RuntimeError as error: - verbose_proxy_logger.warning("Trace read unavailable: %s", error) - raise HTTPException(status_code=503, detail="Traces are temporarily unavailable. Please try again.") from error + except (TraceChanged, ValueError, OverflowError, RuntimeError) as error: + raise read_failure(error) from error if span is None: raise HTTPException(status_code=404, detail=f"Span {span_id} not found") return span @@ -290,15 +314,8 @@ async def get_agent_trace_span_error( try: tracing, scope = context.reader() page: Final = await tracing.get_span_error(trace_id, span_id, scope, trace_ref, cursor) - except ValueError as error: - raise HTTPException(status_code=400, detail=str(error)) from error - except OverflowError as error: - raise HTTPException( - status_code=413, detail="Trace is too large for this view. Use a filtered trace query." - ) from error - except RuntimeError as error: - verbose_proxy_logger.warning("Trace read unavailable: %s", error) - raise HTTPException(status_code=503, detail="Traces are temporarily unavailable. Please try again.") from error + except (TraceChanged, ValueError, OverflowError, RuntimeError) as error: + raise read_failure(error) from error if page is None: raise HTTPException(status_code=404, detail="Span diagnostic not found or no longer available") return page diff --git a/litellm/rust_bridge/trace/errors.py b/litellm/rust_bridge/trace/errors.py new file mode 100644 index 00000000000..3c84d233be4 --- /dev/null +++ b/litellm/rust_bridge/trace/errors.py @@ -0,0 +1,5 @@ +class TraceChanged(Exception): + """The paging snapshot no longer matches the stored trace, so the client must start a new traversal. + + Raised by the Rust trace reader when a cursor's snapshot version differs from the graph it rebuilt. + """ diff --git a/scripts/lens_dev.sh b/scripts/lens_dev.sh index d8918f02fc5..1add480c227 100755 --- a/scripts/lens_dev.sh +++ b/scripts/lens_dev.sh @@ -7,7 +7,8 @@ # (default: random, generated once into .lens-dev/master_key) # LENS_DEV_CONFIG proxy config to use instead of the generated one # LENS_DEV_DATABASE_URL Postgres URL (default: the tracing stack's litellm DB on :15432) -# LENS_DEV_REBUILD_RUST=1 rebuild the Rust bridge even if it imports +# LENS_DEV_SEED trace seed profile (default|large), same as --seed +# LENS_DEV_SEED_LOGS request-log seed profile (default|large), same as --seed-logs # # State (master key, worker token, generated config, logs) lives in .lens-dev/ (gitignored). set -euo pipefail @@ -216,36 +217,55 @@ build_dashboard() { ) > "$log_dir/ui-build.log" 2>&1 || die "UI build failed; see $log_dir/ui-build.log" } +# Trace fixtures feed Lens (ClickHouse + spend rows); request logs feed the Logs page +# (Postgres only) with rows sized to stress the log detail drawer. seed_data() { ( proxy_env "" - "$py" -m scripts.seed_tracing_fixtures --profile "$seed_profile" ${seed_options[@]+"${seed_options[@]}"} + export LENS_DEV_UI_URL="http://localhost:$ui_port" + if [ -n "$seed_profile" ]; then + "$py" -m scripts.seed_tracing_fixtures --profile "$seed_profile" ${seed_options[@]+"${seed_options[@]}"} + fi + if [ -n "$seed_logs_profile" ]; then + "$py" -m scripts.seed_request_logs --profile "$seed_logs_profile" + fi ) } +# --seed and --seed-logs take an optional profile; a bare flag means default. +seed_profile_arg() { + if [ "${1:-}" = default ] || [ "${1:-}" = large ]; then echo "$1"; else echo default; fi +} + parse_args() { seed_profile="${LENS_DEV_SEED:-}" + seed_logs_profile="${LENS_DEV_SEED_LOGS:-}" seed_only=0 seed_options=() while [ "$#" -gt 0 ]; do case "$1" in --seed) - seed_profile=default - if [ "${2:-}" = default ] || [ "${2:-}" = large ]; then seed_profile="$2"; shift; fi + seed_profile="$(seed_profile_arg "${2:-}")" + [ "$seed_profile" = "${2:-}" ] && shift + ;; + --seed-logs) + seed_logs_profile="$(seed_profile_arg "${2:-}")" + [ "$seed_logs_profile" = "${2:-}" ] && shift ;; --copies) [ "$#" -ge 2 ] && [[ "$2" =~ ^[1-9][0-9]*$ ]] || die "--copies requires a positive integer" seed_options=(--copies "$2"); shift ;; --seed-only) seed_only=1 ;; --help) - echo "Usage: $0 [--seed [default|large]] [--copies N] [--seed-only]" + echo "Usage: $0 [--seed [default|large]] [--seed-logs [default|large]] [--copies N] [--seed-only]" exit 0 ;; *) die "unknown argument: $1 (use --help)" ;; esac shift done - if [ "$seed_only" = 1 ] && [ -z "$seed_profile" ]; then seed_profile=default; fi + if [ "$seed_only" = 1 ] && [ -z "$seed_profile" ] && [ -z "$seed_logs_profile" ]; then seed_profile=default; fi case "$seed_profile" in ""|default|large) ;; *) die "seed profile must be default or large" ;; esac + case "$seed_logs_profile" in ""|default|large) ;; *) die "seed-logs profile must be default or large" ;; esac [ "${#seed_options[@]}" = 0 ] || [ -n "$seed_profile" ] || die "--copies requires --seed" } @@ -277,11 +297,14 @@ main() { ensure_services "$py" scripts/prisma_generate_if_needed.py - if [ "${LENS_DEV_REBUILD_RUST:-0}" = "1" ] || ! "$py" -c "import litellm.rust_bridge._native" >/dev/null 2>&1; then - echo "lens-dev: building the Rust bridge (litellm.rust_bridge._native); the ClickHouse trace store uses it" - VIRTUAL_ENV="$repo_root/.venv" uvx --from maturin==1.15.0 maturin develop \ - --release --manifest-path litellm-rust/crates/python-bridge/Cargo.toml --features extension-module - fi + # cargo/maturin already fingerprint every crate's sources, so re-running this on each + # start is a no-op (a couple seconds) when nothing changed and only rebuilds the + # subset that did. An import check can't tell content-stale from content-fresh: a + # `.so` built from an older commit still imports fine, it just no longer matches + # what the current Python bindings (e.g. the trace store protocol) expect. + echo "lens-dev: checking the Rust bridge (litellm.rust_bridge._native) is current; the ClickHouse trace store uses it" + PYO3_PYTHON="$py" VIRTUAL_ENV="$repo_root/.venv" uvx --from maturin==1.15.0 maturin develop \ + --release --manifest-path litellm-rust/crates/python-bridge/Cargo.toml --features extension-module if [ ! -x ui/litellm-dashboard/node_modules/.bin/next ]; then (cd ui/litellm-dashboard && "$repo_root/scripts/with_dashboard_node.sh" npm ci) @@ -317,7 +340,7 @@ main() { wait_for_ui "$ui_pid" wait_for_proxy "$proxy_pid" ensure_worker_token - if [ -n "$seed_profile" ]; then seed_data; fi + if [ -n "$seed_profile" ] || [ -n "$seed_logs_profile" ]; then seed_data; fi LITELLM_RELEASE_TAG="$source_release_tag" \ LITELLM_MODE=PRODUCTION LITELLM_URL="$proxy_url" LENS_WORKER_TOKEN="$(cat "$token_file")" \ @@ -332,6 +355,7 @@ main() { Lens dev is up. Ctrl-C stops everything. Log in: http://localhost:$ui_port/ui/login/ (admin / $key_hint) Lens: http://localhost:$ui_port/ui/lens/ (hot-reloads) + Logs: http://localhost:$ui_port/ui/?page=logs API: $proxy_url Logs: $log_dir/proxy.log $log_dir/worker.log diff --git a/scripts/seed_request_logs.py b/scripts/seed_request_logs.py new file mode 100644 index 00000000000..1217a9fb7e4 --- /dev/null +++ b/scripts/seed_request_logs.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import random +import sys +from collections.abc import Iterator, Sequence +from datetime import datetime, timedelta, timezone +from itertools import chain +from typing import TYPE_CHECKING, Final, Literal + +import httpx +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from scripts.seed_tracing_fixtures import JSON_OBJECT, spend_fixtures + +if TYPE_CHECKING: + from prisma.types import LiteLLM_SpendLogsCreateWithoutRelationsInput + +REQUEST_ID_PREFIX: Final = "seed-logs-" +SESSION_ID_PREFIX: Final = "seed-logs-session-" +WINDOW_HOURS: Final = 23 +RNG_SEED: Final = 20261004 +LARGE_COPIES: Final = 3000 +PROFILES: Final = ("default", "large") +Profile = Literal["default", "large"] +JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + +WORDS: Final = ( + "trace", "span", "token", "request", "response", "latency", "router", "fallback", "cache", "budget", + "guardrail", "stream", "deployment", "proxy", "callback", "cursor", "schema", "payload", "retry", "quota", +) + + +class SeededLog(BaseModel): + """One synthetic spend-log row before it is shaped for Postgres.""" + + model_config = ConfigDict(frozen=True) + request_id: str + label: str + call_type: str + model: str + provider: str + status: Literal["success", "failure"] + session_id: str | None + offset_minutes: int + duration_ms: int + prompt_tokens: int + completion_tokens: int + spend: float + messages: JsonValue + response: JsonValue + proxy_server_request: JsonValue + error_information: dict[str, JsonValue] | None = None + + +def prose(rng: random.Random, chars: int) -> str: + words: Final[list[str]] = [] + length = 0 + while length < chars: + word: Final = rng.choice(WORDS) + words.append(word) + length += len(word) + 1 # rebind-ok: accumulates generated text length + return " ".join(words)[:chars] + + +def tool_definition(index: int) -> dict[str, JsonValue]: + return { + "type": "function", + "function": { + "name": f"seed_tool_{index}", + "description": f"Synthetic tool number {index} used only by the request-log seeder.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "What to look up"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 100}, + }, + "required": ["query"], + }, + }, + } + + +def tool_call(index: int, rng: random.Random) -> dict[str, JsonValue]: + return { + "id": f"call_seed_{index}", + "type": "function", + "function": {"name": f"seed_tool_{index}", "arguments": json.dumps({"query": prose(rng, 40), "limit": index})}, + } + + +def chat_turns(rng: random.Random, turns: int, chars_per_turn: int) -> list[JsonValue]: + def turn(index: int) -> Iterator[JsonValue]: + yield {"role": "user", "content": prose(rng, chars_per_turn)} + if index % 3 == 0: + yield {"role": "assistant", "content": None, "tool_calls": [tool_call(index % 7, rng)]} + yield {"role": "tool", "tool_call_id": f"call_seed_{index % 7}", "content": prose(rng, chars_per_turn * 4)} + else: + yield {"role": "assistant", "content": prose(rng, chars_per_turn)} + + return list(chain.from_iterable(turn(index) for index in range(turns))) + + +def chat_response(content: str, tool_calls: list[JsonValue] | None, prompt_tokens: int, completion_tokens: int) -> JsonValue: + message: dict[str, JsonValue] = {"role": "assistant", "content": content} + if tool_calls: + message["tool_calls"] = tool_calls + return { + "id": "chatcmpl-seed", + "object": "chat.completion", + "model": "gpt-5.5", + "choices": [{"index": 0, "finish_reason": "tool_calls" if tool_calls else "stop", "message": message}], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + } + + +def chat_log( + rng: random.Random, + label: str, + *, + messages: list[JsonValue], + response_chars: int, + tools: int = 0, + called_tools: int = 0, + offset_minutes: int, + session_id: str | None = None, +) -> SeededLog: + prompt_tokens: Final = len(json.dumps(messages)) // 4 + completion_tokens: Final = max(response_chars // 4, 1) + tool_calls: Final = [tool_call(index, rng) for index in range(called_tools)] or None + request: dict[str, JsonValue] = {"model": "gpt-5.5", "messages": messages, "stream": False} + if tools: + request["tools"] = [tool_definition(index) for index in range(tools)] + return SeededLog( + request_id=f"{REQUEST_ID_PREFIX}{label}", + label=label, + call_type="acompletion", + model="gpt-5.5", + provider="openai", + status="success", + session_id=session_id, + offset_minutes=offset_minutes, + duration_ms=1500 + completion_tokens // 10, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + spend=prompt_tokens * 0.000002 + completion_tokens * 0.000008, + messages=messages, + response=chat_response(prose(rng, response_chars), tool_calls, prompt_tokens, completion_tokens), + proxy_server_request=request, + ) + + +def anthropic_log(rng: random.Random, offset_minutes: int) -> SeededLog: + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": [{"type": "text", "text": prose(rng, 2000)}]}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": prose(rng, 500)}, + {"type": "tool_use", "id": "toolu_seed_1", "name": "seed_tool_1", "input": {"query": "spend"}}, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_seed_1", "content": prose(rng, 20_000)}]}, + ] + tools: Final[list[JsonValue]] = [ + {"name": f"seed_tool_{index}", "description": "Synthetic Anthropic tool", "input_schema": {"type": "object"}} + for index in range(3) + ] + response: Final[JsonValue] = { + "id": "msg_seed", + "type": "message", + "role": "assistant", + "model": "claude-opus-5-5", + "content": [ + {"type": "text", "text": prose(rng, 50_000)}, + {"type": "tool_use", "id": "toolu_seed_2", "name": "seed_tool_2", "input": {"query": "latency", "limit": 5}}, + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 6000, "output_tokens": 12_500}, + } + return SeededLog( + request_id=f"{REQUEST_ID_PREFIX}anthropic-tool-use", + label="anthropic-tool-use", + call_type="anthropic_messages", + model="claude-opus-5-5", + provider="anthropic", + status="success", + session_id=None, + offset_minutes=offset_minutes, + duration_ms=9000, + prompt_tokens=6000, + completion_tokens=12_500, + spend=6000 * 0.000015 + 12_500 * 0.000075, + messages=messages, + response=response, + proxy_server_request={"model": "claude-opus-5-5", "max_tokens": 16_000, "messages": messages, "tools": tools}, + ) + + +def failure_log(rng: random.Random, offset_minutes: int) -> SeededLog: + messages: Final[list[JsonValue]] = [{"role": "user", "content": prose(rng, 300_000)}] + return SeededLog( + request_id=f"{REQUEST_ID_PREFIX}context-window-failure", + label="context-window-failure", + call_type="acompletion", + model="gpt-5.5", + provider="openai", + status="failure", + session_id=None, + offset_minutes=offset_minutes, + duration_ms=800, + prompt_tokens=75_000, + completion_tokens=0, + spend=0.0, + messages=messages, + response={}, + proxy_server_request={"model": "gpt-5.5", "messages": messages}, + error_information={ + "error_code": "400", + "error_class": "ContextWindowExceededError", + "llm_provider": "openai", + "error_message": "This model's maximum context length is 128000 tokens. Your messages resulted in 75000 tokens plus 300000 characters of synthetic prose.", + "traceback": "Traceback (most recent call last):\n" + "\n".join(f" File seed_{index}.py, line {index}" for index in range(40)), + }, + ) + + +def seeded_logs(rng: random.Random) -> tuple[SeededLog, ...]: + """The size ladder: one axis per thing that can make the log drawer slow.""" + session: Final = f"{SESSION_ID_PREFIX}agent-run" + single: Final = [{"role": "user", "content": "Summarise the seeded request logs in one paragraph."}] + return ( + chat_log(rng, "baseline-small", messages=single, response_chars=400, offset_minutes=5), + chat_log(rng, "response-100kb", messages=single, response_chars=100_000, offset_minutes=20), + chat_log(rng, "response-1mb", messages=single, response_chars=1_000_000, offset_minutes=35), + chat_log(rng, "response-5mb", messages=single, response_chars=5_000_000, offset_minutes=50), + chat_log(rng, "turns-200", messages=chat_turns(rng, 200, 500), response_chars=2000, offset_minutes=70), + chat_log(rng, "turns-1000", messages=chat_turns(rng, 1000, 500), response_chars=2000, offset_minutes=90), + chat_log(rng, "system-prompt-200kb", messages=[{"role": "system", "content": prose(rng, 200_000)}, *single], response_chars=1500, offset_minutes=110), + chat_log(rng, "tools-50", messages=single, response_chars=800, tools=50, called_tools=6, offset_minutes=130), + anthropic_log(rng, offset_minutes=150), + failure_log(rng, offset_minutes=170), + *( + chat_log( + rng, + f"session-call-{index:02d}", + messages=chat_turns(rng, index + 1, 400), + response_chars=3000, + tools=4, + called_tools=index % 3, + offset_minutes=200 + index, + session_id=session, + ) + for index in range(30) + ), + ) + + +def spread_offsets(logs: tuple[SeededLog, ...]) -> tuple[SeededLog, ...]: + """Fit every row into the page's default 24h window, newest first.""" + last: Final = max(log.offset_minutes for log in logs) + scale: Final = min(1.0, WINDOW_HOURS * 60 / max(last, 1)) + return tuple(log.model_copy(update={"offset_minutes": int(log.offset_minutes * scale)}) for log in logs) + + +def metadata(log: SeededLog, template: dict[str, JsonValue]) -> dict[str, JsonValue]: + usage: Final[dict[str, JsonValue]] = { + "prompt_tokens": log.prompt_tokens, + "completion_tokens": log.completion_tokens, + "total_tokens": log.prompt_tokens + log.completion_tokens, + "prompt_tokens_details": {"cached_tokens": 0, "text_tokens": log.prompt_tokens}, + } + seeded: dict[str, JsonValue] = { + **template, + "status": log.status, + "model_group": log.model, + "deployment": f"{log.provider}/{log.model}", + "deployment_model_name": f"{log.provider}/{log.model}", + "user_api_key_team_alias": "seed-logs", + "usage_object": usage, + "additional_usage_values": {"cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, **usage}, + "cost_breakdown": { + "input_cost": log.prompt_tokens * 0.000002, + "output_cost": log.completion_tokens * 0.000008, + "total_cost": log.spend, + }, + "litellm_overhead_time_ms": 12.5, + "attempted_retries": 0, + "max_retries": 2, + "hidden_params": {"litellm_overhead_time_ms": 12.5, "response_cost": log.spend}, + "fixture_capture": None, + "seed_label": log.label, + } + if log.error_information is not None: + seeded["error_information"] = log.error_information + return seeded + + +def postgres_row(log: SeededLog, template: dict[str, JsonValue], now: datetime) -> LiteLLM_SpendLogsCreateWithoutRelationsInput: + from prisma import Json + from prisma.types import LiteLLM_SpendLogsCreateWithoutRelationsInput + + end: Final = now - timedelta(minutes=log.offset_minutes) + start: Final = end - timedelta(milliseconds=log.duration_ms) + return LiteLLM_SpendLogsCreateWithoutRelationsInput( + request_id=log.request_id, + litellm_call_id=log.request_id, + call_type=log.call_type, + api_key=str(template.get("user_api_key", "seed-logs-key")), + user="seed-logs-user", + team_id="seed-logs-team", + spend=log.spend, + model=log.model, + model_id=f"seed-logs-{log.model}", + model_group=log.model, + custom_llm_provider=log.provider, + api_base=f"https://api.{log.provider}.example", + prompt_tokens=log.prompt_tokens, + completion_tokens=log.completion_tokens, + total_tokens=log.prompt_tokens + log.completion_tokens, + startTime=start, + endTime=end, + completionStartTime=start + timedelta(milliseconds=min(400, log.duration_ms // 2)), + request_duration_ms=log.duration_ms, + session_id=log.session_id, + status=log.status, + cache_hit="False", + request_tags=Json(["seed-logs", log.label.split("-")[0]]), + metadata=Json(metadata(log, template)), + messages=Json(log.messages), + response=Json(log.response), + proxy_server_request=Json(log.proxy_server_request), + ) + + +COPY_SQL: Final = """INSERT INTO "LiteLLM_SpendLogs" +SELECT (jsonb_populate_record(s, jsonb_build_object( + 'request_id', s.request_id || '-copy-' || c.n, + 'litellm_call_id', s.request_id || '-copy-' || c.n, + 'session_id', NULL, + 'startTime', s."startTime" - make_interval(secs => c.n * $3::bigint / 1000.0), + 'endTime', s."endTime" - make_interval(secs => c.n * $3::bigint / 1000.0), + 'completionStartTime', s."completionStartTime" - make_interval(secs => c.n * $3::bigint / 1000.0) +))).* +FROM "LiteLLM_SpendLogs" AS s CROSS JOIN generate_series(1, $2::int) AS c(n) +WHERE s.request_id = $1""" + + +class SeedOptions(BaseModel): + model_config = ConfigDict(frozen=True) + profile: Profile + timeout_seconds: float = 120 + + +def seed_arguments(argv: Sequence[str] | None = None) -> SeedOptions: + parser: Final = argparse.ArgumentParser(description="Insert synthetic request logs of controlled sizes into a local proxy DB") + parser.add_argument("--profile", choices=PROFILES, default="default") + parser.add_argument("--timeout-seconds", type=float, default=os.environ.get("LENS_DEV_SEED_TIMEOUT_SECONDS", "120")) + arguments: Final = SeedOptions.model_validate(vars(parser.parse_args(argv))) + if not math.isfinite(arguments.timeout_seconds) or arguments.timeout_seconds <= 0: + parser.error("--timeout-seconds must be finite and positive") + return arguments + + +def metadata_template() -> dict[str, JsonValue]: + """A real captured row's metadata, so the drawer sees the keys the gateway writes.""" + _, rows = spend_fixtures()[0] + return JSON_OBJECT.validate_json(rows[0]["metadata"]) + + +async def verify(client: httpx.AsyncClient, logs: tuple[SeededLog, ...], ui_base: str) -> None: + async def fetch(log: SeededLog) -> dict[str, JsonValue]: + detail: Final = await client.get(f"/spend/logs/ui/{log.request_id}") + detail.raise_for_status() + payload: Final = JSON.validate_json(detail.content) + return { + "label": log.label, + "bytes": len(detail.content), + "found": isinstance(payload, dict) and bool(payload), + "url": f"{ui_base}/ui/?page=logs&log_id={log.request_id}" + + (f"&session_id={log.session_id}" if log.session_id else ""), + } + + results: Final = tuple(await asyncio.gather(*(fetch(log) for log in logs))) + sys.stdout.write(json.dumps(list(results), indent=2) + "\n") + if not all(result["found"] for result in results): + raise RuntimeError("Seeded request logs did not round-trip through /spend/logs/ui/{request_id}") + + +async def seed(profile: Profile = "default", timeout_seconds: float = 120) -> int: + from prisma import Prisma + + logs: Final = spread_offsets(seeded_logs(random.Random(RNG_SEED))) + template: Final = metadata_template() + now: Final = datetime.now(timezone.utc) + proxy_url: Final = os.environ.get("PROXY_BASE_URL", "http://127.0.0.1:4000") + ui_base: Final = os.environ.get("LENS_DEV_UI_URL", proxy_url) + async with ( + httpx.AsyncClient( + base_url=proxy_url, + headers={"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}"}, + timeout=timeout_seconds, + ) as client, + Prisma(http={"timeout": httpx.Timeout(600)}) as database, + ): + await database.litellm_spendlogs.delete_many(where={"request_id": {"startswith": REQUEST_ID_PREFIX}}) + await database.litellm_spendlogs.create_many(data=[postgres_row(log, template, now) for log in logs]) + if profile == "large": + step_ms: Final = WINDOW_HOURS * 60 * 60 * 1000 // LARGE_COPIES + await database.execute_raw(COPY_SQL, f"{REQUEST_ID_PREFIX}baseline-small", LARGE_COPIES, step_ms) + await verify(client, tuple(log for log in logs if not log.session_id or log.label.endswith("-00")), ui_base) + total: Final = len(logs) + (LARGE_COPIES if profile == "large" else 0) + sys.stdout.write(f"Request log seed complete: profile={profile}, rows={total}, prefix={REQUEST_ID_PREFIX}\n") + return 0 + + +if __name__ == "__main__": + arguments: Final = seed_arguments() + raise SystemExit(asyncio.run(seed(arguments.profile, arguments.timeout_seconds))) diff --git a/scripts/seed_tracing_fixtures.py b/scripts/seed_tracing_fixtures.py index 3febe49c451..be11f7cc90b 100644 --- a/scripts/seed_tracing_fixtures.py +++ b/scripts/seed_tracing_fixtures.py @@ -43,6 +43,9 @@ TRACE: Final = TypeAdapter(Trace) NANOSECOND_FIELDS: Final = frozenset({"startTimeUnixNano", "endTimeUnixNano", "timeUnixNano"}) TRACE_ID_FIELDS: Final = frozenset({"traceId", "trace_id", "session_id"}) SPAN_ID_FIELDS: Final = frozenset({"spanId", "parentSpanId", "span_id"}) +COPY_WINDOW_MS: Final = 24 * 60 * 60 * 1000 +LONG_SESSION_SOURCE: Final = "openai_agents_swarm" +LONG_SESSION_REPEATS: Final = (50, 400, 4000) class TenantIdentity(BaseModel): @@ -245,7 +248,6 @@ class SeedOptions(BaseModel): model_config = ConfigDict(frozen=True) profile: Literal["default", "large"] copies: int | None - batch_copies: int = 4 timeout_seconds: float = 120 @@ -258,33 +260,23 @@ def seed_arguments(argv: Sequence[str] | None = None) -> SeedOptions: default=os.environ.get("LENS_DEV_SEED_COPIES"), help="Override fixture copies (default: 1, large: 2000; env: LENS_DEV_SEED_COPIES)", ) - parser.add_argument( - "--batch-copies", - type=int, - default=os.environ.get("LENS_DEV_SEED_BATCH_COPIES", "4"), - help="Copies per bulk insert (default: 4; env: LENS_DEV_SEED_BATCH_COPIES)", - ) parser.add_argument("--timeout-seconds", type=float, default=os.environ.get("LENS_DEV_SEED_TIMEOUT_SECONDS", "120")) arguments: Final = SeedOptions.model_validate(vars(parser.parse_args(argv))) if arguments.copies is not None and arguments.copies < 1: parser.error("--copies must be positive") - if arguments.batch_copies < 1: - parser.error("--batch-copies must be positive") if not math.isfinite(arguments.timeout_seconds) or arguments.timeout_seconds <= 0: parser.error("--timeout-seconds must be finite and positive") return arguments -async def seed_batch( +async def seed_copy( client: httpx.AsyncClient, storage: ClickHouseStorage, database: Prisma, replays: tuple[FixtureReplay, ...], fixtures: tuple[tuple[str, tuple[SpendLogRecord, ...]], ...], pattern: re.Pattern[str], - tenant: TenantIdentity | None, - verify: bool, -) -> TenantIdentity: +) -> tuple[tuple[str, tuple[SpendLogRecord, ...]], ...]: by_name: Final = MappingProxyType(dict(fixtures)) paired: Final = tuple( ( @@ -294,36 +286,36 @@ async def seed_batch( for replay in replays if replay.name in by_name ) - rebased_spends: Final = tuple(chain.from_iterable(rows for _, rows in paired)) - resolved_tenant: Final = await ingest_replays( - client, storage, replays, fixture_capture(*next((name, rows[0]) for name, rows in paired)).trace_id, tenant - ) - stamped_spends: Final[tuple[SpendLogRecord, ...]] = tuple( - {**row, "team_id": resolved_tenant.team_id, "api_key": resolved_tenant.api_key, "user": resolved_tenant.user} - for row in rebased_spends + tenant: Final = await ingest_replays( + client, storage, replays, fixture_capture(*next((name, rows[0]) for name, rows in paired)).trace_id ) + stamped: Final = tuple((name, tuple(stamp(row, tenant) for row in rows)) for name, rows in paired) + stamped_spends: Final = tuple(chain.from_iterable(rows for _, rows in stamped)) await storage.insert_rows("spend_logs", stamped_spends) await database.litellm_spendlogs.create_many(data=[postgres_row(row) for row in stamped_spends]) - if verify: - verified: Final = tuple(await asyncio.gather(*(verify_capture(client, name, rows) for name, rows in paired))) - sys.stdout.write(json.dumps({"spend_rows": len(stamped_spends), "captures": verified}, indent=2) + "\n") - if not all(capture["verified"] for capture in verified): - raise RuntimeError("Seed spend verification failed") - return resolved_tenant + return stamped + + +def stamp(row: SpendLogRecord, tenant: TenantIdentity) -> SpendLogRecord: + return {**row, "team_id": tenant.team_id, "api_key": tenant.api_key, "user": tenant.user} + + +async def verify( + client: httpx.AsyncClient, captures: tuple[tuple[str, tuple[SpendLogRecord, ...]], ...], trace_salt: str +) -> None: + verified: Final = tuple( + await asyncio.gather(*(verify_capture(client, name, rows, trace_salt) for name, rows in captures)) + ) + sys.stdout.write( + json.dumps({"spend_rows": sum(len(rows) for _, rows in captures), "captures": verified}, indent=2) + "\n" + ) + if not all(capture["verified"] for capture in verified): + raise RuntimeError("Seed spend verification failed") async def ingest_replays( - client: httpx.AsyncClient, - storage: ClickHouseStorage, - replays: tuple[FixtureReplay, ...], - trace_id: str, - tenant: TenantIdentity | None, + client: httpx.AsyncClient, storage: ClickHouseStorage, replays: tuple[FixtureReplay, ...], trace_id: str ) -> TenantIdentity: - if tenant is not None: - await storage.insert_rows( - "otel_traces", bulk_span_rows(replays, Tenant(tenant.team_id, tenant.api_key, user_id=tenant.user)) - ) - return tenant for replay in replays: ( await client.post( @@ -347,24 +339,161 @@ def bulk_span_rows(replays: tuple[FixtureReplay, ...], tenant: Tenant) -> tuple[ ) -def replay_batches( - count: int, now_ms: int, namespace: str, pattern: re.Pattern[str], batch_copies: int = 4 -) -> Iterator[tuple[int, tuple[FixtureReplay, ...]]]: - for start, stop in ((start, min(start + batch_copies, count)) for start in range(1, count, batch_copies)): - yield ( - stop, - tuple( - chain.from_iterable( - fixture_replays(TRACE_FIXTURES, now_ms - index * 1000, f"{namespace}-{index}", pattern) - for index in range(start, stop) - ) - ), +@dataclass(frozen=True, slots=True) +class Copies: + """Server-side copies of seeded traces and their spend. + + Each copy `n` hashes trace ids with `session` (or `n` when empty), keeps root span ids, hashes the + other span ids with `n`, rewrites seeded call ids from `source` to `{target}{n}-` and moves `n * step_ms` + earlier. A `session` folds every copy into one trace under a single root that spans all of them. + """ + + trace_ids: tuple[str, ...] + request_ids: tuple[str, ...] + numbers: range + step_ms: int + source: str + target: str + session: str = "" + + +def copied_trace_id(trace_id: str, salt: str) -> str: + return hashlib.sha256(f"{trace_id}:{salt}".encode()).hexdigest()[:32] + + +def clickhouse_call_id(column: str) -> str: + target: Final = "concat({target:String}, toString(c.n), '-')" + return ( + f"if(startsWith({column}, 'resp_'), concat('resp_', base64Encode(replaceAll(" + f"tryBase64Decode(substring({column}, 6)), {{source:String}}, {target}))), " + f"replaceAll({column}, {{source:String}}, {target}))" + ) + + +def clickhouse_hash(column: str, salt: str, length: int) -> str: + return f"if({column} = '', '', substring(lower(hex(SHA256(concat({column}, ':', {salt})))), 1, {length}))" + + +def clickhouse_copy_sql(database: str) -> tuple[str, str]: + trace_salt: Final = "if({session:String} = '', toString(c.n), {session:String})" + roots: Final = ( + f"(SELECT SpanId FROM {database}.otel_traces " + "WHERE TraceId IN {trace_ids:Array(String)} AND ParentSpanId = '')" + ) + folded_root: Final = "{session:String} != '' AND t.ParentSpanId = ''" + shift: Final = f"toIntervalMillisecond(if({folded_root}, {{last:UInt64}}, c.n) * {{step_ms:UInt64}})" + numbers: Final = "CROSS JOIN (SELECT number AS n FROM numbers({first:UInt64}, {count:UInt64})) AS c" + spans: Final = f"""INSERT INTO {database}.otel_traces +SELECT t.* REPLACE ( + t.Timestamp - {shift} AS Timestamp, + {clickhouse_hash("t.TraceId", trace_salt, 32)} AS TraceId, + if(t.ParentSpanId = '', t.SpanId, {clickhouse_hash("t.SpanId", "toString(c.n)", 16)}) AS SpanId, + if(t.ParentSpanId IN {roots}, t.ParentSpanId, {clickhouse_hash("t.ParentSpanId", "toString(c.n)", 16)}) + AS ParentSpanId, + t.Duration + if({folded_root}, {{last:UInt64}} * {{step_ms:UInt64}} * 1000000, 0) AS Duration, + arrayMap(at -> at - {shift}, t.`Events.Timestamp`) AS `Events.Timestamp`, + mapApply((name, value) -> (name, {clickhouse_call_id("value")}), t.SpanAttributes) AS SpanAttributes, + {clickhouse_call_id("t.LiteLLMRequestId")} AS LiteLLMRequestId, + arrayMap(key -> concat(extract(key, '^[^:]*:'), {clickhouse_call_id("replaceRegexpOne(key, '^[^:]*:', '')")}), + t.CallKeys) AS CallKeys +) +FROM {database}.otel_traces AS t {numbers} +WHERE t.TraceId IN {{trace_ids:Array(String)}} + AND ({{session:String}} = '' OR t.ParentSpanId != '' OR c.n = {{first:UInt64}})""" + spend: Final = f"""INSERT INTO {database}.spend_logs +SELECT s.* REPLACE ( + {clickhouse_call_id("s.request_id")} AS request_id, + {clickhouse_call_id("s.response_id")} AS response_id, + {clickhouse_call_id("s.litellm_call_id")} AS litellm_call_id, + {clickhouse_hash("s.trace_id", trace_salt, 32)} AS trace_id, + {clickhouse_hash("s.session_id", trace_salt, 32)} AS session_id, + if(s.span_id IN {roots}, s.span_id, {clickhouse_hash("s.span_id", "toString(c.n)", 16)}) AS span_id, + s.start_time - toIntervalMillisecond(c.n * {{step_ms:UInt64}}) AS start_time, + s.end_time - toIntervalMillisecond(c.n * {{step_ms:UInt64}}) AS end_time, + s.completion_start_time - toIntervalMillisecond(c.n * {{step_ms:UInt64}}) AS completion_start_time +) +FROM {database}.spend_logs AS s {numbers} +WHERE s.request_id IN {{request_ids:Array(String)}}""" + return spans, spend + + +def clickhouse_array(values: tuple[str, ...]) -> str: + return "[" + ",".join("'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" for value in values) + "]" + + +async def copy_clickhouse(client: httpx.AsyncClient, database: str, copies: Copies) -> None: + parameters: Final = { + "param_trace_ids": clickhouse_array(copies.trace_ids), + "param_request_ids": clickhouse_array(copies.request_ids), + "param_first": str(copies.numbers.start), + "param_count": str(len(copies.numbers)), + "param_last": str(copies.numbers.stop - 1), + "param_step_ms": str(copies.step_ms), + "param_source": copies.source, + "param_target": copies.target, + "param_session": copies.session, + } + for sql in clickhouse_copy_sql(database): + (await client.post("/", params=parameters, content=sql)).raise_for_status() + + +POSTGRES_COPY_TARGET: Final = "($2 || c.n || '-')" +POSTGRES_COPY_SHIFT: Final = "make_interval(secs => c.n * $6::bigint / 1000.0)" +POSTGRES_COPY_SQL: Final = f"""INSERT INTO "LiteLLM_SpendLogs" +SELECT (jsonb_populate_record(s, jsonb_build_object( + 'request_id', CASE WHEN left(s.request_id, 5) = 'resp_' + THEN 'resp_' || translate(encode(convert_to(replace(convert_from(decode(substr(s.request_id, 6), 'base64'), + 'UTF8'), $1, {POSTGRES_COPY_TARGET}), 'UTF8'), 'base64'), E'\\n', '') + ELSE replace(s.request_id, $1, {POSTGRES_COPY_TARGET}) END, + 'session_id', CASE WHEN coalesce(s.session_id, '') = '' THEN s.session_id + ELSE substr(encode(sha256(convert_to( + s.session_id || ':' || CASE WHEN $3 = '' THEN c.n::text ELSE $3 END, 'UTF8')), 'hex'), 1, 32) END, + 'startTime', s."startTime" - {POSTGRES_COPY_SHIFT}, + 'endTime', s."endTime" - {POSTGRES_COPY_SHIFT}, + 'completionStartTime', s."completionStartTime" - {POSTGRES_COPY_SHIFT} +))).* +FROM "LiteLLM_SpendLogs" AS s CROSS JOIN generate_series($4::int, $5::int) AS c(n) +WHERE s.request_id = ANY(string_to_array($7, E'\\n'))""" + + +async def copy_postgres(database: Prisma, copies: Copies) -> None: + await database.execute_raw( + POSTGRES_COPY_SQL, + copies.source, + copies.target, + copies.session, + copies.numbers.start, + copies.numbers.stop - 1, + copies.step_ms, + "\n".join(copies.request_ids), + ) + + +def long_sessions( + replays: tuple[FixtureReplay, ...], + captures: tuple[tuple[str, tuple[SpendLogRecord, ...]], ...], + source: str, + target: str, + repeats: tuple[int, ...] = LONG_SESSION_REPEATS, +) -> tuple[Copies, ...]: + replay: Final = next(replay for replay in replays if replay.name == LONG_SESSION_SOURCE) + rows: Final = dict(captures)[LONG_SESSION_SOURCE] + span_ns: Final = tuple(timestamps(replay.export)) + return tuple( + Copies( + trace_ids=(fixture_capture(LONG_SESSION_SOURCE, rows[0]).trace_id,), + request_ids=tuple(row["request_id"] for row in rows), + numbers=range(count), + step_ms=(max(span_ns) - min(span_ns)) // 1_000_000 + 1000, + source=source, + target=f"{target}s{count}x", + session=f"session{count}", ) + for count in repeats + ) -async def seed( - profile: str = "default", copies: int | None = None, batch_copies: int = 4, timeout_seconds: float = 120 -) -> int: +async def seed(profile: str = "default", copies: int | None = None, timeout_seconds: float = 120) -> int: from prisma import Prisma fixtures: Final = spend_fixtures() @@ -372,29 +501,43 @@ async def seed( count: Final = copies if copies is not None else (2000 if profile == "large" else 1) namespace: Final = uuid4().hex now_ms: Final = time.time_ns() // 1_000_000 - storage: Final = ClickHouseStorage(trace_storage_config({})) + config: Final = trace_storage_config({}) + storage: Final = ClickHouseStorage(config) + replays: Final = fixture_replays(TRACE_FIXTURES, now_ms, namespace + "-0", pattern) + source: Final = f"seed-{namespace}-0-" + target: Final = f"seed-{namespace}-" async with ( httpx.AsyncClient( base_url=os.environ.get("PROXY_BASE_URL", "http://127.0.0.1:4002"), headers={"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}"}, timeout=timeout_seconds, ) as client, - Prisma() as database, + httpx.AsyncClient(base_url=config.url, params={"database": config.database}, timeout=600) as clickhouse, + Prisma(http={"timeout": httpx.Timeout(600)}) as database, ): - tenant: Final = await seed_batch( - client, - storage, - database, - fixture_replays(TRACE_FIXTURES, now_ms, namespace + "-0", pattern), - fixtures, - pattern, - None, - True, + captures: Final = await seed_copy(client, storage, database, replays, fixtures, pattern) + await verify(client, captures, "") + repeated: Final = Copies( + trace_ids=tuple( + sorted(frozenset(str(span["TraceId"]) for span in bulk_span_rows(replays, Tenant("", "")))) + ), + request_ids=tuple(row["request_id"] for _, rows in captures for row in rows), + numbers=range(1, count), + step_ms=COPY_WINDOW_MS // count, + source=source, + target=target, ) - for stop, replays in replay_batches(count, now_ms, namespace, pattern, batch_copies): - await seed_batch(client, storage, database, replays, fixtures, pattern, tenant, stop == count) - sys.stdout.write(f"Seeded {stop}/{count} fixture copies\n") - sys.stdout.flush() + sessions: Final = long_sessions(replays, captures, source, target) if profile == "large" else () + for plan in (repeated, *sessions) if count > 1 else sessions: + await copy_clickhouse(clickhouse, config.database, plan) + await copy_postgres(database, plan) + if count > 1: + await verify(client, captures, str(count - 1)) + for plan in sessions: + sys.stdout.write( + f"Long session: {len(plan.numbers)} repeats, " + f"trace_id={copied_trace_id(plan.trace_ids[0], plan.session)}\n" + ) sys.stdout.write(f"Seed complete: profile={profile}, copies={count}, namespace={namespace}\n") return 0 @@ -410,17 +553,18 @@ def fixture_capture(name: str, row: SpendLogRecord) -> FixtureCapture: async def verify_capture( - client: httpx.AsyncClient, name: str, rows: tuple[SpendLogRecord, ...] + client: httpx.AsyncClient, name: str, rows: tuple[SpendLogRecord, ...], trace_salt: str = "" ) -> Mapping[str, JsonValue]: capture: Final = fixture_capture(name, rows[0]) - detail: Final = await client.get(f"/v1/traces/{capture.trace_id}") + trace_id: Final = copied_trace_id(capture.trace_id, trace_salt) if trace_salt else capture.trace_id + detail: Final = await client.get(f"/v1/traces/{trace_id}") detail.raise_for_status() trace: Final = TRACE.validate_json(detail.content) expected: Final = sum(row["spend"] or 0 for row in rows) actual: Final = trace["summary"]["spend"] return { "fixture": name, - "trace_id": capture.trace_id, + "trace_id": trace_id, "spend_rows": len(rows), "recorded_spend": expected, "trace_spend": actual, @@ -432,6 +576,4 @@ async def verify_capture( if __name__ == "__main__": arguments: Final = seed_arguments() - raise SystemExit( - asyncio.run(seed(arguments.profile, arguments.copies, arguments.batch_copies, arguments.timeout_seconds)) - ) + raise SystemExit(asyncio.run(seed(arguments.profile, arguments.copies, arguments.timeout_seconds))) diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index c1847a52760..b8af0f0c374 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -4,13 +4,15 @@ import json import math import re import time -from collections.abc import Iterator +from collections.abc import Generator, Iterator +from contextlib import closing from dataclasses import dataclass from itertools import chain from types import MappingProxyType from typing import Final from urllib.parse import parse_qs, urlsplit +import httpx import pytest from fastapi import FastAPI from fastapi.testclient import TestClient @@ -19,16 +21,21 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter from litellm.constants import OTLP_MAX_ATTRIBUTE_VALUE_BYTES from litellm.rust_bridge._native import NativeTraceConfig, NativeTraceStorage from litellm.rust_bridge.trace.generated.models import ActivityAvailability, LensAccessParams, TraceQueryHelp -from litellm.rust_bridge.trace.generated.types import TraceScope +from litellm.rust_bridge.trace.generated.types import Trace, TraceScope from litellm.rust_bridge.trace.storage import ClickHouseStorage, TraceStorageConfig, span_rows from litellm.tracing import Tenant, TraceReceiver, TracingPayloadTooLargeError from litellm.tracing.types import SpendLogRecord from scripts.seed_tracing_fixtures import ( TRACE, TRACE_FIXTURES, + Copies, FixtureReplay, + bulk_span_rows, + copied_trace_id, + copy_clickhouse, fixture_capture, fixture_replays, + long_sessions, rebase_spend, response_pattern, spend_fixtures, @@ -503,7 +510,7 @@ def seeded_trace_api(clickhouse_url: str) -> Iterator[SeededTraceAPI]: def _fixture_trace_api( clickhouse_url: str, replays: tuple[FixtureReplay, ...], stamped: tuple[SpendLogRecord, ...] -) -> Iterator[SeededTraceAPI]: +) -> Generator[SeededTraceAPI]: from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.tracing_endpoints import provide_receiver, provide_trace_query_secret, router @@ -595,24 +602,34 @@ def test_query_correlation_requires_key_or_user_ownership_within_a_team(seeded_t assert all(row["request_id"] != unrelated["request_id"] for row in matches) -@pytest.fixture(scope="module") -def captured_trace_api() -> Iterator[SeededTraceAPI]: +def _captured_replays( + namespace: str, +) -> tuple[tuple[FixtureReplay, ...], tuple[tuple[str, tuple[SpendLogRecord, ...]], ...]]: captures: Final = spend_fixtures() - originals: Final = tuple(chain.from_iterable(rows for _, rows in captures)) - pattern: Final = response_pattern(originals) - replays: Final = fixture_replays(TRACE_FIXTURES, time.time_ns() // 1_000_000, "captured-api", pattern) + pattern: Final = response_pattern(tuple(chain.from_iterable(rows for _, rows in captures))) + replays: Final = fixture_replays(TRACE_FIXTURES, time.time_ns() // 1_000_000, namespace, pattern) by_name: Final = MappingProxyType(dict(captures)) - paired: Final = tuple( - rebase_spend(by_name[replay.name], replay.offset_ms, replay.namespace, pattern) + return replays, tuple( + ( + replay.name, + tuple( + _stamp(row) for row in rebase_spend(by_name[replay.name], replay.offset_ms, replay.namespace, pattern) + ), + ) for replay in replays if replay.name in by_name ) - stamped: Final[tuple[SpendLogRecord, ...]] = tuple( - {**row, "team_id": "team-a", "api_key": "fixture-key", "user": "fixture-user"} - for row in chain.from_iterable(paired) - ) + + +def _stamp(row: SpendLogRecord) -> SpendLogRecord: + return {**row, "team_id": "team-a", "api_key": "fixture-key", "user": "fixture-user"} + + +@pytest.fixture(scope="module") +def captured_trace_api() -> Iterator[SeededTraceAPI]: + replays, paired = _captured_replays("captured-api") with clickhouse_service() as url: - yield from _fixture_trace_api(url, replays, stamped) + yield from _fixture_trace_api(url, replays, tuple(chain.from_iterable(rows for _, rows in paired))) @pytest.mark.parametrize("name", tuple(name for name, _ in spend_fixtures())) @@ -644,3 +661,64 @@ def test_captured_sdk_cost_survives_seeding_and_is_queryable(name: str, captured assert math.isclose(sum(row.spend for row in records), sum(row["spend"] or 0 for row in rows)) assert sum(row.prompt_tokens for row in records) == sum(row["prompt_tokens"] for row in rows) assert sum(row.completion_tokens for row in records) == sum(row["completion_tokens"] for row in rows) + + +def test_server_side_copies_keep_every_capture_linked_to_its_spend() -> None: + replays, paired = _captured_replays("copied-api") + copies: Final = Copies( + trace_ids=tuple(sorted(frozenset(str(span["TraceId"]) for span in bulk_span_rows(replays, Tenant("", ""))))), + request_ids=tuple(row["request_id"] for _, rows in paired for row in rows), + numbers=range(1, 3), + step_ms=60_000, + source="seed-copied-api-", + target="seed-copied-api-c", + ) + (session,) = long_sessions(replays, paired, "seed-copied-api-", "seed-copied-api-c", (3,)) + session_spend: Final = sum(row["spend"] or 0 for row in dict(paired)["openai_agents_swarm"]) + session_spans: Final = len( + span_rows((TRACE_FIXTURES / "openai_agents_swarm.json").read_bytes(), "application/json") + ) + with ( + clickhouse_service() as url, + closing(_fixture_trace_api(url, replays, tuple(chain.from_iterable(rows for _, rows in paired)))) as seeded, + ): + api: Final = next(seeded) + assert api.client.portal is not None + for plan in (copies, session): + api.client.portal.call(_copy_clickhouse, url, plan) + for name, rows in paired: + _assert_capture(api, name, rows, fixture_capture(name, rows[0]).trace_id) + _assert_capture(api, name, rows, copied_trace_id(fixture_capture(name, rows[0]).trace_id, "2")) + trace: Final = _trace(api, copied_trace_id(session.trace_ids[0], session.session)) + assert trace["summary"]["span_count"] == 1 + 3 * (session_spans - 1) + (root,) = (span for span in trace["spans"] if not span["parent_span_id"]) + assert {span["parent_span_id"] for span in trace["spans"] if span["parent_span_id"]} <= { + span["span_id"] for span in trace["spans"] + } + assert root["start_offset_ms"] == min(span["start_offset_ms"] for span in trace["spans"]) + assert root["start_offset_ms"] + root["duration_ms"] >= max( + span["start_offset_ms"] + span["duration_ms"] for span in trace["spans"] + ) + assert trace["summary"]["spend"] == pytest.approx(3 * session_spend) + + +def _trace(api: SeededTraceAPI, trace_id: str) -> Trace: + response: Final = api.client.get(f"/v1/traces/{trace_id}") + assert response.status_code == 200, response.text + return TRACE.validate_json(response.content) + + +def _assert_capture(api: SeededTraceAPI, name: str, rows: tuple[SpendLogRecord, ...], trace_id: str) -> None: + capture: Final = fixture_capture(name, rows[0]) + summary: Final = _trace(api, trace_id)["summary"] + assert summary["span_count"] == len(span_rows((TRACE_FIXTURES / f"{name}.json").read_bytes(), "application/json")) + assert summary["spend"] == ( + pytest.approx(sum(row["spend"] or 0 for row in rows)) + if capture.spend_linked and capture.spend_complete + else None + ) + + +async def _copy_clickhouse(url: str, copies: Copies) -> None: + async with httpx.AsyncClient(base_url=url, params={"database": "trace_test"}) as client: + await copy_clickhouse(client, "trace_test", copies) diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index a0d01878277..54dbaac1e42 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -18,7 +18,7 @@ from litellm.proxy.lens.endpoints import ( watching, worker_supports_model, ) -from litellm.proxy.lens.models import Lens, LensSettings, RunRequest, Scope +from litellm.proxy.lens.models import ActivitySelection, Lens, LensSettings, RunRequest, Scope @pytest.fixture @@ -281,6 +281,25 @@ def test_model_errors_reach_worker_with_status_and_redacted_provider_message(pro assert error.headers == {"retry-after": "60"} +@pytest.mark.asyncio +async def test_preview_samples_a_selection_without_investigation_settings() -> None: + from litellm.proxy.lens.endpoints import Preview, preview_sample + + class SelectionStorage: + async def lens_sample(self, parameters): + assert (parameters.source, parameters.agent_name, parameters.selected_team) == ("requests", "billing", "t1") + assert parameters.preview == 1 and parameters.offset == 3 + return [] + + body: Final = Preview.model_validate( + {"selection": {"source": "requests", "agent_name": "billing", "team_id": "t1"}, "offset": 3} + ) + sample: Final = await preview_sample( + body, UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), SelectionStorage() + ) + assert sample.eligible == 0 and not sample.executions + + @pytest.mark.asyncio async def test_preview_reports_calendar_overflow_as_a_validation_error() -> None: from datetime import datetime, timezone @@ -288,7 +307,7 @@ async def test_preview_reports_calendar_overflow_as_a_validation_error() -> None from litellm.proxy.lens.endpoints import Preview, preview_sample body: Final = Preview( - settings=LensSettings(name="Calendar regression", model="analysis", context="Read recorded activity"), + selection=ActivitySelection(), as_of=datetime.min.replace(tzinfo=timezone.utc), ) with pytest.raises(HTTPException) as error: @@ -323,7 +342,9 @@ async def test_unknown_gateway_release_refuses_registration_and_claims(monkeypat monkeypatch.setenv("LITELLM_RELEASE_TAG", "") monkeypatch.setenv("LENS_WORKER_IMAGE", "registry.example/lens-worker:old") with pytest.raises(HTTPException) as registration_error: - await register_worker(WorkerName(analysis_key_id="a" * 64), UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)) + await register_worker( + WorkerName(analysis_key_id="a" * 64), UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + ) assert registration_error.value.status_code == 503 assert "LITELLM_RELEASE_TAG" in registration_error.value.detail with pytest.raises(HTTPException) as claim_error: diff --git a/tests/unit/proxy/test_tracing_endpoints.py b/tests/unit/proxy/test_tracing_endpoints.py index cff67d1f57d..913b1aec618 100644 --- a/tests/unit/proxy/test_tracing_endpoints.py +++ b/tests/unit/proxy/test_tracing_endpoints.py @@ -12,6 +12,7 @@ import pytest from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient +from litellm.constants import TRACE_READ_RETRY_AFTER_SECONDS from litellm.proxy import tracing_endpoints from litellm.proxy._types import LitellmUserRoles, ProxyLifespanState, UserAPIKeyAuth from litellm.proxy.auth.authorization import OwnedRows, ReadScope @@ -19,6 +20,7 @@ from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.tracing_runtime import manage_tracing, provide_storage from litellm.rust_bridge import loader +from litellm.rust_bridge.trace.errors import TraceChanged from litellm.rust_bridge.trace.generated.models import TraceQueryHelp from litellm.rust_bridge.trace.generated.types import AllQueryScope, TraceScope from litellm.rust_bridge.trace.queries import TraceSQLResponse @@ -297,19 +299,45 @@ def test_trace_detail_passes_scoped_reference(client, receiver, suffix, cursor, ), ) @pytest.mark.parametrize( - "error,status,message", + "error,status,code,message", ( - (RuntimeError("private database details"), 503, "Traces are temporarily unavailable. Please try again."), - (OverflowError("private query details"), 413, "Trace is too large for this view. Use a filtered trace query."), + ( + RuntimeError("private database details"), + 503, + "unavailable", + "Traces are temporarily unavailable. Please try again.", + ), + ( + OverflowError("private query details"), + 413, + "too_large", + "Trace is too large for this view. Use a filtered trace query.", + ), + ( + TraceChanged("Trace changed while paging; refresh the trace to continue"), + 409, + "trace_changed", + "Trace changed while paging; refresh the trace to continue", + ), + (ValueError("Invalid span cursor"), 400, "invalid_request", "Invalid span cursor"), ), ) -def test_read_failures_are_actionable_without_exposing_database_details( - client: TestClient, receiver: MagicMock, path: str, method: str, error: Exception, status: int, message: str +def test_read_failures_carry_a_code_per_kind_without_exposing_database_details( + client: TestClient, + receiver: MagicMock, + path: str, + method: str, + error: Exception, + status: int, + code: str, + message: str, ) -> None: getattr(receiver, method).side_effect = error response: Final = client.get(path) assert response.status_code == status - assert response.json() == {"detail": message} + assert response.json() == {"detail": {"code": code, "message": message}} + retry_after: Final = response.headers.get("Retry-After") + assert (retry_after == str(TRACE_READ_RETRY_AFTER_SECONDS)) == (status == 503), retry_after @pytest.mark.parametrize("query", ("page_size=0", "page_size=501", "cursor=" + "x" * 513)) diff --git a/tests/unit/test_lens_dev.py b/tests/unit/test_lens_dev.py index af61734731e..bc73695f898 100644 --- a/tests/unit/test_lens_dev.py +++ b/tests/unit/test_lens_dev.py @@ -194,12 +194,11 @@ def test_seed_only_with_no_cli_count_preserves_env_controls(tmp_path: Path) -> N proc = _run( tmp_path, "parse_args --seed-only; master_key=sk-local; py() { " - 'printf "%s %s %s\\n" "$LENS_DEV_SEED_COPIES" "$LENS_DEV_SEED_BATCH_COPIES" "$@"; }; py=py; seed_data', + 'printf "%s %s\\n" "$LENS_DEV_SEED_COPIES" "$@"; }; py=py; seed_data', LENS_DEV_SEED_COPIES="3", - LENS_DEV_SEED_BATCH_COPIES="1", ) assert proc.returncode == 0, proc.stderr - assert proc.stdout.startswith("3 1 -m") + assert proc.stdout.startswith("3 -m") def test_proxy_uses_this_checkouts_ui_build(tmp_path: Path) -> None: diff --git a/tests/unit/test_seed_tracing_fixtures.py b/tests/unit/test_seed_tracing_fixtures.py index 30cfde9feec..8dac769b3c1 100644 --- a/tests/unit/test_seed_tracing_fixtures.py +++ b/tests/unit/test_seed_tracing_fixtures.py @@ -11,23 +11,22 @@ import pytest from prisma import Json, Prisma from pydantic import InstanceOf, TypeAdapter +from litellm.rust_bridge.trace.queries import TraceSQLResponse from litellm.rust_bridge.trace.storage import Tenant, span_rows from litellm.tracing.types import SpendLogRecord from scripts.seed_tracing_fixtures import ( JSON, TRACE_FIXTURES, - TenantIdentity, bulk_span_rows, fixture_capture, fixture_replays, postgres_row, rebase, rebase_spend, - replay_batches, response_ids, response_pattern, seed_arguments, - seed_batch, + seed_copy, seed_id, spend_fixtures, timestamps, @@ -196,29 +195,39 @@ def test_bulk_export_preserves_all_spans_and_disjoint_copy_ids() -> None: @pytest.mark.requires_rust_extension @pytest.mark.asyncio -async def test_bulk_seed_stamps_authenticated_tenant_and_writes_both_stores() -> None: - from litellm.rust_bridge.trace.storage import ClickHouseStorage, Tenant +async def test_first_copy_stamps_the_authenticated_tenant_and_writes_both_stores( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.rust_bridge.trace.storage import ClickHouseStorage + monkeypatch.setenv("LITELLM_MASTER_KEY", "sk-local") fixtures: Final = spend_fixtures() pattern: Final = response_pattern(tuple(chain.from_iterable(rows for _, rows in fixtures))) - replays: Final = fixture_replays(TRACE_FIXTURES, 1_800_000_000_000, "bulk", pattern) + replays: Final = fixture_replays(TRACE_FIXTURES, 1_800_000_000_000, "first", pattern) storage: Final = AsyncMock(spec=ClickHouseStorage) + storage.query_sql.return_value = TraceSQLResponse.model_validate( + { + "meta": (), + "data": [{"team_id": "local-team", "api_key": "local-hash", "user": "admin"}], + "rows": 1, + "statistics": {"elapsed": 0, "rows_read": 1, "bytes_read": 1}, + } + ) database: Final = AsyncMock(spec=Prisma, litellm_spendlogs=AsyncMock()) - tenant: Final = TenantIdentity(team_id="local-team", api_key="local-hash", user="admin") - async with httpx.AsyncClient() as client: - result: Final = await seed_batch(client, storage, database, replays, fixtures, pattern, tenant, False) - assert result == tenant - trace_table, trace_rows = storage.insert_rows.call_args_list[0].args - assert trace_table == "otel_traces" - assert trace_rows == bulk_span_rows(replays, Tenant("local-team", "local-hash", user_id="admin")) - table, rows = storage.insert_rows.call_args_list[1].args - assert table == "spend_logs" + client: Final = AsyncMock(spec=httpx.AsyncClient) + client.post.return_value = httpx.Response(200, request=httpx.Request("POST", "http://proxy/v1/traces")) + captures: Final = await seed_copy(client, storage, database, replays, fixtures, pattern) + assert tuple(JSON.validate_json(call.kwargs["content"]) for call in client.post.call_args_list) == tuple( + replay.export for replay in replays + ) + rows: Final = tuple(chain.from_iterable(rows for _, rows in captures)) + assert {name for name, _ in captures} == {name for name, _ in fixtures} + assert storage.insert_rows.call_args.args == ("spend_logs", rows) assert len(rows) == sum(len(original) for _, original in fixtures) assert all((row["team_id"], row["api_key"], row["user"]) == ("local-team", "local-hash", "admin") for row in rows) saved: Final = database.litellm_spendlogs.create_many.call_args.kwargs["data"] assert tuple(row["request_id"] for row in saved) == tuple(row["request_id"] for row in rows) assert tuple(row["spend"] for row in saved) == tuple(row["spend"] for row in rows) - storage.query_sql.assert_not_called() def test_seed_cli_rejects_nonpositive_copies() -> None: @@ -228,23 +237,6 @@ def test_seed_cli_rejects_nonpositive_copies() -> None: assert seed_arguments(["--profile", "large", "--copies", "5"]).copies == 5 -def test_bulk_batches_cover_every_copy_including_partial_tail() -> None: - batches: Final = tuple(replay_batches(8, 1_800_000_000_000, "batch", re.compile(r"(?!)"), 3)) - assert tuple(stop for stop, _ in batches) == (4, 7, 8) - expected: Final = tuple( - tuple(fixture_replays(TRACE_FIXTURES, 1_800_000_000_000 - index * 1000, f"batch-{index}", re.compile(r"(?!)"))) - for index in range(1, 8) - ) - assert tuple(chain.from_iterable(replays for _, replays in batches)) == tuple(chain.from_iterable(expected)) - - -def test_seed_cli_rejects_nonpositive_batch_size() -> None: - with pytest.raises(SystemExit) as error: - seed_arguments(["--batch-copies", "0"]) - assert error.value.code == 2 - assert seed_arguments(["--batch-copies", "2"]).batch_copies == 2 - - @pytest.mark.parametrize("timeout", ("0", "-1", "inf", "nan")) def test_seed_cli_rejects_invalid_http_timeouts(timeout: str) -> None: with pytest.raises(SystemExit) as error: diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 37027f215fe..9aa5764548c 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -47,6 +47,7 @@ "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-reconciler": "0.33.0", + "react-resizable-panels": "4.14.1", "react-syntax-highlighter": "15.6.6", "recharts": "3.9.2", "remark-gfm": "4.0.1", @@ -10649,6 +10650,16 @@ } } }, + "node_modules/react-resizable-panels": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-4.14.1.tgz", + "integrity": "sha512-OB1bXDNTLcGgTTbaX6Dn5efZhlMboSnkfr1w4xpsJTHPEbRJpACBhXpgRCXozgPO1ztgFeZRUe5J3mFNCVKm5g==", + "license": "MIT", + "peerDependencies": { + "react": "^18.0.0 || ^19.0.0", + "react-dom": "^18.0.0 || ^19.0.0" + } + }, "node_modules/react-syntax-highlighter": { "version": "15.6.6", "resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 388b6c7eab9..81b89337325 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -64,6 +64,7 @@ "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", "react-reconciler": "0.33.0", + "react-resizable-panels": "4.14.1", "react-syntax-highlighter": "15.6.6", "recharts": "3.9.2", "remark-gfm": "4.0.1", diff --git a/ui/litellm-dashboard/src/app/globals.css b/ui/litellm-dashboard/src/app/globals.css index cad79f64ea0..cfa85817ea6 100644 --- a/ui/litellm-dashboard/src/app/globals.css +++ b/ui/litellm-dashboard/src/app/globals.css @@ -208,9 +208,12 @@ --trace-tab-active: oklch(0.95 0.025 205); --trace-tab-hover: oklch(0.93 0.02 215); --trace-tag: oklch(0.95 0.02 205); - --trace-chain: oklch(0.56 0.17 255); - --trace-llm: oklch(0.6 0.13 215); - --trace-tool: oklch(0.64 0.14 165); + --trace-chain: #2a78d6; + --trace-llm: #eb6834; + --trace-tool: #1baf7a; + --trace-chain-soft: color-mix(in oklab, #2a78d6 12%, transparent); + --trace-llm-soft: color-mix(in oklab, #eb6834 12%, transparent); + --trace-tool-soft: color-mix(in oklab, #1baf7a 14%, transparent); --trace-glyph: oklch(0.99 0 0); --trace-human: oklch(0.5 0.15 260); --trace-human-glyph: oklch(0.95 0.04 210); @@ -282,9 +285,12 @@ --trace-tab-active: oklch(0.28 0.03 215); --trace-tab-hover: oklch(0.32 0.03 220); --trace-tag: oklch(0.28 0.03 215); - --trace-chain: oklch(0.6 0.17 255); - --trace-llm: oklch(0.64 0.13 215); - --trace-tool: oklch(0.68 0.14 165); + --trace-chain: #3987e5; + --trace-llm: #d95926; + --trace-tool: #199e70; + --trace-chain-soft: color-mix(in oklab, #3987e5 24%, transparent); + --trace-llm-soft: color-mix(in oklab, #d95926 24%, transparent); + --trace-tool-soft: color-mix(in oklab, #199e70 24%, transparent); --trace-glyph: oklch(0.99 0 0); --trace-human: oklch(0.56 0.15 260); --trace-human-glyph: oklch(0.95 0.04 210); @@ -320,6 +326,9 @@ --color-trace-chain: var(--trace-chain); --color-trace-llm: var(--trace-llm); --color-trace-tool: var(--trace-tool); + --color-trace-chain-soft: var(--trace-chain-soft); + --color-trace-llm-soft: var(--trace-llm-soft); + --color-trace-tool-soft: var(--trace-tool-soft); --color-trace-glyph: var(--trace-glyph); --color-trace-human: var(--trace-human); --color-trace-human-glyph: var(--trace-human-glyph); diff --git a/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx b/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx index 11d02567799..0cfa9e47bce 100644 --- a/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx @@ -87,7 +87,7 @@ export function LensModeSwitch({ {label} {view === "investigations" && setup && ( - + {setup} )} diff --git a/ui/litellm-dashboard/src/components/lens/LensPage.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensPage.integration.test.tsx index 32e4e069cc1..b842e6a0bd3 100644 --- a/ui/litellm-dashboard/src/components/lens/LensPage.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensPage.integration.test.tsx @@ -2,7 +2,7 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; -import { dismissLensIntro } from "@/../tests/lens-test-utils"; +import { dismissLensIntro, requestPath } from "@/../tests/lens-test-utils"; import LensPage from "@/app/(dashboard)/lens/page"; const { auth } = vi.hoisted(() => ({ auth: vi.fn() })); @@ -26,7 +26,7 @@ describe("Lens navigation", () => { vi.stubGlobal( "fetch", vi.fn(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/v1/traces") return Response.json({ data: [{}] }); if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: true }); return Response.json({ traces: true, requests: false, data: [] }); diff --git a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx index cfc9a7183d4..c74b266714d 100644 --- a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx @@ -2,7 +2,7 @@ import { act, fireEvent, screen, within, waitFor } from "@testing-library/react" import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; -import { dismissLensIntro } from "@/../tests/lens-test-utils"; +import { dismissLensIntro, readRequest, requestPath } from "@/../tests/lens-test-utils"; import { readStorage } from "@/lib/storage"; import { LENS_INTRO_DISMISSED, LENS_INTRO_SEEN } from "./storage"; import { LensWorkspace } from "./LensWorkspace"; @@ -23,14 +23,14 @@ const worker = () => ({ function serve({ enabled = false, traces = false, requests = false, connected = false } = {}) { list.mockResolvedValue({ lenses: [], workers: connected ? [worker()] : [], tracing_enabled: enabled }); network.mockImplementation(async (input, init) => { - const path = new URL(String(input), "http://localhost").pathname; + const { path, method, body } = await readRequest(input, init); if (path === "/v1/traces") return enabled ? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] }) : Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); if (path === "/lens/activity/available") return Response.json({ traces, requests }); - if (path === "/lens" && init?.method === "POST") { - const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...JSON.parse(String(init.body)) } }; + if (path === "/lens" && method === "POST") { + const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...(body as object) } }; list.mockResolvedValue({ lenses: [saved], workers: [worker()], tracing_enabled: true }); return Response.json(saved); } @@ -154,9 +154,7 @@ describe("Lens setup journey", () => { serve({ enabled: true, traces: true }); const normal = network.getMockImplementation()!; network.mockImplementation((input, init) => - new URL(String(input), "http://localhost").pathname === pendingPath - ? new Promise(() => {}) - : normal(input, init), + requestPath(input) === pendingPath ? new Promise(() => {}) : normal(input, init), ); renderWorkspace(); expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); @@ -171,9 +169,7 @@ describe("Lens setup journey", () => { list.mockResolvedValue({ lenses: data.lenses, workers: [worker()], tracing_enabled: false }); const normal = network.getMockImplementation()!; network.mockImplementation((input, init) => - new URL(String(input), "http://localhost").pathname === pendingPath - ? new Promise(() => {}) - : normal(input, init), + requestPath(input) === pendingPath ? new Promise(() => {}) : normal(input, init), ); renderWorkspace({ searchParams: `?lens=${data.lenses[0].id}` }); expect(await screen.findByRole("heading", { name: data.lenses[0].settings.name })).toBeVisible(); @@ -247,7 +243,7 @@ describe("Lens setup journey", () => { await intro.findByRole("button", { name: "Check for traces" }); const normal = network.getMockImplementation()!; network.mockImplementation(async (input, init) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/v1/traces") return Response.json({ detail: "Trace storage unavailable" }, { status: 503 }); return normal(input, init); }); @@ -268,7 +264,7 @@ describe("Lens setup journey", () => { expect(await screen.findByRole("button", { name: "New investigation" })).toBeEnabled(); const normal = network.getMockImplementation()!; network.mockImplementation((input, init) => - new URL(String(input), "http://localhost").pathname === "/lens/activity/available" + requestPath(input) === "/lens/activity/available" ? Promise.resolve(Response.json({ detail: "Activity unavailable" }, { status: 503 })) : normal(input, init), ); @@ -288,9 +284,7 @@ describe("Lens setup journey", () => { const intro = within(await screen.findByRole("dialog")); expect(await intro.findByText(/A gateway administrator can connect a worker/)).toBeVisible(); expect(intro.getByRole("button", { name: "Connect worker" })).toBeDisabled(); - expect(network.mock.calls.some(([input]) => new URL(String(input), "http://localhost").pathname === "/lens")).toBe( - false, - ); + expect(network.mock.calls.some(([input]) => requestPath(input) === "/lens")).toBe(false); }); it.each(["traces", "requests with trace errors", "requests with pending traces", "traces with activity errors"])( @@ -302,7 +296,7 @@ describe("Lens setup journey", () => { const normal = network.getMockImplementation()!; const failingPath = scenario === "requests with trace errors" ? "/v1/traces" : "/lens/activity/available"; network.mockImplementation((input, init) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/v1/traces" && scenario === "requests with pending traces") return new Promise(() => {}); if (scenario.endsWith("errors") && path === failingPath) @@ -328,13 +322,10 @@ describe("Lens setup journey", () => { within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Investigations" }), ).toHaveAttribute("aria-selected", "true"); await waitFor(() => expect(setupParam(onUrlUpdate)).toBeNull()); - const create = network.mock.calls.find( - ([input, init]) => new URL(String(input), "http://localhost").pathname === "/lens" && init?.method === "POST", - ); + const requests = await Promise.all(network.mock.calls.map(([input, init]) => readRequest(input, init))); + const create = requests.find((request) => request.path === "/lens" && request.method === "POST"); expect(create).toBeDefined(); - expect(JSON.parse(String(create?.[1]?.body))).toEqual( - expect.objectContaining({ name: "My first review", source }), - ); + expect(create?.body).toEqual(expect.objectContaining({ name: "My first review", source })); }, ); }); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index 9e7ece7cf7c..ce99bbae283 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -2,7 +2,7 @@ import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; -import { dismissLensIntro } from "@/../tests/lens-test-utils"; +import { dismissLensIntro, readRequest, requestPath } from "@/../tests/lens-test-utils"; import { LensWorkspace } from "./LensWorkspace"; import { lensKeys } from "./data/queries"; import { createLensDemoData } from "./data/demo/fixtures"; @@ -21,7 +21,7 @@ beforeEach(() => { vi.stubGlobal("fetch", network); network.mockReset(); network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: false }); return Response.json({ data: [], traces: false, requests: false }); @@ -173,7 +173,7 @@ describe("Lens interactive demo", () => { const data = createLensDemoData(); const saved = data.lenses[0]; network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: true }); if (path.endsWith("/runs")) return Response.json(saved.jobs); if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) }); @@ -194,7 +194,7 @@ describe("Lens interactive demo", () => { const user = userEvent.setup(); const saved = createLensDemoData().lenses[0]; network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: false }); if (path.endsWith("/runs")) return Response.json(saved.jobs); if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); @@ -220,7 +220,7 @@ describe("Lens interactive demo", () => { }); const lenses = vi.fn(() => [withJob("running")]); network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: lenses(), workers: [], tracing_enabled: false }); if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); return Response.json({ data: [], traces: false, requests: false }); @@ -237,7 +237,7 @@ describe("Lens interactive demo", () => { const user = userEvent.setup(); const saved = createLensDemoData().lenses[0]; network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [saved], workers: [], tracing_enabled: true }); if (path.endsWith("/runs")) return Response.json(saved.jobs); if (path === "/lens/agents") return Response.json([]); @@ -272,7 +272,7 @@ describe("Lens interactive demo", () => { }; const workers = vi.fn(() => [worker]); network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [saved], workers: workers(), tracing_enabled: true }); if (path.endsWith("/runs")) return Response.json(saved.jobs); if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); @@ -308,7 +308,7 @@ describe("Lens interactive demo", () => { const user = userEvent.setup(); const onUrlUpdate = vi.fn(); network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: true }); if (path === "/v1/traces") return Response.json({ data: [{}] }); return Response.json({ data: [], traces: true, requests: false }); @@ -339,9 +339,9 @@ describe("Lens interactive demo", () => { }; const workers = vi.fn((): (typeof worker)[] => []); network.mockImplementation(async (input, init) => { - const path = new URL(String(input), "http://localhost").pathname; + const { path, method } = await readRequest(input, init); if (path === "/lens") return Response.json({ lenses: [], workers: workers(), tracing_enabled: true }); - if (path === "/lens/workers/register" && init?.method === "POST") { + if (path === "/lens/workers/register" && method === "POST") { workers.mockReturnValue([worker]); return Response.json({ token: "lens-test-token", image: "lens-worker:v1", worker }); } @@ -405,7 +405,7 @@ describe("Lens interactive demo", () => { last_seen: new Date().toISOString(), }; network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [saved], workers: [worker], tracing_enabled: true }); if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); return Response.json({ data: [], traces: true, requests: false }); @@ -434,10 +434,9 @@ describe("Lens interactive demo", () => { last_seen: new Date(Date.now() - 600_000).toISOString(), }; const workers = vi.fn(() => [worker]); - const listCalls = () => - network.mock.calls.filter(([input]) => new URL(String(input), "http://localhost").pathname === "/lens").length; + const listCalls = () => network.mock.calls.filter(([input]) => requestPath(input) === "/lens").length; network.mockImplementation(async (input) => { - const path = new URL(String(input), "http://localhost").pathname; + const path = requestPath(input); if (path === "/lens") return Response.json({ lenses: [saved], workers: workers(), tracing_enabled: true }); if (path === "/v1/traces") return Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); return Response.json({ data: [], traces: true, requests: false }); diff --git a/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx b/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx index 9ef2c316225..a4804212012 100644 --- a/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx +++ b/ui/litellm-dashboard/src/components/lens/data/LensServices.tsx @@ -2,6 +2,7 @@ import { createContext, useContext, useMemo, type ReactNode } from "react"; import { apiClient } from "@/components/networking"; +import { fetchClient } from "@/lib/http/api"; import { liveTracesApi, TracesApiContext, type TracesApi } from "@/components/lens/traces/api"; import { liveLensApi, type LensApi } from "./service"; @@ -14,7 +15,7 @@ export interface LensServices { const LensServicesContext = createContext(null); export function liveLensServices(accessToken: string): LensServices { - return { accessToken, lens: liveLensApi(apiClient, accessToken), traces: liveTracesApi(accessToken) }; + return { accessToken, lens: liveLensApi(fetchClient, apiClient, accessToken), traces: liveTracesApi(accessToken) }; } function useLensServices(): LensServices { diff --git a/ui/litellm-dashboard/src/components/lens/data/service.ts b/ui/litellm-dashboard/src/components/lens/data/service.ts index 6f23aa0dda7..326ee398c0e 100644 --- a/ui/litellm-dashboard/src/components/lens/data/service.ts +++ b/ui/litellm-dashboard/src/components/lens/data/service.ts @@ -1,6 +1,8 @@ import { z } from "zod"; import type { ApiClient } from "@/lib/http/client"; -import type { components } from "@/lib/http/schema"; +import { getAuthHeaderName } from "@/lib/http/runtime"; +import type { Client } from "openapi-fetch"; +import type { components, paths } from "@/lib/http/schema"; import type { ActivitySelection, AnalysisModelInfo, @@ -42,7 +44,7 @@ export interface LensApi { /** Partitions query caches between backends (one token, or the demo). */ readonly scope: string; lenses(): Promise; - activity(): Promise<{ traces: boolean; requests: boolean }>; + activity(): Promise; runs(lensId: string, offset: number): Promise; run(lensId: string, jobId: string): Promise; execution(lensId: string, executionId: string, offset: number): Promise; @@ -57,45 +59,73 @@ export interface LensApi { watchAll(): Promise; cancelRun(lensId: string): Promise; reviewFinding(lensId: string, findingId: string, status: FindingStatus, reason: string): Promise; - registerWorker(analysisKeyId: string | null): Promise; - setWorkerBillingKey(workerId: string, analysisKeyId: string | null): Promise; + registerWorker(analysisKeyId: string): Promise; + setWorkerBillingKey(workerId: string, analysisKeyId: string): Promise; revokeWorker(workerId: string): Promise; generateAnalysisKey(request: AnalysisKeyRequest): Promise<{ token_id?: string }>; deleteKeys(keys: readonly string[]): Promise; } -export function liveLensApi(apiClient: ApiClient, accessToken: string): LensApi { - const encode = encodeURIComponent; +type LensClient = Client; + +async function required(request: Promise<{ data?: T }>): Promise { + const { data } = await request; + if (data === undefined) throw new Error("The proxy returned an empty response"); + return data; +} + +async function sent(request: Promise): Promise { + await request; +} + +export function liveLensApi(client: LensClient, apiClient: ApiClient, accessToken: string): LensApi { + const headers = { [getAuthHeaderName()]: `Bearer ${accessToken}` }; + const lens = (lens_id: string) => ({ headers, params: { path: { lens_id } } }); + const worker = (worker_id: string) => ({ headers, params: { path: { worker_id } } }); return { scope: accessToken, - lenses: () => apiClient.get("/lens", { accessToken }), - activity: () => apiClient.get("/lens/activity/available", { accessToken }), - runs: (lensId, offset) => apiClient.get(`/lens/${lensId}/runs`, { accessToken, query: { offset } }), - run: (lensId, jobId) => apiClient.get(`/lens/${lensId}/runs/${jobId}`, { accessToken }), + lenses: () => required(client.GET("/lens", { headers })), + activity: () => required(client.GET("/lens/activity/available", { headers })), + runs: (lensId, offset) => + required( + client.GET("/lens/{lens_id}/runs", { headers, params: { path: { lens_id: lensId }, query: { offset } } }), + ), + run: (lensId, jobId) => + required( + client.GET("/lens/{lens_id}/runs/{job_id}", { + headers, + params: { path: { lens_id: lensId, job_id: jobId } }, + }), + ), execution: (lensId, executionId, offset) => - apiClient.get(`/lens/${lensId}/executions/${encode(executionId)}`, { - accessToken, - query: { offset }, - }), - sample: (selection, offset, asOf) => { - const { lookback_hours, ...selectionSettings } = selection; - return apiClient.post("/lens/preview/sample", { - accessToken, - body: { - offset, - as_of: asOf, - settings: { - ...selectionSettings, - execution_ids: [], - name: "Preview", - model: "preview", - checks: [{ id: "preview", instruction: "Preview recorded activity" }], + required( + client.GET("/lens/{lens_id}/executions/{execution_id}", { + headers, + params: { path: { lens_id: lensId, execution_id: executionId }, query: { offset } }, + }), + ), + sample: (selection, offset, asOf) => + required( + client.POST("/lens/preview/sample", { + headers, + body: { + offset, + as_of: asOf, + selection: { + source: selection.source, + service: selection.service ?? "", + agent_name: selection.agent_name ?? "", + filters: selection.filters ?? [], + sample_size: selection.sample_size, + sample_percent: selection.sample_percent ?? 100, + team_id: selection.team_id ?? "", + execution_ids: [], + }, + lookback_hours: selection.lookback_hours ?? 24, }, - lookback_hours: lookback_hours ?? 24, - }, - }); - }, - agents: () => apiClient.get("/lens/agents", { accessToken }), + }), + ), + agents: () => required(client.GET("/lens/agents", { headers })), models: () => apiClient.get("/models", { accessToken }), modelDetails: () => apiClient.get("/model_group/info", { accessToken }), keys: async (alias, page, signal) => @@ -118,21 +148,37 @@ export function liveLensApi(apiClient: ApiClient, accessToken: string): LensApi keyInfo: async (keyId) => keyInfoSchema.parse(await apiClient.get("/key/info", { accessToken, query: { key: keyId } })).info, saveLens: (id, settings) => - apiClient.request(id ? "PUT" : "POST", id ? `/lens/${id}` : "/lens", { accessToken, body: settings }), - startRun: (lensId, request = {}) => apiClient.post(`/lens/${lensId}/runs`, { accessToken, body: request }), - watchAll: () => - apiClient.post("/lens/watch-all", { accessToken, body: {} }), - cancelRun: (lensId) => apiClient.post(`/lens/${lensId}/cancel`, { accessToken, body: {} }), + required( + id + ? client.PUT("/lens/{lens_id}", { ...lens(id), body: settings }) + : client.POST("/lens", { headers, body: settings }), + ), + startRun: (lensId, request = {}) => sent(client.POST("/lens/{lens_id}/runs", { ...lens(lensId), body: request })), + watchAll: () => required(client.POST("/lens/watch-all", { headers })), + cancelRun: (lensId) => sent(client.POST("/lens/{lens_id}/cancel", lens(lensId))), reviewFinding: (lensId, findingId, status, reason) => - apiClient.patch(`/lens/${lensId}/findings/${findingId}`, { accessToken, body: { status, reason } }), + sent( + client.PATCH("/lens/{lens_id}/findings/{finding_id}", { + headers, + params: { path: { lens_id: lensId, finding_id: findingId } }, + body: { status, reason }, + }), + ), registerWorker: (analysisKeyId) => - apiClient.post("/lens/workers/register", { - accessToken, - body: { name: "Lens worker", analysis_key_id: analysisKeyId }, - }), + required( + client.POST("/lens/workers/register", { + headers, + body: { name: "Lens worker", analysis_key_id: analysisKeyId }, + }), + ), setWorkerBillingKey: (workerId, analysisKeyId) => - apiClient.put(`/lens/workers/${workerId}/billing-key`, { accessToken, body: { analysis_key_id: analysisKeyId } }), - revokeWorker: (workerId) => apiClient.delete(`/lens/workers/${workerId}`, { accessToken }), + sent( + client.PUT("/lens/workers/{worker_id}/billing-key", { + ...worker(workerId), + body: { analysis_key_id: analysisKeyId }, + }), + ), + revokeWorker: (workerId) => sent(client.DELETE("/lens/workers/{worker_id}", worker(workerId))), generateAnalysisKey: (request) => apiClient.post<{ token_id?: string }>("/key/generate", { accessToken, diff --git a/ui/litellm-dashboard/src/components/lens/investigations/Evidence.tsx b/ui/litellm-dashboard/src/components/lens/investigations/Evidence.tsx index 05017e76b68..fa4b716691c 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/Evidence.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/Evidence.tsx @@ -5,7 +5,7 @@ import { useQuery } from "@tanstack/react-query"; import { Inspector } from "@/components/shared/Inspector"; import { Button } from "@/components/ui/button"; -import { RunView } from "@/components/lens/traces/detail/TraceDrawer"; +import { RunView } from "@/components/lens/traces/detail/run/RunView"; import { useLocalRunSelection } from "@/components/lens/traces/routing"; import { lensQueries } from "../data/queries"; diff --git a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx index 93942cb8fa9..544a7272e14 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx @@ -11,7 +11,7 @@ import type { OwnedFinding } from "../model/inbox"; import type { Finding, Lens } from "../model/types"; import { FindingPanel, ownedFindingKey } from "./FindingDetails"; -vi.mock("@/components/lens/traces/detail/TraceDrawer", () => ({ +vi.mock("@/components/lens/traces/detail/run/RunView", () => ({ RunView: ({ traceId, selection }: { traceId: string; selection: RunSelection }) => (
{traceId} at {selection.spanId} diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx index fded81f230a..039e884c793 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx @@ -2,9 +2,8 @@ import { act, fireEvent, screen, within, waitFor } from "@testing-library/react" import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { testQueryClient } from "@/../tests/test-utils"; -import { renderWithLens } from "@/../tests/lens-test-utils"; +import { renderWithLens, stubGateway } from "@/../tests/lens-test-utils"; import { ApiError } from "@/lib/http/client"; -import { apiClient } from "@/components/networking"; import { lensKeys } from "../data/queries"; import { InvestigationsView } from "./InvestigationsView"; import { LensPreviewContext } from "@/components/lens/ui/LensPreviewButton"; @@ -21,16 +20,20 @@ function renderWithProviders(ui: React.ReactElement, options?: Parameters ({ - apiClient: { get: vi.fn(), post: vi.fn(), patch: vi.fn(), request: vi.fn() }, +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), proxyBaseUrl: "", getProxyBaseUrl: () => "", })); +let proxy = stubGateway(); +const sentBody = (handler: typeof proxy.post, path: string) => + handler.mock.calls.filter(([called]) => called === path).map(([, request]) => request.body); + beforeEach(() => { window.history.replaceState({}, "", "/lens/?lens=lens"); - vi.mocked(apiClient.post).mockReset(); - vi.mocked(apiClient.post).mockResolvedValue({ eligible: 0, selected: 0, executions: [] }); + proxy = stubGateway(); + proxy.post.mockResolvedValue({ eligible: 0, selected: 0, executions: [] }); }); const executionId = btoa(JSON.stringify(["traces", "", "trace-42"])); @@ -161,8 +164,8 @@ const lens: Lens = { describe("Lens findings and runs", () => { beforeEach(() => { - vi.mocked(apiClient.get).mockReset(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockReset(); + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return lens.jobs; return { data: [] }; @@ -197,7 +200,7 @@ describe("Lens findings and runs", () => { async function openIssue(finding: Finding) { testQueryClient.clear(); const jobs = lens.jobs.map((job) => ({ ...job, findings: [finding] })); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [{ ...lens, findings: [finding], jobs }], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return jobs; @@ -245,7 +248,7 @@ describe("Lens findings and runs", () => { it("closes the open run when the keyboard switches to another investigation run", async () => { testQueryClient.clear(); const older = { ...lens.jobs[0], id: "older", created_at: "2026-09-29T10:00:00Z" }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return [...lens.jobs, older]; return { data: [] }; @@ -269,7 +272,7 @@ describe("Lens findings and runs", () => { it("runs with saved settings from Run now without opening setup, then accepts an agent and window", async () => { testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], @@ -289,32 +292,29 @@ it("runs with saved settings from Run now without opening setup, then accepts an if (path === "/lens/activity/available") return { traces: true, requests: false }; return { data: [] }; }); - vi.mocked(apiClient.post).mockResolvedValue(lens); + proxy.post.mockResolvedValue(lens); const user = userEvent.setup(); renderWithProviders(); await user.click(await screen.findByRole("button", { name: "Run now" })); const choices = await screen.findByRole("dialog", { name: "Run now" }); expect(within(choices).getByRole("button", { name: "Since last run" })).toHaveAttribute("aria-pressed", "true"); await user.click(within(choices).getByRole("button", { name: "Run now" })); - expect(apiClient.post).toHaveBeenCalledWith("/lens/lens/runs", { accessToken: "test", body: {} }); + expect(sentBody(proxy.post, "/lens/lens/runs")).toEqual([{}]); await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); - vi.mocked(apiClient.post).mockClear(); + proxy.post.mockClear(); await user.click(screen.getByRole("button", { name: "Run now" })); const custom = await screen.findByRole("dialog", { name: "Run now" }); fireEvent.change(within(custom).getByRole("combobox", { name: "Agent" }), { target: { value: "billing" } }); await user.click(within(custom).getByRole("button", { name: "Last 24h" })); await user.click(within(custom).getByRole("button", { name: "Run now" })); - expect(apiClient.post).toHaveBeenCalledWith("/lens/lens/runs", { - accessToken: "test", - body: { agent_name: "billing", lookback_hours: 24 }, - }); + expect(sentBody(proxy.post, "/lens/lens/runs")).toEqual([{ agent_name: "billing", lookback_hours: 24 }]); }); it("offers the interactive demo without starting an investigation", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [], workers: [], tracing_enabled: false }; return { traces: false, requests: false }; }); @@ -323,13 +323,13 @@ it("offers the interactive demo without starting an investigation", async () => renderWithProviders(withPreview(, onPreview)); await user.click(await screen.findByRole("button", { name: "Preview sample" })); expect(onPreview).toHaveBeenCalledOnce(); - expect(apiClient.post).not.toHaveBeenCalled(); + expect(proxy.post).not.toHaveBeenCalled(); }); it("guides a first-time administrator into worker connection and lens setup", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [], workers: [], tracing_enabled: true }; if (path === "/lens/agents") return []; return { traces: true, requests: false, data: [] }; @@ -341,7 +341,10 @@ it("guides a first-time administrator into worker connection and lens setup", as onboarding: { connect, create }, }); const guide = within(await screen.findByRole("region", { name: "Get Lens running" })); - expect(apiClient.get).toHaveBeenCalledWith("/lens/activity/available", { accessToken: "test" }); + expect(proxy.get).toHaveBeenCalledWith( + "/lens/activity/available", + expect.objectContaining({ authorization: "Bearer test" }), + ); expect(guide.getByRole("button", { name: /Send your first trace/ })).toContainElement( guide.getByLabelText("Step 2 complete"), ); @@ -382,7 +385,7 @@ it("opens the saved results of an older batch", async () => { finished_at: "2026-09-29T10:02:13Z", findings: [{ ...issue, title: "Earlier batch finding" }], }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return [lens.jobs[0], older]; if (path === "/lens/lens/runs/older") return older; @@ -413,11 +416,11 @@ it("reads request content from the beginning after its abbreviated preview", asy executions: [{ ...lens.jobs[0].sample!.executions[0], id: requestId, source: "requests" as const }], }, }; - vi.mocked(apiClient.get).mockImplementation(async (path, options) => { + proxy.get.mockImplementation(async (path, options) => { if (path === "/lens") return { lenses: [{ ...lens, jobs: [job] }], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return [job]; if (!path.includes("/executions/")) return { data: [] }; - const offset = options?.query?.offset ?? 0; + const offset = Number(options.query.offset ?? 0); return { parts: [ { @@ -453,7 +456,7 @@ it.each([false, true])( async (enabled) => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => + proxy.get.mockImplementation(async (path) => path === "/lens" ? { lenses: [], workers: [], tracing_enabled: enabled } : { data: [] }, ); const user = userEvent.setup(); @@ -472,7 +475,7 @@ it("enables first-lens setup when a trace arrives without leaving Investigations window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); const traceCheck = vi.fn().mockResolvedValue({ traces: false, requests: false }); - vi.mocked(apiClient.get).mockImplementation(async (path) => + proxy.get.mockImplementation(async (path) => path === "/lens" ? { lenses: [], workers: [], tracing_enabled: true } : traceCheck(), ); vi.useFakeTimers(); @@ -502,7 +505,7 @@ it("allows retrying a failed trace readiness check without treating it as an emp .fn() .mockRejectedValueOnce(new ApiError("Trace storage unavailable", 503, {})) .mockResolvedValue({ data: [] }); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [], workers: [], tracing_enabled: true }; if (path === "/lens/activity/available") return traceCheck(); return { data: [] }; @@ -523,7 +526,7 @@ it("shows a centered failure with a retry when investigations cannot load, then .fn() .mockRejectedValueOnce(new ApiError("Proxy timed out", 504, {})) .mockResolvedValue({ lenses: [], workers: [], tracing_enabled: true }); - vi.mocked(apiClient.get).mockImplementation(async (path) => (path === "/lens" ? list() : { data: [] })); + proxy.get.mockImplementation(async (path) => (path === "/lens" ? list() : { data: [] })); const user = userEvent.setup(); renderWithProviders(); const alert = await screen.findByRole("alert"); @@ -536,7 +539,7 @@ it("shows a centered failure with a retry when investigations cannot load, then it("keeps saved investigations accessible when tracing is disabled", async () => { testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: false }; if (path === "/lens/lens/runs") return lens.jobs; return { data: [] }; @@ -549,7 +552,7 @@ it("keeps saved investigations accessible when tracing is disabled", async () => it("allows request-only accounts to connect a worker without requiring agent traces", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [], workers: [], tracing_enabled: true }; if (path === "/lens/activity/available") return { traces: false, requests: true }; return { data: [] }; @@ -565,7 +568,7 @@ it("allows request-only accounts to connect a worker without requiring agent tra it("reopens the inline editor from a shared link and drops it from the URL on cancel", async () => { testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return lens.jobs; if (path === "/lens/agents") return []; @@ -587,12 +590,13 @@ it("reopens the inline editor from a shared link and drops it from the URL on ca const url = new URLSearchParams(String(onUrlUpdate.mock.lastCall?.[0].queryString ?? "")); expect(url.has("dialog")).toBe(false); expect(url.get("lens")).toBe(lens.id); - expect(apiClient.request).not.toHaveBeenCalled(); + expect(proxy.put).not.toHaveBeenCalled(); + expect(sentBody(proxy.post, "/lens")).toEqual([]); }); it("reopens a finding and a results section from shared links", async () => { testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return lens.jobs; return { data: [] }; @@ -628,7 +632,7 @@ it("steps across findings and investigations with J and K, skipping hidden findi findings: [twinIssue], jobs: lens.jobs.map((job) => ({ ...job, findings: [twinIssue] })), }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens, twin], tracing_enabled: true, workers: [] }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path.endsWith("/runs")) return []; @@ -665,7 +669,7 @@ it("steps across findings and investigations with J and K, skipping hidden findi it("opens an investigation beside the list and walks from it into its findings with J and K", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], tracing_enabled: true, workers: [] }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path.endsWith("/runs")) return []; @@ -698,13 +702,13 @@ it("lists each finding under the investigation that owns it and resolves only th window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); const twin: Lens = { ...lens, id: "twin", settings: { ...lens.settings, name: "Twin reviews" } }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens, twin], tracing_enabled: true, workers: [] }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path.endsWith("/runs")) return []; return { data: [] }; }); - vi.mocked(apiClient.patch).mockResolvedValue(undefined); + proxy.patch.mockResolvedValue(undefined); const user = userEvent.setup(); renderWithProviders(); const rows = await screen.findAllByRole("row", { name: issue.title }); @@ -714,14 +718,14 @@ it("lists each finding under the investigation that owns it and resolves only th expect(remaining.previousElementSibling).toBe(screen.getByRole("row", { name: twin.settings.name })); await user.click(remaining); await user.click(await screen.findByRole("button", { name: "Mark resolved" })); - await waitFor(() => expect(apiClient.patch).toHaveBeenCalledTimes(1)); - expect(vi.mocked(apiClient.patch).mock.calls[0][0]).toBe("/lens/twin/findings/issue"); + await waitFor(() => expect(proxy.patch).toHaveBeenCalledTimes(1)); + expect(proxy.patch.mock.calls[0][0]).toBe("/lens/twin/findings/issue"); }); it("lists investigations without edit or run controls for read-only viewers", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], tracing_enabled: true, workers: [] }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path === "/lens/lens/runs") return []; @@ -740,7 +744,7 @@ it("lists investigations without edit or run controls for read-only viewers", as it("opens investigations from the keyboard without treating nested edit keys as row activation", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], tracing_enabled: true, workers: [] }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path === "/lens/lens/runs") return []; @@ -779,7 +783,7 @@ it("opens a failed investigation's details from its row and edits only from the error: "boom", findings: [], }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [{ ...lens, jobs: [job] }], workers: [], tracing_enabled: true }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path === "/lens/lens/runs") return [job]; @@ -804,7 +808,7 @@ it("shows the actual saved failure and run context without opening backend logs" "Grouping observations failed: Clusters response invalid after 2 attempts.\n" + "candidates.0.check_id: Field required [missing]"; const job = { ...lens.jobs[0], id: "failed-run", status: "failed" as const, stage: "Failed", error, findings: [] }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [{ ...lens, jobs: [job] }], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return [job]; if (path === "/lens/lens/runs/failed-run") return job; @@ -822,14 +826,14 @@ it("keeps a finding open to retry when its update fails", async () => { window.history.replaceState({}, "", "/lens/"); testQueryClient.clear(); const twin: Lens = { ...lens, id: "twin", settings: { ...lens.settings, name: "Twin reviews" } }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens, twin], tracing_enabled: true, workers: [] }; if (path === "/lens/activity/available") return { traces: true, requests: false }; if (path.endsWith("/runs")) return []; return { data: [] }; }); - vi.mocked(apiClient.patch).mockReset(); - vi.mocked(apiClient.patch).mockImplementation(async (path) => { + proxy.patch.mockReset(); + proxy.patch.mockImplementation(async (path) => { if (String(path).startsWith("/lens/twin/")) throw new Error("Twin reviews could not be updated"); }); const user = userEvent.setup(); @@ -844,28 +848,24 @@ it("keeps a finding open to retry when its update fails", async () => { it("pauses monitoring from the detail menu by saving the investigation with monitoring off", async () => { testQueryClient.clear(); const watching: Lens = { ...lens, settings: { ...lens.settings, enabled: true } }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [watching], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return watching.jobs; return { data: [] }; }); - vi.mocked(apiClient.request).mockReset(); - vi.mocked(apiClient.request).mockResolvedValue({ ...watching, settings: lens.settings }); + proxy.put.mockResolvedValue({ ...watching, settings: lens.settings }); const user = userEvent.setup(); renderWithProviders(); await user.click(await screen.findByRole("button", { name: "Investigation actions" })); await user.click(await screen.findByRole("menuitem", { name: "Pause monitoring" })); - await waitFor(() => expect(apiClient.request).toHaveBeenCalledTimes(1)); - expect(apiClient.request).toHaveBeenCalledWith("PUT", "/lens/lens", { - accessToken: "test", - body: { ...watching.settings, enabled: false }, - }); + await waitFor(() => expect(proxy.put).toHaveBeenCalledTimes(1)); + expect(sentBody(proxy.put, "/lens/lens")).toEqual([{ ...watching.settings, enabled: false }]); }); it("cancels the running job from the progress banner", async () => { testQueryClient.clear(); const running = { ...lens.jobs[0], id: "live", status: "running" as const, stage: "Reading executions" }; - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [{ ...lens, jobs: [running, lens.jobs[0]] }], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return [running, lens.jobs[0]]; @@ -874,15 +874,13 @@ it("cancels the running job from the progress banner", async () => { const user = userEvent.setup(); renderWithProviders(); await user.click(await screen.findByRole("button", { name: "Cancel" })); - await waitFor(() => - expect(apiClient.post).toHaveBeenCalledWith("/lens/lens/cancel", { accessToken: "test", body: {} }), - ); + await waitFor(() => expect(proxy.post).toHaveBeenCalledWith("/lens/lens/cancel", expect.anything())); }); it("refreshes run history as soon as the list reports a job the scheduler started", async () => { testQueryClient.clear(); const runs = vi.fn().mockResolvedValue(lens.jobs); - vi.mocked(apiClient.get).mockImplementation(async (path) => { + proxy.get.mockImplementation(async (path) => { if (path === "/lens") return { lenses: [lens], workers: [], tracing_enabled: true }; if (path === "/lens/lens/runs") return runs(); return { data: [] }; diff --git a/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx b/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx index 7f27678c344..7b64391c85a 100644 --- a/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx +++ b/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx @@ -30,9 +30,9 @@ export function LensGettingStarted({ state, onStart, onExit, onDemo }: LensGetti setupRef.current?.focus({ preventScroll: true }); }; return ( -
+
-
+
!next && close()}>