Merge remote-tracking branch 'upstream/main' into litellm_add_bitdeer_provider

.github/workflows/test-litellm-ui-unit.yml

@@ -49,6 +49,10 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         run: npm ci
 
+      - name: Check UI production source types
+        if: steps.changes.outputs.decision != 'skip'
+        run: npm run typecheck
+
       - name: Run UI type tests (Vitest)
         if: steps.changes.outputs.decision != 'skip'
         env:

.github/workflows/test-rust.yml

@@ -5,6 +5,8 @@ on:
     paths:
       - "litellm-rust/**"
       - "litellm/rust_bridge/**"
+      - "scripts/generate_trace_types.py"
+      - "scripts/trace_codegen/**"
       - "tests/test_litellm_rust/**"
       - "litellm/integrations/custom_logger.py"
       - "litellm/litellm_core_utils/litellm_logging.py"
@@ -32,6 +34,8 @@ on:
     paths:
       - "litellm-rust/**"
       - "litellm/rust_bridge/**"
+      - "scripts/generate_trace_types.py"
+      - "scripts/trace_codegen/**"
       - "tests/test_litellm_rust/**"
       - "litellm/integrations/custom_logger.py"
       - "litellm/litellm_core_utils/litellm_logging.py"
@@ -85,7 +89,7 @@ jobs:
           cache-on-failure: true
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - run: cargo clippy --workspace --all-targets --locked -- -D warnings
+      - run: cargo clippy --workspace --all-targets --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema -- -D warnings
 
   rust-test:
     runs-on: ubuntu-latest
@@ -124,7 +128,11 @@ jobs:
           cache-on-failure: true
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - run: cargo nextest run --workspace --locked
+      - name: Check generated trace contracts
+        working-directory: .
+        run: uv run scripts/generate_trace_types.py --check
+
+      - run: cargo nextest run --workspace --locked --features litellm-traces/schema,litellm-traces-clickhouse/schema
 
       - run: cargo test --workspace --doc --locked
 

.github/workflows/test-unit.yml

@@ -61,7 +61,7 @@ jobs:
 
           - shard: core-utils
             artifact-name: core-utils
-            test-path: ""
+            test-path: tests/unit/decisions
             unit-flag: core-utils
             workers: 2
             reruns: 1
@@ -141,6 +141,7 @@ jobs:
             artifact-name: proxy-endpoints
             test-path: >-
               tests/unit/proxy/analytics_endpoints
+              tests/unit/proxy/decisions_endpoints
               tests/unit/proxy/management_endpoints
               tests/unit/proxy/list_api
               tests/unit/proxy/memory
@@ -165,6 +166,7 @@ jobs:
               tests/unit/proxy/response_api_endpoints
               tests/unit/proxy/image_endpoints
               tests/unit/proxy/ocr_endpoints
+              tests/unit/proxy/search_endpoints
               tests/unit/proxy/vector_store_endpoints
               tests/unit/proxy/agent_endpoints
               tests/unit/proxy/a2a

.gitignore

@@ -151,3 +151,6 @@ litellm.log
 
 .coverage-rust
 coverage-rust.xml
+
+# make lens-dev worker token, generated config and logs
+.lens-dev/

Makefile

@@ -4,7 +4,7 @@
 .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc test-unit-proxy-root \
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
-	test-rust-extension rust-sqlx-prepare \
+	test-rust-extension rust-sqlx-prepare lens-dev \
 	info lint lint-inner lint-dev lint-checks format \
 	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
@@ -58,6 +58,7 @@ help:
 	@echo "  make test-unit-helm     - Run helm unit tests"
 	@echo "  make test-rust-extension - Build the Rust extension and run its public Python tests"
 	@echo "  make rust-sqlx-prepare  - Refresh litellm-rust/crates/db/.sqlx against a migrated Postgres container"
+	@echo "  make lens-dev           - Run proxy + Lens worker + hot-reload dashboard (LENS_DEV_PROXY_PORT, LENS_DEV_UI_PORT)"
 	@echo ""
 	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
 	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
@@ -311,6 +312,9 @@ test-rust-extension:
 rust-sqlx-prepare:
 	cd litellm-rust && cargo run -p litellm-db-testing --bin sqlx-prepare
 
+lens-dev:
+	./scripts/lens_dev.sh
+
 test: install-test-deps
 	$(UV_RUN) pytest tests/
 

README.md

@@ -390,11 +390,13 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) |  |  |  |  |  |  |  |  |  |  |
 | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
 | [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Typesafe Decisions API (`typesafe`)](https://docs.litellm.ai/docs/providers) |  |  |  |  |  |  |  |  |  |  |
 | [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |

deploy/lens/Dockerfile

@@ -2,5 +2,6 @@ FROM python:3.12-slim
 WORKDIR /app
 RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
 COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/
+COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
 USER 65532:65532
 CMD ["python", "-m", "lens.worker"]

deploy/lens/Dockerfile.dockerignore

@@ -4,5 +4,8 @@
 !litellm/proxy/lens/
 !litellm/proxy/lens/__init__.py
 !litellm/proxy/lens/models.py
+!litellm/proxy/lens/trace_store.py
 !litellm/proxy/lens/analysis.py
 !litellm/proxy/lens/worker.py
+!litellm/proxy/lens/prompts/
+!litellm/proxy/lens/prompts/**

deploy/lens/README.md

@@ -33,7 +33,7 @@ For deployments managed with Compose, download `compose.yaml` and provide `LITEL
 docker compose --env-file /path/to/lens.env -f compose.yaml up -d
 ```
 
-Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
+Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`. To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000
 
 The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
 

docker/docker-compose.tracing.yml

@@ -7,7 +7,8 @@ services:
       target: runtime
     command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
     environment:
-      LITELLM_MASTER_KEY: local-tracing-master-key
+      LITELLM_MASTER_KEY: sk-1234
+      LITELLM_DANGEROUSLY_PERMIT_WEAK_OR_UNSET_MASTER_KEY: "true"
       LITELLM_SALT_KEY: sk-local-tracing-salt-key
       DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
       STORE_MODEL_IN_DB: "True"

enterprise/enterprise_hooks/blocked_user_list.py

@@ -7,15 +7,19 @@
 ## This accepts a list of user id's for whom calls will be rejected
 
 
-from typing import Optional, Literal
-import litellm
-from litellm.proxy.utils import PrismaClient
-from litellm.caching.caching import DualCache
-from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
-from litellm.integrations.custom_logger import CustomLogger
-from litellm._logging import verbose_proxy_logger
+from typing import Literal, Optional
+
 from fastapi import HTTPException
 
+import litellm
+from litellm._internal_context import with_service_target
+from litellm._logging import verbose_proxy_logger
+from litellm.caching.caching import DualCache
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
+from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
+from litellm.proxy.utils import PrismaClient
+
 
 class _ENTERPRISE_BlockedUserList(CustomLogger):
     enforces_request_content: bool = True
@@ -54,6 +58,7 @@ class _ENTERPRISE_BlockedUserList(CustomLogger):
         if litellm.set_verbose is True:
             print(print_statement)  # noqa
 
+    @with_service_target(AUTH_OBJECTS_TARGET)
     async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -6,7 +6,7 @@ Base class for sending emails to user after creating keys or invite links
 import html
 import json
 import os
-from typing import List, Literal, Optional
+from typing import Final, List, Literal, Optional
 
 from litellm_enterprise.types.enterprise_callbacks.send_emails import (
     EmailEvent,
@@ -15,6 +15,7 @@ from litellm_enterprise.types.enterprise_callbacks.send_emails import (
     SendKeyRotatedEmailEvent,
 )
 
+from litellm._internal_context import with_service_target
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.constants import (
@@ -48,6 +49,8 @@ from litellm.proxy._types import (
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
 
+_BUDGET_ALERT_CLAIMS_TARGET: Final = "budget_alert_claims"
+
 
 def _max_budget_alert_id(user_info: CallInfo) -> str:
     if user_info.event_group == Litellm_EntityType.TEAM_MEMBER:
@@ -437,6 +440,7 @@ class BaseEmailLogger(CustomLogger):
                 html_body=email_html_content,
             )
 
+    @with_service_target(_BUDGET_ALERT_CLAIMS_TARGET)
     async def budget_alerts(
         self,
         type: Literal[
@@ -606,6 +610,7 @@ class BaseEmailLogger(CustomLogger):
                             await self._release_budget_alert_claim(_cache, _cache_key)
             return
 
+    @with_service_target(_BUDGET_ALERT_CLAIMS_TARGET)
     async def _handle_multi_threshold_max_budget_alert(
         self,
         user_info: CallInfo,
@@ -691,6 +696,7 @@ class BaseEmailLogger(CustomLogger):
                 )
                 await self._release_budget_alert_claim(_cache, _cache_key)
 
+    @with_service_target(_BUDGET_ALERT_CLAIMS_TARGET)
     async def _release_budget_alert_claim(self, cache: DualCache, cache_key: str) -> None:
         try:
             await cache.async_delete_cache(key=cache_key)

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/endpoints.py

@@ -17,6 +17,7 @@ from litellm_enterprise.types.enterprise_callbacks.send_emails import (
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.db.db_span import db_span
 
 router = APIRouter()
 
@@ -94,16 +95,17 @@ async def _save_email_settings(prisma_client, settings: Dict[str, bool]):
         json_settings = json.dumps(general_settings, default=str)
 
         # Save updated general settings
-        await prisma_client.db.litellm_config.upsert(
-            where={"param_name": "general_settings"},
-            data={
-                "create": {
-                    "param_name": "general_settings",
-                    "param_value": json_settings,
+        async with db_span("save_email_settings", "LiteLLM_Config"):
+            await prisma_client.db.litellm_config.upsert(
+                where={"param_name": "general_settings"},
+                data={
+                    "create": {
+                        "param_name": "general_settings",
+                        "param_value": json_settings,
+                    },
+                    "update": {"param_value": json_settings},
                 },
-                "update": {"param_value": json_settings},
-            },
-        )
+            )
     except Exception as e:
         raise HTTPException(
             status_code=500,

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -3,7 +3,7 @@
 
 import base64
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
@@ -26,6 +26,7 @@ from pydantic import ValidationError
 
 import litellm
 from litellm import Router, verbose_logger
+from litellm._internal_context import with_service_target
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
 from litellm.constants import MAX_FILE_LIST_LIMIT
@@ -144,6 +145,7 @@ def _parse_managed_file_object(raw_file_object: object, unified_file_id: str) ->
 class _ManagedFileRow(Protocol):
     unified_file_id: str
     file_object: OpenAIFileObject
+    flat_model_file_ids: Sequence[str]
     storage_backend: Optional[str]
     storage_url: Optional[str]
     created_by: Optional[str]
@@ -201,6 +203,16 @@ def _managed_file_table(prisma_client: PrismaClient) -> _ManagedFileTableActions
     return prisma_client.db.litellm_managedfiletable
 
 
+def _iter_provider_file_id_pairs(
+    rows: Sequence[_ManagedFileRow],
+    requested_provider_file_ids: frozenset[str],
+) -> Iterator[tuple[str, str]]:
+    for row in rows:
+        for provider_file_id in row.flat_model_file_ids:
+            if provider_file_id in requested_provider_file_ids:
+                yield provider_file_id, row.unified_file_id
+
+
 def _managed_object_table(prisma_client: PrismaClient) -> _ManagedObjectTableActions:
     return prisma_client.db.litellm_managedobjecttable
 
@@ -218,6 +230,9 @@ def _storage_metadata_of(file_object: OpenAIFileObject | None) -> Mapping[str, s
     )
 
 
+_MANAGED_FILES_TARGET: Final = "managed_files"
+
+
 class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
     # Class variables or attributes
     def __init__(self, internal_usage_cache: InternalUsageCache, prisma_client: PrismaClient):
@@ -231,6 +246,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
         return PrometheusLogger.get_instance()
 
+    @with_service_target(_MANAGED_FILES_TARGET)
     async def store_unified_file_id(
         self,
         file_id: str,
@@ -314,6 +330,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             verbose_logger.warning(f"could not resolve org for managed object attribution: {e}")
             return None
 
+    @with_service_target(_MANAGED_FILES_TARGET)
     async def store_unified_object_id(
         self,
         unified_object_id: str,
@@ -401,6 +418,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             },
         )
 
+    @with_service_target(_MANAGED_FILES_TARGET)
     async def get_unified_file_id(
         self, file_id: str, litellm_parent_otel_span: Optional[Span] = None
     ) -> Optional[LiteLLM_ManagedFileTable]:
@@ -423,6 +441,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump())
         return None
 
+    @with_service_target(_MANAGED_FILES_TARGET)
     async def delete_unified_file_id(
         self, file_id: str, litellm_parent_otel_span: Optional[Span] = None
     ) -> OpenAIFileObject:
@@ -710,6 +729,39 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             return None
         return batch_obj
 
+    async def get_unified_file_ids_for_provider_file_ids(
+        self,
+        provider_file_ids: Sequence[str],
+        user_api_key_dict: UserAPIKeyAuth,
+    ) -> Mapping[str, str]:
+        if not provider_file_ids:
+            return MappingProxyType({})
+
+        unique_provider_file_ids: Final = tuple(dict.fromkeys(provider_file_ids))
+        owner_filter: Final = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return MappingProxyType({})
+
+        provider_file_ids_list: Final = [  # mutable-ok: Prisma hasSome requires a list
+            provider_file_id for provider_file_id in unique_provider_file_ids
+        ]
+        rows: Final = await _managed_file_table(self.prisma_client).find_many(
+            where={  # mutable-ok: Prisma requires a plain dictionary for where
+                **owner_filter,
+                "flat_model_file_ids": {  # mutable-ok: Prisma requires a plain filter dictionary
+                    "hasSome": provider_file_ids_list,
+                },
+            }
+        )
+        return MappingProxyType(
+            dict(
+                _iter_provider_file_id_pairs(
+                    rows,
+                    frozenset(unique_provider_file_ids),
+                )
+            )
+        )
+
     async def get_user_created_file_ids(
         self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str]
     ) -> List[OpenAIFileObject]:

gateway/routes/allowlist.py

@@ -1,7 +1,7 @@
 """Path allowlist for the gateway component.
 
 The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
-audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
+audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image,
 responses, vector stores, passthrough providers, realtime websockets, MCP
 tool-call endpoints, and operational endpoints (/health, /metrics, and the
 /debug/memory/summary read of the serving worker's RSS).
@@ -60,6 +60,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/v1/rerank",
     "/v2/rerank",
     "/rerank",
+    "/v1/decisions",
+    "/decisions",
     "/v1/ocr",
     "/ocr",
     "/v1/rag/",

litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_background_interaction_settlement/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_BackgroundInteractionSettlement" (
+    "interaction_id" TEXT NOT NULL,
+    "custom_llm_provider" TEXT NOT NULL,
+    "create_context" JSONB NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "claimed_at" TIMESTAMP(3),
+    "claimed_by" TEXT,
+    "settled_at" TIMESTAMP(3),
+    "outcome" TEXT,
+
+    CONSTRAINT "LiteLLM_BackgroundInteractionSettlement_pkey" PRIMARY KEY ("interaction_id")
+);
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "idx_background_interaction_settlement_claimed_at" ON "LiteLLM_BackgroundInteractionSettlement"("claimed_at");

litellm-proxy-extras/litellm_proxy_extras/migrations/20261003000000_add_managed_file_flat_ids_gin_index/migration.sql

@@ -0,0 +1,12 @@
+-- CreateIndex (CONCURRENTLY)
+--
+-- Disclaimer:
+-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
+--   single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
+-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
+--   interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
+-- - Do not edit this file after it has been applied to any database: Prisma checksums
+--   migrations; add a new migration instead.
+-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
+--   without IF NOT EXISTS if you must support older versions).
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_ManagedFileTable_flat_model_file_ids_idx" ON "LiteLLM_ManagedFileTable" USING GIN ("flat_model_file_ids");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1144,6 +1144,7 @@ model LiteLLM_ManagedFileTable {
   updated_by String?
 
   @@index([unified_file_id])
+  @@index([flat_model_file_ids], type: Gin)
   @@index([team_id, created_at(sort: Desc)])
 }
 
@@ -1916,6 +1917,22 @@ model LiteLLM_WorkflowMessage {
   @@index([run_id])
 }
 
+// Pending billing settlements for background interactions, keyed by the
+// interaction id so any replica can settle one that another replica created.
+// `claimed_at` is the exactly-once gate: the first conditional update wins.
+model LiteLLM_BackgroundInteractionSettlement {
+  interaction_id      String    @id
+  custom_llm_provider String
+  create_context      Json
+  created_at          DateTime  @default(now())
+  claimed_at          DateTime?
+  claimed_by          String?
+  settled_at          DateTime?
+  outcome             String?
+
+  @@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
+}
+
 model LiteLLM_Lens {
   id      String @id
   version Int @default(0)

litellm-rust/Cargo.lock

@@ -4453,15 +4453,33 @@ dependencies = [
 name = "litellm-traces"
 version = "0.1.0"
 dependencies = [
+ "askama",
  "criterion",
  "indexmap 2.14.0",
+ "litellm-llms-types",
+ "macro_rules_attribute",
  "opentelemetry-proto",
  "prost",
  "rstest",
+ "schemars 1.2.2",
  "serde",
  "serde_json",
  "strum",
  "thiserror 2.0.19",
+ "time",
+]
+
+[[package]]
+name = "litellm-traces-cache"
+version = "0.1.0"
+dependencies = [
+ "litellm-traces",
+ "moka",
+ "rstest",
+ "serde_json",
+ "sha2 0.10.9",
+ "thiserror 2.0.19",
+ "tokio",
 ]
 
 [[package]]
@@ -4469,15 +4487,21 @@ name = "litellm-traces-clickhouse"
 version = "0.1.0"
 dependencies = [
  "askama",
+ "base64 0.22.1",
  "flate2",
  "futures-util",
  "hmac 0.12.1",
+ "itertools 0.14.0",
+ "jsonschema",
  "litellm-http",
  "litellm-migrate",
  "litellm-storage-clickhouse",
  "litellm-traces",
+ "litellm-traces-cache",
+ "macro_rules_attribute",
  "moka",
  "rstest",
+ "schemars 1.2.2",
  "serde",
  "serde_json",
  "sha2 0.10.9",
@@ -4486,6 +4510,7 @@ dependencies = [
  "thiserror 2.0.19",
  "time",
  "tokio",
+ "tracing",
  "url",
  "wiremock",
 ]

litellm-rust/Cargo.toml

@@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" }
 litellm-router = { path = "crates/router" }
 litellm-tracing = { path = "crates/tracing" }
 litellm-traces = { path = "crates/traces" }
+litellm-traces-cache = { path = "crates/traces-cache" }
 litellm-traces-clickhouse = { path = "crates/traces-clickhouse" }
 litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
 litellm-migrate = { path = "crates/migrate" }

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -467,6 +467,10 @@ pub struct ModelInfo {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_audio_output: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_bedrock_runtime_chat_completions_response_format: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub supports_bedrock_runtime_chat_completions_tools_with_reasoning: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_computer_use: Option<bool>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub supports_embedding_image_input: Option<bool>,

litellm-rust/crates/python-bridge/src/lib.rs

@@ -45,8 +45,7 @@ mod _native {
     use crate::routes::token_counter::TokenCounter;
     #[pymodule_export]
     use crate::routes::traces::{
-        NativeTraceConfig, NativeTraceStorage, trace_decode_otlp, trace_encode_error,
-        trace_normalized_field_definitions,
+        NativeTraceConfig, NativeTraceStorage, trace_encode_error, trace_span_rows,
     };
     #[cfg(feature = "huggingface")]
     #[pymodule_export]
@@ -114,9 +113,8 @@ mod tests {
                 "NativeDiagnosticProcessor",
                 "NativeTraceConfig",
                 "NativeTraceStorage",
-                "trace_decode_otlp",
                 "trace_encode_error",
-                "trace_normalized_field_definitions",
+                "trace_span_rows",
                 "TokenCounter",
                 "Tokenizer",
                 "gil_stats",

litellm-rust/crates/python-bridge/src/routes/traces.rs

@@ -1,14 +1,13 @@
 use std::collections::BTreeMap;
 
-use litellm_host_python::{FromPythonCache, ToPythonCache};
 use litellm_http::ClientVariant;
-use litellm_traces::{QueryScope, ReadQuery, Shared};
+use litellm_traces::{QueryScope, ReadQuery, Tenant, query::named::ReadAccessParams};
 use litellm_traces_clickhouse::{Config, Error, InsertTable, Parameter, QueryReaders};
 use prost::Message;
 use pyo3::{
     exceptions::{PyOverflowError, PyRuntimeError, PyValueError},
     prelude::*,
-    types::{PyBytes, PyDict, PyList, PyMapping, PyString},
+    types::PyBytes,
 };
 
 #[derive(Message)]
@@ -36,14 +35,21 @@ fn map_error_ref(error: &Error) -> PyErr {
     use litellm_storage_clickhouse::Error as StorageError;
 
     match error {
+        Error::Decode(litellm_traces::Error::TooLarge)
+        | Error::InsertTooLarge
+        | Error::ReadTooLarge => PyOverflowError::new_err(error.to_string()),
         Error::InvalidRow
         | Error::InvalidTable
+        | Error::InvalidCursor(_)
+        | Error::AmbiguousTrace
+        | Error::TraceChanged
+        | Error::Decode(_)
         | Error::InvalidSchema
         | Error::InvalidQuery
         | Error::InvalidParameters
         | Error::InvalidScope => PyValueError::new_err(error.to_string()),
-        Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()),
-        Error::SchemaFailed(_)
+        Error::Task
+        | Error::SchemaFailed(_)
         | Error::SchemaTransport
         | Error::MissingSecret
         | Error::Busy
@@ -87,9 +93,15 @@ pub struct NativeTraceConfig {
 #[pymethods]
 impl NativeTraceConfig {
     #[new]
-    fn new(database: String, url: &str, retention_days: u32) -> PyResult<Self> {
+    fn new(
+        database: String,
+        url: &str,
+        retention_days: u32,
+        max_attribute_value_bytes: usize,
+    ) -> PyResult<Self> {
         Ok(Self {
-            inner: Config::new(database, url, retention_days).map_err(map_error)?,
+            inner: Config::new(database, url, retention_days, max_attribute_value_bytes)
+                .map_err(map_error)?,
         })
     }
 }
@@ -137,7 +149,9 @@ impl NativeTraceStorage {
         &self,
         py: Python<'py>,
         table: &str,
-        #[pyo3(from_py_with = insert_rows_from_py)] rows: Vec<litellm_traces_clickhouse::InsertRow>,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec<
+            BTreeMap<String, serde_json::Value>,
+        >,
     ) -> PyResult<Bound<'py, PyAny>> {
         let table = InsertTable::parse(table).map_err(map_error)?;
         let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
@@ -146,13 +160,174 @@ impl NativeTraceStorage {
         crate::execution::run_async(
             py,
             async move {
+                litellm_traces_clickhouse::insert_rows(&client, &connection, &database, table, rows)
+                    .await
+            },
+            map_error,
+        )
+    }
+
+    fn ingest<'py>(
+        &self,
+        py: Python<'py>,
+        payload: &[u8],
+        content_type: Option<String>,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] tenant: Tenant,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let payload = payload.to_vec();
+        let max_value_bytes = self.config.max_attribute_value_bytes();
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.config.storage().writer().clone();
+        let database = self.config.storage().database().to_owned();
+        crate::execution::run_async(
+            py,
+            async move {
+                let rows = tokio::task::spawn_blocking(move || {
+                    litellm_traces::decode_otlp(&payload, content_type.as_deref()).map(|spans| {
+                        litellm_traces_clickhouse::span_rows(spans, &tenant, max_value_bytes)
+                    })
+                })
+                .await
+                .map_err(|_| Error::Task)??;
+                let count = rows.len();
                 litellm_traces_clickhouse::insert_shared_rows(
                     &client,
                     &connection,
                     &database,
-                    table,
+                    InsertTable::OtelTraces,
                     rows,
                 )
+                .await?;
+                Ok(count)
+            },
+            map_error,
+        )
+    }
+
+    #[pyo3(signature = (scope, start_ms, end_ms, cursor, limit))]
+    fn list_traces<'py>(
+        &self,
+        py: Python<'py>,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] scope: ReadAccessParams,
+        start_ms: i64,
+        end_ms: i64,
+        cursor: Option<String>,
+        limit: u32,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.config.storage().reader().clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces_clickhouse::list_traces(
+                    &client,
+                    &connection,
+                    &scope,
+                    start_ms,
+                    end_ms,
+                    cursor.as_deref(),
+                    limit,
+                )
+                .await
+            },
+            map_error,
+        )
+    }
+
+    #[pyo3(signature = (trace_id, scope, trace_ref, cursor=None, page_size=None))]
+    fn get_trace<'py>(
+        &self,
+        py: Python<'py>,
+        trace_id: String,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] scope: ReadAccessParams,
+        trace_ref: String,
+        cursor: Option<String>,
+        page_size: Option<u32>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.config.storage().reader().clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                if let Some(page_size) = page_size {
+                    litellm_traces_clickhouse::get_trace_page(
+                        &client,
+                        &connection,
+                        &scope,
+                        &trace_id,
+                        &trace_ref,
+                        cursor.as_deref(),
+                        page_size,
+                    )
+                    .await
+                } else if cursor.is_some() {
+                    Err(Error::InvalidParameters)
+                } else {
+                    litellm_traces_clickhouse::get_trace(
+                        &client,
+                        &connection,
+                        &scope,
+                        &trace_id,
+                        &trace_ref,
+                    )
+                    .await
+                }
+            },
+            map_error,
+        )
+    }
+
+    fn get_span<'py>(
+        &self,
+        py: Python<'py>,
+        trace_id: String,
+        span_id: String,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] scope: ReadAccessParams,
+        trace_ref: String,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.config.storage().reader().clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces_clickhouse::get_span(
+                    &client,
+                    &connection,
+                    &scope,
+                    &trace_id,
+                    &span_id,
+                    &trace_ref,
+                )
+                .await
+            },
+            map_error,
+        )
+    }
+
+    #[pyo3(signature = (trace_id, span_id, scope, trace_ref, cursor))]
+    fn get_span_error<'py>(
+        &self,
+        py: Python<'py>,
+        trace_id: String,
+        span_id: String,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] scope: ReadAccessParams,
+        trace_ref: String,
+        cursor: Option<String>,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.config.storage().reader().clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces_clickhouse::get_span_error(
+                    &client,
+                    &connection,
+                    &scope,
+                    &trace_id,
+                    &span_id,
+                    &trace_ref,
+                    cursor.as_deref(),
+                )
                 .await
             },
             map_error,
@@ -232,101 +407,23 @@ impl NativeTraceStorage {
     }
 }
 
+/// The `otel_traces` rows an export would be stored as, without writing them.
 #[pyfunction]
-pub fn trace_decode_otlp<'py>(
+pub fn trace_span_rows<'py>(
     py: Python<'py>,
     body: &[u8],
     content_type: Option<&str>,
+    #[pyo3(from_py_with = litellm_host_python::from_py_argument)] tenant: Tenant,
+    max_attribute_value_bytes: usize,
 ) -> PyResult<Bound<'py, PyAny>> {
-    let spans = py
-        .detach(|| litellm_traces::decode_otlp(body, content_type))
-        .map_err(|error| match error {
-            litellm_traces::Error::TooLarge => PyOverflowError::new_err(error.to_string()),
-            _ => PyValueError::new_err(error.to_string()),
-        })?;
-    spans_to_py(py, &spans).map(Bound::into_any)
-}
-
-fn insert_rows_from_py(
-    value: &Bound<'_, PyAny>,
-) -> PyResult<Vec<litellm_traces_clickhouse::InsertRow>> {
-    let mut resources = FromPythonCache::default();
-    value
-        .try_iter()?
-        .map(|row| {
-            let row = row?;
-            let mut fields = BTreeMap::new();
-            for item in row.cast::<PyMapping>()?.items()?.iter() {
-                let (key, value): (String, Bound<'_, PyAny>) = item.extract()?;
-                let converted = if matches!(
-                    key.as_str(),
-                    "ResourceAttributes" | "ScopeName" | "ScopeVersion"
-                ) {
-                    resources
-                        .get_or_try_insert_with(&value, |value| {
-                            litellm_host_python::from_py_argument::<serde_json::Value>(value)
-                                .map(Shared::new)
-                        })?
-                        .clone()
-                } else {
-                    Shared::new(litellm_host_python::from_py_argument(&value)?)
-                };
-                fields.insert(key, converted);
-            }
-            Ok(fields)
+    let rows = py
+        .detach(|| {
+            litellm_traces::decode_otlp(body, content_type).map(|spans| {
+                litellm_traces_clickhouse::span_rows(spans, &tenant, max_attribute_value_bytes)
+            })
         })
-        .collect()
-}
-
-fn spans_to_py<'py>(
-    py: Python<'py>,
-    spans: &[litellm_traces::DecodedSpan],
-) -> PyResult<Bound<'py, PyList>> {
-    let mut resources = ToPythonCache::default();
-    let mut scopes = ToPythonCache::default();
-    let result = PyList::empty(py);
-    for span in spans {
-        let resource = resources
-            .get_or_try_insert_with(span.resource_attributes.as_ref(), |value| {
-                litellm_host_python::Pythonized(value).into_pyobject(py)
-            })?;
-        let row = PyDict::new(py);
-        row.set_item("trace_id", &span.trace_id)?;
-        row.set_item("span_id", &span.span_id)?;
-        row.set_item("parent_span_id", &span.parent_span_id)?;
-        row.set_item("trace_state", &span.trace_state)?;
-        row.set_item("name", &span.name)?;
-        row.set_item("kind", &span.kind)?;
-        row.set_item("resource_attributes", resource)?;
-        for (key, value) in [
-            ("scope_name", &span.scope_name),
-            ("scope_version", &span.scope_version),
-        ] {
-            let value = scopes.get_or_try_insert_with(value.as_ref(), |value| {
-                Ok(PyString::new(py, value).into_any())
-            })?;
-            row.set_item(key, value)?;
-        }
-        row.set_item("attributes", &span.attributes)?;
-        row.set_item("start_ns", span.start_ns)?;
-        row.set_item("end_ns", span.end_ns)?;
-        row.set_item("status_code", &span.status_code)?;
-        row.set_item("status_message", &span.status_message)?;
-        row.set_item(
-            "events",
-            litellm_host_python::Pythonized(&span.events).into_pyobject(py)?,
-        )?;
-        row.set_item(
-            "normalized",
-            litellm_host_python::Pythonized(&span.normalized).into_pyobject(py)?,
-        )?;
-        row.set_item(
-            "consumed_attributes",
-            litellm_host_python::Pythonized(&span.consumed_attributes).into_pyobject(py)?,
-        )?;
-        result.append(row)?;
-    }
-    Ok(result)
+        .map_err(|error| map_error(error.into()))?;
+    litellm_host_python::Pythonized(rows).into_pyobject(py)
 }
 
 #[cfg(test)]
@@ -383,50 +480,22 @@ mod tests {
     }
 
     #[rstest]
-    fn insert_projection_preserves_identity_without_merging_equal_resources() {
+    #[case::decode_budget(Error::Decode(litellm_traces::Error::TooLarge), "OverflowError")]
+    #[case::invalid_export(Error::Decode(litellm_traces::Error::InvalidPayload), "ValueError")]
+    #[case::cursor(Error::InvalidCursor("trace"), "ValueError")]
+    #[case::ambiguous(Error::AmbiguousTrace, "ValueError")]
+    #[case::changed_snapshot(Error::TraceChanged, "ValueError")]
+    #[case::read_budget(Error::ReadTooLarge, "OverflowError")]
+    fn trace_read_and_ingest_failures_preserve_public_exception_types(
+        #[case] error: Error,
+        #[case] exception_name: &str,
+    ) {
         Python::initialize();
         Python::attach(|py| {
-            let resource = PyDict::new(py);
-            resource.set_item("service.name", "shared").unwrap();
-            let equal_resource = resource.copy().unwrap();
-            let rows = PyList::empty(py);
-            for value in [&resource, &resource, &equal_resource] {
-                let row = PyDict::new(py);
-                row.set_item("ResourceAttributes", value).unwrap();
-                rows.append(row).unwrap();
-            }
-            let projected = insert_rows_from_py(rows.as_any()).unwrap();
-            assert!(Shared::shares_storage_with(
-                &projected[0]["ResourceAttributes"],
-                &projected[1]["ResourceAttributes"]
-            ));
-            assert!(!Shared::shares_storage_with(
-                &projected[0]["ResourceAttributes"],
-                &projected[2]["ResourceAttributes"]
-            ));
-            assert_eq!(projected[0], projected[2]);
-        });
-    }
-
-    #[rstest]
-    fn shared_conversion_preserves_every_decoded_field() {
-        Python::initialize();
-        Python::attach(|py| {
-            let spans = litellm_traces::decode_otlp(
-                include_bytes!("../../../../../tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json"),
-                Some("application/json"),
-            ).unwrap();
-            let expected = litellm_host_python::Pythonized(&spans)
-                .into_pyobject(py)
-                .unwrap();
-            let actual = spans_to_py(py, &spans).unwrap();
-            assert!(actual.eq(expected).unwrap());
+            assert_eq!(
+                map_error(error).get_type(py).name().unwrap(),
+                exception_name
+            );
         });
     }
 }
-
-#[pyfunction]
-pub fn trace_normalized_field_definitions<'py>(py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
-    litellm_host_python::Pythonized(litellm_traces_clickhouse::NORMALIZED_FIELD_DEFINITIONS)
-        .into_pyobject(py)
-}

litellm-rust/crates/storage-clickhouse/src/read.rs

@@ -110,6 +110,13 @@ pub async fn execute_read(
         .body(sql.to_owned());
     let mut response = request.send().await.map_err(|_| Error::Transport)?;
     if !response.status().is_success() {
+        if response
+            .headers()
+            .get("x-clickhouse-exception-code")
+            .is_some_and(|code| code == "396")
+        {
+            return Err(Error::ResponseTooLarge);
+        }
         return Err(Error::QueryFailed(response.status().as_u16()));
     }
 

litellm-rust/crates/storage-clickhouse/tests/transport.rs

@@ -111,3 +111,36 @@ async fn typed_fetch_encodes_parameters_and_validates_rows(
         assert!(matches!(envelope, Err(Error::InvalidResponse)));
     }
 }
+
+#[rstest]
+#[case::result_limit("396", true)]
+#[case::memory_limit("241", false)]
+#[case::timeout("159", false)]
+#[case::unknown("", false)]
+#[tokio::test]
+async fn server_result_limits_allow_smaller_pages_without_retrying_other_failures(
+    #[case] code: &str,
+    #[case] result_limit: bool,
+) {
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
+    let server = MockServer::start().await;
+    Mock::given(method("POST"))
+        .respond_with(ResponseTemplate::new(500).insert_header("X-ClickHouse-Exception-Code", code))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let connection = Connection::parse(&server.uri()).unwrap();
+    let error = execute_read(
+        &Client::no_redirect_for_test(),
+        &connection,
+        "SELECT 1",
+        &BTreeMap::new(),
+    )
+    .await
+    .unwrap_err();
+    if result_limit {
+        assert!(matches!(error, Error::ResponseTooLarge));
+    } else {
+        assert!(matches!(error, Error::QueryFailed(500)));
+    }
+}

litellm-rust/crates/traces-cache/AGENTS.md

@@ -0,0 +1,5 @@
+Own resolved-trace snapshot storage, cache identity, weighting, and expiry
+Depend on trace domain types, never storage, HTTP, or Python
+Preserve the full source and authorization scope in every cache key
+Keep snapshots immutable and expose borrowed data
+Keep cursor formats and database reads in their existing owners

litellm-rust/crates/traces-cache/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "litellm-traces-cache"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-traces.workspace = true
+moka.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tokio.workspace = true

litellm-rust/crates/traces-cache/src/error.rs

@@ -0,0 +1,7 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("trace snapshot serialization failed")]
+    Serialization(#[from] serde_json::Error),
+    #[error("trace snapshot exceeds the size limit")]
+    ReadTooLarge,
+}

litellm-rust/crates/traces-cache/src/lib.rs

@@ -0,0 +1,166 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_traces::{Trace, query::named::ReadAccessParams};
+use moka::future::Cache;
+use sha2::{Digest, Sha256};
+
+mod error;
+
+pub use error::Error;
+
+#[derive(Clone, Eq, Hash, PartialEq)]
+pub struct SnapshotKey(String);
+
+impl SnapshotKey {
+    pub fn new(
+        source: &str,
+        access: &ReadAccessParams,
+        trace_id: &str,
+        trace_ref: &str,
+        snapshot_ms: u64,
+    ) -> Result<Self, Error> {
+        let encoded = serde_json::to_vec(&(source, access, trace_id, trace_ref, snapshot_ms))?;
+        Ok(Self(format!("{:x}", Sha256::digest(encoded))))
+    }
+}
+
+pub struct Snapshot {
+    trace: Trace,
+    version: String,
+    weight: u32,
+}
+
+impl Snapshot {
+    pub fn trace(&self) -> &Trace {
+        &self.trace
+    }
+
+    pub fn version(&self) -> &str {
+        &self.version
+    }
+}
+
+pub struct SnapshotCache {
+    entries: Cache<SnapshotKey, Arc<Snapshot>>,
+    max_graph_bytes: usize,
+}
+
+impl SnapshotCache {
+    pub fn new(max_graph_bytes: usize, ttl: Duration) -> Self {
+        Self {
+            entries: Cache::builder()
+                .max_capacity((max_graph_bytes as u64).saturating_mul(2))
+                .weigher(|_: &SnapshotKey, snapshot: &Arc<Snapshot>| snapshot.weight)
+                .time_to_live(ttl)
+                .build(),
+            max_graph_bytes,
+        }
+    }
+
+    pub async fn get(&self, key: &SnapshotKey) -> Option<Arc<Snapshot>> {
+        self.entries.get(key).await
+    }
+
+    pub async fn insert(&self, key: SnapshotKey, trace: Trace) -> Result<Arc<Snapshot>, Error> {
+        let encoded = serde_json::to_vec(&trace)?;
+        if encoded.len() > self.max_graph_bytes {
+            return Err(Error::ReadTooLarge);
+        }
+
+        let span_ids: Vec<&str> = trace
+            .spans
+            .iter()
+            .map(|span| span.span_id.as_str())
+            .collect();
+
+        let version = format!("{:x}", Sha256::digest(serde_json::to_vec(&span_ids)?));
+
+        let snapshot = Arc::new(Snapshot {
+            trace,
+            version,
+            weight: u32::try_from(encoded.len().saturating_mul(2)).unwrap_or(u32::MAX),
+        });
+
+        self.entries.insert(key, Arc::clone(&snapshot)).await;
+        Ok(snapshot)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_traces::{
+        SpanStatus,
+        query::named::{SpendByResponseIdsRow, TraceSpansRow},
+        resolve_trace,
+    };
+
+    use super::*;
+
+    fn trace(span_id: &str) -> Trace {
+        let rows = [TraceSpansRow {
+            trace_id: String::new(),
+            span_id: span_id.into(),
+            parent_span_id: String::new(),
+            name: "run".into(),
+            kind: litellm_traces::ObservationType::Agent,
+            wrapper_candidate: false,
+            agent: "agent".into(),
+            framework: String::new(),
+            status: SpanStatus::Ok,
+            status_message: String::new(),
+            error_truncated: false,
+            start_ns: 1_790_742_989_000_000_000,
+            duration_ns: 10_000_000,
+            service: "agent-demo".into(),
+            input_preview: format!("input of {span_id}"),
+            model: String::new(),
+            input_tokens: 0,
+            output_tokens: 0,
+            litellm_request_id: String::new(),
+            call_keys: Vec::new(),
+            call_evidence: None,
+            tool_call_id: String::new(),
+            team_id: String::new(),
+            api_key_hash: String::new(),
+            user_id: String::new(),
+        }];
+        resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow])
+            .expect("fixture should resolve")
+    }
+
+    fn key(suffix: &str) -> SnapshotKey {
+        SnapshotKey::new(
+            "source",
+            &ReadAccessParams {
+                all_teams: false,
+                user_id: String::new(),
+                team_ids: vec!["team".into()],
+            },
+            suffix,
+            "ref",
+            100,
+        )
+        .unwrap()
+    }
+
+    #[tokio::test]
+    async fn weighted_capacity_bounds_retained_snapshots() {
+        let limit = ["first", "second", "third"]
+            .iter()
+            .map(|span_id| serde_json::to_vec(&trace(span_id)).unwrap().len())
+            .max()
+            .unwrap();
+        let cache = SnapshotCache::new(limit, Duration::from_secs(120));
+
+        for (key, span_id) in [
+            (key("a"), "first"),
+            (key("b"), "second"),
+            (key("c"), "third"),
+        ] {
+            cache.insert(key, trace(span_id)).await.unwrap();
+        }
+
+        cache.entries.run_pending_tasks().await;
+        assert!(cache.entries.weighted_size() <= (limit as u64) * 2);
+    }
+}

litellm-rust/crates/traces-cache/tests/snapshots.rs

@@ -0,0 +1,191 @@
+use std::time::Duration;
+
+use litellm_traces::{
+    SpanStatus, Trace,
+    query::named::{ReadAccessParams, SpendByResponseIdsRow, TraceSpansRow},
+    resolve_trace,
+};
+use litellm_traces_cache::{Error, SnapshotCache, SnapshotKey};
+use rstest::{fixture, rstest};
+
+const T0: i64 = 1_790_742_989_000_000_000;
+const MS: i64 = 1_000_000;
+const TTL: Duration = Duration::from_secs(120);
+
+fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
+    TraceSpansRow {
+        trace_id: String::new(),
+        span_id: span_id.into(),
+        parent_span_id: parent.into(),
+        name: name.into(),
+        kind: kind.parse().unwrap(),
+        wrapper_candidate: false,
+        agent: agent.into(),
+        framework: String::new(),
+        status: SpanStatus::Ok,
+        status_message: String::new(),
+        error_truncated: false,
+        start_ns: T0,
+        duration_ns: 10 * MS as u64,
+        service: "agent-demo".into(),
+        input_preview: format!("input of {name}"),
+        model: String::new(),
+        input_tokens: 0,
+        output_tokens: 0,
+        litellm_request_id: String::new(),
+        call_keys: Vec::new(),
+        call_evidence: None,
+        tool_call_id: String::new(),
+        team_id: String::new(),
+        api_key_hash: String::new(),
+        user_id: String::new(),
+    }
+}
+
+fn access() -> ReadAccessParams {
+    ReadAccessParams {
+        all_teams: false,
+        user_id: String::new(),
+        team_ids: vec!["team".into()],
+    }
+}
+
+fn key(
+    source: &str,
+    access: &ReadAccessParams,
+    trace_id: &str,
+    trace_ref: &str,
+    ms: u64,
+) -> SnapshotKey {
+    SnapshotKey::new(source, access, trace_id, trace_ref, ms).unwrap()
+}
+
+#[fixture]
+fn trace() -> Trace {
+    resolve_trace(
+        "trace",
+        "ref",
+        &[row("root", "", "run", "agent", "agent")],
+        &[] as &[SpendByResponseIdsRow],
+    )
+    .expect("fixture should resolve")
+}
+
+#[rstest]
+#[case::different_team(false, "", "other-team")]
+#[case::different_user(false, "other-user", "team")]
+#[case::different_scope(true, "", "team")]
+#[tokio::test]
+async fn cached_trace_is_isolated_by_access_scope(
+    trace: Trace,
+    #[case] all_teams: bool,
+    #[case] user_id: &str,
+    #[case] team_id: &str,
+) {
+    let cache = SnapshotCache::new(1024 * 1024, TTL);
+    let stored = key("source", &access(), "trace", "ref", 100);
+
+    cache.insert(stored.clone(), trace.clone()).await.unwrap();
+
+    let other_access = ReadAccessParams {
+        all_teams,
+        user_id: user_id.into(),
+        team_ids: vec![team_id.into()],
+    };
+    let other = key("source", &other_access, "trace", "ref", 100);
+
+    assert!(cache.get(&other).await.is_none());
+    let cached = cache.get(&stored).await.unwrap();
+    assert_eq!(cached.trace(), &trace);
+}
+
+#[rstest]
+#[case::different_source("other-source", "trace", "ref", 100)]
+#[case::different_trace_id("source", "other-trace", "ref", 100)]
+#[case::different_trace_ref("source", "trace", "other-ref", 100)]
+#[case::different_snapshot_ms("source", "trace", "ref", 200)]
+#[tokio::test]
+async fn cached_trace_is_isolated_by_key_fields(
+    trace: Trace,
+    #[case] source: &str,
+    #[case] trace_id: &str,
+    #[case] trace_ref: &str,
+    #[case] snapshot_ms: u64,
+) {
+    let cache = SnapshotCache::new(1024 * 1024, TTL);
+    let stored = key("source", &access(), "trace", "ref", 100);
+
+    cache.insert(stored.clone(), trace.clone()).await.unwrap();
+
+    let other = key(source, &access(), trace_id, trace_ref, snapshot_ms);
+    assert!(cache.get(&other).await.is_none());
+    assert!(cache.get(&stored).await.is_some());
+}
+
+#[rstest]
+#[tokio::test]
+async fn snapshot_at_the_size_limit_is_accepted(trace: Trace) {
+    let size = serde_json::to_vec(&trace).unwrap().len();
+    let cache = SnapshotCache::new(size, TTL);
+    let stored = key("source", &access(), "trace", "ref", 100);
+
+    cache.insert(stored.clone(), trace).await.unwrap();
+    assert!(cache.get(&stored).await.is_some());
+}
+
+#[rstest]
+#[tokio::test]
+async fn snapshot_one_byte_over_the_size_limit_is_rejected(trace: Trace) {
+    let size = serde_json::to_vec(&trace).unwrap().len();
+    let cache = SnapshotCache::new(size - 1, TTL);
+    let stored = key("source", &access(), "trace", "ref", 100);
+
+    assert!(matches!(
+        cache.insert(stored.clone(), trace).await,
+        Err(Error::ReadTooLarge)
+    ));
+    assert!(cache.get(&stored).await.is_none());
+}
+
+#[rstest]
+#[case::same_ids(&["root", "child"], &["root", "child"], true)]
+#[case::different_ids(&["root", "child"], &["root", "other"], false)]
+#[tokio::test]
+async fn snapshot_version_tracks_the_ordered_span_ids(
+    #[case] first_ids: &[&str],
+    #[case] second_ids: &[&str],
+    #[case] equal: bool,
+) {
+    let build = |ids: &[&str]| -> Trace {
+        let rows: Vec<TraceSpansRow> = ids
+            .iter()
+            .map(|span_id| row(span_id, "", "run", "agent", "agent"))
+            .collect();
+        resolve_trace("trace", "ref", &rows, &[] as &[SpendByResponseIdsRow])
+            .expect("fixture should resolve")
+    };
+    let cache = SnapshotCache::new(1024 * 1024, TTL);
+
+    let first = cache
+        .insert(key("source", &access(), "a", "ref", 100), build(first_ids))
+        .await
+        .unwrap();
+    let second = cache
+        .insert(key("source", &access(), "b", "ref", 100), build(second_ids))
+        .await
+        .unwrap();
+
+    assert_eq!(first.version() == second.version(), equal);
+}
+
+#[rstest]
+#[tokio::test]
+async fn snapshots_expire_after_the_ttl(trace: Trace) {
+    let cache = SnapshotCache::new(1024 * 1024, Duration::from_millis(50));
+    let stored = key("source", &access(), "trace", "ref", 100);
+
+    cache.insert(stored.clone(), trace).await.unwrap();
+    tokio::time::sleep(Duration::from_millis(200)).await;
+
+    assert!(cache.get(&stored).await.is_none());
+}

litellm-rust/crates/traces-clickhouse/Cargo.toml

@@ -5,15 +5,23 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+schema = ["dep:schemars", "litellm-traces/schema"]
+
 [dependencies]
+macro_rules_attribute.workspace = true
+schemars = { workspace = true, optional = true }
 askama.workspace = true
+base64.workspace = true
 flate2.workspace = true
 futures-util.workspace = true
 hmac = "0.12.1"
+itertools = "0.14.0"
 litellm-http.workspace = true
 litellm-migrate.workspace = true
 litellm-storage-clickhouse.workspace = true
 litellm-traces.workspace = true
+litellm-traces-cache.workspace = true
 moka.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -22,10 +30,17 @@ strum.workspace = true
 thiserror.workspace = true
 time = { workspace = true, features = ["formatting"] }
 tokio.workspace = true
+tracing.workspace = true
 url.workspace = true
 
 [dev-dependencies]
+jsonschema = { version = "0.55.1", default-features = false }
 litellm-http = { workspace = true, features = ["test-support"] }
 rstest.workspace = true
 testcontainers-modules = { version = "0.15.0", features = ["clickhouse"] }
 wiremock.workspace = true
+
+[[bin]]
+name = "export-traces-clickhouse-schema"
+path = "src/bin/export_schema.rs"
+required-features = ["schema"]

litellm-rust/crates/traces-clickhouse/migrations/0012_otel_traces_call_evidence.sql

@@ -0,0 +1,5 @@
+ALTER TABLE {database}.otel_traces
+    ADD COLUMN IF NOT EXISTS WrapperCandidate Bool DEFAULT false AFTER ObservationType,
+    ADD COLUMN IF NOT EXISTS CallKeys Array(String) DEFAULT [] AFTER LiteLLMRequestId,
+    ADD COLUMN IF NOT EXISTS CallEvidence LowCardinality(String) DEFAULT '' AFTER CallKeys,
+    ADD COLUMN IF NOT EXISTS ToolCallId String DEFAULT '' AFTER Output

litellm-rust/crates/traces-clickhouse/migrations/0013_otel_traces_agent_metadata.sql

@@ -0,0 +1,2 @@
+ALTER TABLE {database}.otel_traces
+    ADD COLUMN IF NOT EXISTS AgentMetadata String DEFAULT '{}' CODEC(ZSTD(3))

litellm-rust/crates/traces-clickhouse/migrations/0014_spend_unknown_cost.sql

@@ -0,0 +1 @@
+ALTER TABLE {database}.spend_logs MODIFY COLUMN spend Nullable(Float64) DEFAULT NULL

litellm-rust/crates/traces-clickhouse/query/help/correlated_calls.sql

@@ -0,0 +1,14 @@
+SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata
+FROM otel_traces AS t
+INNER JOIN (
+    SELECT *
+    FROM spend_logs FINAL
+    WHERE start_time >= now() - INTERVAL 1 DAY
+) AS s
+    ON t.LiteLLMRequestId = s.response_id
+    AND t.TeamId = s.team_id
+    AND ((t.UserId != '' AND t.UserId = s.user)
+        OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
+WHERE t.Timestamp >= now() - INTERVAL 1 DAY
+    AND t.LiteLLMRequestId != ''
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/custom_metadata.sql

@@ -0,0 +1,8 @@
+SELECT
+    request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project
+FROM spend_logs FINAL
+WHERE start_time >= now() - INTERVAL 1 DAY
+    AND JSONHas(metadata, 'project')
+    AND JSONExtractString(metadata, 'project') = 'example'
+ORDER BY start_time DESC
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/discover_keys.sql

@@ -0,0 +1,6 @@
+SELECT
+    DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key
+FROM spend_logs FINAL
+WHERE start_time >= now() - INTERVAL 30 DAY
+ORDER BY key
+LIMIT 200

litellm-rust/crates/traces-clickhouse/query/help/failed_spans.sql

@@ -0,0 +1,7 @@
+SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
+       SpanId AS span_id, StatusMessage AS message
+FROM otel_traces
+WHERE Timestamp >= now() - INTERVAL 1 DAY
+    AND StatusCode = 'STATUS_CODE_ERROR'
+ORDER BY Timestamp DESC, team, api_key, trace_id, span_id
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/metadata_filter.sql

@@ -1,5 +1,8 @@
 SELECT team_id AS team, api_key, request_id, spend,
        JSONExtractString(metadata, 'labels', 'priority') AS priority
 FROM spend_logs FINAL
-WHERE JSONExtractString(metadata, 'labels', 'priority') = 'high'
+WHERE start_time >= now() - INTERVAL 1 DAY
+    AND JSONHas(metadata, 'labels', 'priority')
+    AND JSONExtractString(metadata, 'labels', 'priority') = 'high'
 ORDER BY team, api_key, request_id
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/model_spend.sql

@@ -0,0 +1,17 @@
+SELECT
+    team_id, model, requests, unknown_cost_requests,
+    if(unknown_cost_requests = 0, recorded_spend, NULL) AS spend,
+    input_tokens, output_tokens
+FROM (
+    SELECT
+        team_id, model, count() AS requests,
+        countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
+        sum(spend) AS recorded_spend,
+        sum(prompt_tokens) AS input_tokens,
+        sum(completion_tokens) AS output_tokens
+    FROM spend_logs FINAL
+    WHERE start_time >= now() - INTERVAL 1 DAY
+    GROUP BY team_id, model
+)
+ORDER BY team_id, model
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/nested_metadata.sql

@@ -0,0 +1,8 @@
+SELECT
+    request_id,
+    JSONType(metadata, 'labels', 'priority') AS type,
+    JSONExtractRaw(metadata, 'labels', 'priority') AS value
+FROM spend_logs FINAL
+WHERE start_time >= now() - INTERVAL 1 DAY
+    AND JSONHas(metadata, 'labels', 'priority')
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/recent_spans.sql

@@ -0,0 +1,8 @@
+SELECT
+    TraceId, SpanId, Model, InputTokens, OutputTokens,
+    Duration / 1000000 AS duration_ms
+FROM otel_traces
+WHERE Timestamp >= now() - INTERVAL 1 DAY
+    AND ObservationType = 'llm'
+ORDER BY Timestamp DESC
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/recent_spend.sql

@@ -0,0 +1,8 @@
+SELECT
+    request_id, response_id, trace_id, span_id, model, spend,
+    prompt_tokens, completion_tokens, status,
+    JSONExtractBool(metadata, 'synthetic_spend') AS synthetic_spend
+FROM spend_logs FINAL
+WHERE start_time >= now() - INTERVAL 1 DAY
+ORDER BY start_time DESC, request_id
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/trace_spend.sql

@@ -0,0 +1,10 @@
+SELECT
+    team_id, api_key, trace_id, count() AS requests,
+    countIf(isNull(spend) OR NOT isFinite(spend)) AS unknown_cost_requests,
+    if(unknown_cost_requests = 0, sum(spend), NULL) AS recorded_spend
+FROM spend_logs FINAL
+WHERE start_time >= now() - INTERVAL 1 DAY
+    AND trace_id != ''
+GROUP BY team_id, api_key, trace_id
+ORDER BY team_id, api_key, trace_id
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/trace_summary.sql

@@ -7,4 +7,6 @@ SELECT TeamId AS team, ApiKeyHash AS api_key, TraceId AS trace_id,
        toUInt32(sum(OutputTokens)) AS output_tokens
 FROM agent_traces_by_key
 GROUP BY TeamId, ApiKeyHash, TraceId
+HAVING min(StartTs) >= now() - INTERVAL 1 DAY
 ORDER BY team, api_key, trace_id
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/help/unmatched_spans.sql

@@ -0,0 +1,18 @@
+SELECT
+    t.TraceId, t.SpanId, t.Model, t.LiteLLMRequestId,
+    t.InputTokens, t.OutputTokens
+FROM otel_traces AS t
+LEFT ANTI JOIN (
+    SELECT *
+    FROM spend_logs FINAL
+    WHERE start_time >= now() - INTERVAL 1 DAY
+) AS s
+    ON t.TeamId = s.team_id
+    AND ((t.UserId != '' AND t.UserId = s.user)
+        OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key))
+    AND t.LiteLLMRequestId != ''
+    AND (t.LiteLLMRequestId = s.response_id OR t.LiteLLMRequestId = s.request_id)
+WHERE t.Timestamp >= now() - INTERVAL 1 DAY
+    AND t.ObservationType = 'llm'
+ORDER BY t.Timestamp DESC, t.SpanId
+LIMIT 100

litellm-rust/crates/traces-clickhouse/query/spend_batch.sql

@@ -0,0 +1,27 @@
+SELECT * FROM (
+SELECT request_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
+       toUnixTimestamp64Milli(start_time) AS start_ms
+FROM (
+    SELECT *,
+           -- A chat request served through the Responses API returns the upstream `resp_` id to the
+           -- client but logs LiteLLM's managed `resp_<base64>` id, which embeds it.
+           if(startsWith(response_id, 'resp_'),
+              extract(tryBase64Decode(substring(response_id, 6)), 'response_id:([^;]+)'),
+              '') AS upstream_response_id
+    FROM spend_logs FINAL
+    WHERE start_time >= fromUnixTimestamp64Milli({start_ms:Int64})
+      AND start_time < fromUnixTimestamp64Milli({end_ms:Int64})
+      AND ({all_teams:UInt8} = 1
+           OR ({user_id:String} != '' AND user = {user_id:String})
+           OR has({team_ids:Array(String)}, team_id))
+)
+WHERE response_id IN {response_ids:Array(String)}
+   OR upstream_response_id IN {response_ids:Array(String)}
+   OR request_id IN {request_ids:Array(String)}
+   OR (trace_id != '' AND trace_id IN {trace_ids:Array(String)})
+ORDER BY start_time DESC
+)
+WHERE {has_cursor:UInt8} = 0
+   OR (team_id, start_ms, request_id) > ({after_team:String}, {after_ms:Int64}, {after_id:String})
+ORDER BY team_id, start_ms, request_id
+LIMIT {page_size:UInt32}

litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql

@@ -1,10 +1,21 @@
-SELECT request_id, response_id, team_id, api_key, user, spend,
+SELECT request_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
        toUnixTimestamp64Milli(start_time) AS start_ms
-FROM spend_logs FINAL
+FROM (
+    SELECT *,
+           -- A chat request served through the Responses API returns the upstream `resp_` id to the
+           -- client but logs LiteLLM's managed `resp_<base64>` id, which embeds it.
+           if(startsWith(response_id, 'resp_'),
+              extract(tryBase64Decode(substring(response_id, 6)), 'response_id:([^;]+)'),
+              '') AS upstream_response_id
+    FROM spend_logs FINAL
+    WHERE start_time >= fromUnixTimestamp64Milli({start_ms:Int64})
+      AND start_time < fromUnixTimestamp64Milli({end_ms:Int64})
+      AND ({all_teams:UInt8} = 1
+           OR ({user_id:String} != '' AND user = {user_id:String})
+           OR has({team_ids:Array(String)}, team_id))
+)
 WHERE response_id IN {response_ids:Array(String)}
-  AND start_time >= fromUnixTimestamp64Milli({start_ms:Int64})
-  AND start_time < fromUnixTimestamp64Milli({end_ms:Int64})
-  AND ({all_teams:UInt8} = 1
-       OR ({user_id:String} != '' AND user = {user_id:String})
-       OR has({team_ids:Array(String)}, team_id))
+   OR upstream_response_id IN {response_ids:Array(String)}
+   OR request_id IN {request_ids:Array(String)}
+   OR (trace_id != '' AND trace_id IN {trace_ids:Array(String)})
 ORDER BY start_time DESC

litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql

@@ -0,0 +1,31 @@
+SELECT * FROM (
+SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+       o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
+       o.Framework AS framework, o.StatusCode AS status,
+       substringUTF8(o.StatusMessage, 1, 128) AS status_message,
+       lengthUTF8(o.StatusMessage) > 128 AS error_truncated,
+       toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns,
+       o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model,
+       o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens,
+       o.LiteLLMRequestId AS litellm_request_id,
+       o.CallKeys AS call_keys, o.CallEvidence AS call_evidence,
+       -- Rows written before ToolCallId keep the call id only in their attributes.
+       if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
+          coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
+          AS tool_call_id,
+       o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
+FROM otel_traces AS o
+WHERE o.Timestamp >= fromUnixTimestamp64Milli({start_ms:Int64})
+  AND o.Timestamp < fromUnixTimestamp64Milli({end_ms:Int64})
+  AND ({all_teams:UInt8} = 1
+       OR ({user_id:String} != '' AND o.UserId = {user_id:String})
+       OR has({team_ids:Array(String)}, o.TeamId))
+  AND hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) IN {trace_refs:Array(String)}
+  AND o.EngineReceivedMs <= {snapshot_ms:UInt64}
+ORDER BY o.Timestamp, o.EngineReceivedMs, o.StatusMessage
+LIMIT 1 BY o.TeamId, o.ApiKeyHash, o.TraceId, o.SpanId
+
+)
+WHERE (team_id, api_key_hash, trace_id, span_id) > ({after_team:String}, {after_key:String}, {after_trace:String}, {after_span:String})
+ORDER BY team_id, api_key_hash, trace_id, span_id
+LIMIT {page_size:UInt32}

litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql

@@ -0,0 +1,24 @@
+SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+       o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
+       o.Framework AS framework, o.StatusCode AS status,
+       substringUTF8(o.StatusMessage, 1, 128) AS status_message,
+       lengthUTF8(o.StatusMessage) > 128 AS error_truncated,
+       toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns,
+       o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model,
+       o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens,
+       o.LiteLLMRequestId AS litellm_request_id,
+       o.CallKeys AS call_keys, o.CallEvidence AS call_evidence,
+       -- Rows written before ToolCallId keep the call id only in their attributes.
+       if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
+          coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
+          AS tool_call_id,
+       o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
+FROM otel_traces AS o
+WHERE o.Timestamp >= fromUnixTimestamp64Milli({start_ms:Int64})
+  AND o.Timestamp < fromUnixTimestamp64Milli({end_ms:Int64})
+  AND ({all_teams:UInt8} = 1
+       OR ({user_id:String} != '' AND o.UserId = {user_id:String})
+       OR has({team_ids:Array(String)}, o.TeamId))
+  AND hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) IN {trace_refs:Array(String)}
+ORDER BY o.Timestamp, o.EngineReceivedMs, o.StatusMessage
+LIMIT 1 BY o.TeamId, o.ApiKeyHash, o.TraceId, o.SpanId

litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql

@@ -0,0 +1,30 @@
+SELECT * FROM (
+SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+       o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
+       o.Framework AS framework, o.StatusCode AS status,
+       substringUTF8(o.StatusMessage, 1, 128) AS status_message,
+       lengthUTF8(o.StatusMessage) > 128 AS error_truncated,
+       toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns,
+       o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model,
+       o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens,
+       o.LiteLLMRequestId AS litellm_request_id,
+       o.CallKeys AS call_keys, o.CallEvidence AS call_evidence,
+       -- Rows written before ToolCallId keep the call id only in their attributes.
+       if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
+          coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
+          AS tool_call_id,
+       o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
+FROM otel_traces AS o
+WHERE o.TraceId = {trace_id:String}
+  AND ({all_teams:UInt8} = 1
+       OR ({user_id:String} != '' AND o.UserId = {user_id:String})
+       OR has({team_ids:Array(String)}, o.TeamId))
+  AND ({trace_ref:String} = '' OR
+       hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) = {trace_ref:String})
+  AND o.EngineReceivedMs <= {snapshot_ms:UInt64}
+ORDER BY o.Timestamp, o.EngineReceivedMs, o.StatusMessage
+LIMIT 1 BY o.SpanId
+)
+WHERE span_id > {after_span_id:String}
+ORDER BY span_id
+LIMIT {page_size:UInt32}

litellm-rust/crates/traces-clickhouse/query/trace_spans.sql

@@ -1,5 +1,5 @@
-SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
-       o.ObservationType AS type, o.AgentName AS agent,
+SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+       o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
        o.Framework AS framework, o.StatusCode AS status,
        substringUTF8(o.StatusMessage, 1, 128) AS status_message,
        lengthUTF8(o.StatusMessage) > 128 AS error_truncated,
@@ -7,6 +7,11 @@ SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name
        o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model,
        o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens,
        o.LiteLLMRequestId AS litellm_request_id,
+       o.CallKeys AS call_keys, o.CallEvidence AS call_evidence,
+       -- Rows written before ToolCallId keep the call id only in their attributes.
+       if(o.ToolCallId != '' OR o.ObservationType != 'tool', o.ToolCallId,
+          coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), ''))
+          AS tool_call_id,
        o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
 FROM otel_traces AS o
 WHERE o.TraceId = {trace_id:String}

litellm-rust/crates/traces-clickhouse/src/bin/export_schema.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "{}",
+        serde_json::to_string_pretty(&litellm_traces_clickhouse::wire_schema::schemas()).unwrap()
+    );
+}

litellm-rust/crates/traces-clickhouse/src/config.rs

@@ -5,14 +5,21 @@ use litellm_storage_clickhouse::Storage;
 pub struct Config {
     storage: Storage,
     retention_days: u32,
+    max_attribute_value_bytes: usize,
 }
 
 impl Config {
-    pub fn new(database: String, url: &str, retention_days: u32) -> Result<Self, Error> {
+    pub fn new(
+        database: String,
+        url: &str,
+        retention_days: u32,
+        max_attribute_value_bytes: usize,
+    ) -> Result<Self, Error> {
         super::schema_statements(&database, retention_days)?;
         Ok(Self {
             storage: Storage::new(database, url)?,
             retention_days,
+            max_attribute_value_bytes,
         })
     }
 
@@ -23,4 +30,9 @@ impl Config {
     pub fn retention_days(&self) -> u32 {
         self.retention_days
     }
+
+    /// Stored span attribute and payload values longer than this are truncated with a marker.
+    pub fn max_attribute_value_bytes(&self) -> usize {
+        self.max_attribute_value_bytes
+    }
 }

litellm-rust/crates/traces-clickhouse/src/error.rs

@@ -14,6 +14,8 @@ pub enum Error {
     InvalidResponse,
     #[error("ClickHouse insert exceeds the encoded size limit")]
     InsertTooLarge,
+    #[error("Trace exceeds the interactive read budget; use a filtered trace query")]
+    ReadTooLarge,
     #[error("ClickHouse schema setup failed with HTTP status {0}")]
     SchemaFailed(u16),
     #[error("ClickHouse schema setup transport failed")]
@@ -30,8 +32,27 @@ pub enum Error {
     ProvisionFailed(u16),
     #[error("ClickHouse reader provisioning transport failed")]
     ProvisionTransport,
+    #[error("Invalid {0} cursor")]
+    InvalidCursor(&'static str),
+    #[error("Multiple traces have this ID; provide trace_ref")]
+    AmbiguousTrace,
+    #[error("Trace changed while paging; refresh the trace to continue")]
+    TraceChanged,
+    #[error(transparent)]
+    Decode(#[from] litellm_traces::Error),
+    #[error("trace ingestion task failed")]
+    Task,
     #[error(transparent)]
     Storage(#[from] litellm_storage_clickhouse::Error),
     #[error(transparent)]
     Cached(#[from] std::sync::Arc<Error>),
 }
+
+impl From<litellm_traces_cache::Error> for Error {
+    fn from(error: litellm_traces_cache::Error) -> Self {
+        match error {
+            litellm_traces_cache::Error::Serialization(_) => Self::InvalidResponse,
+            litellm_traces_cache::Error::ReadTooLarge => Self::ReadTooLarge,
+        }
+    }
+}

litellm-rust/crates/traces-clickhouse/src/lib.rs

@@ -1,11 +1,28 @@
+macro_rules_attribute::attribute_alias! {
+    #[apply(wire_type)] =
+        #[derive(serde::Serialize, serde::Deserialize)]
+        #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))];
+    #[apply(response_type)] =
+        #[derive(serde::Serialize)]
+        #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))];
+    #[apply(request_type)] =
+        #[derive(serde::Deserialize)]
+        #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))];
+}
+
 mod config;
 mod error;
 mod insert;
 pub mod query;
 mod query_access;
+mod reads;
 mod schema;
+mod span_batches;
+mod span_row;
 mod sql;
 mod table;
+#[cfg(feature = "schema")]
+pub mod wire_schema;
 
 pub use config::Config;
 pub use error::Error;
@@ -14,8 +31,10 @@ pub use litellm_storage_clickhouse::{Connection, Parameter};
 pub use litellm_traces::{QueryScope, ReadQuery};
 pub use query::{QueryHelp, execute_read, query_help, query_sql};
 pub use query_access::QueryReaders;
+pub use reads::{get_span, get_span_error, get_trace, get_trace_page, list_traces};
 pub use schema::{
     NORMALIZED_FIELD_DEFINITIONS, NormalizedFieldDefinition, ensure_schema, schema_statements,
 };
+pub use span_row::span_rows;
 pub use sql::execute_named_read;
 pub use table::TraceTable;

litellm-rust/crates/traces-clickhouse/src/query.rs

@@ -6,6 +6,7 @@ use futures_util::{
     stream::{self, TryStreamExt},
 };
 use litellm_http::Client;
+use litellm_traces::query::guide::{Example, QueryGuide, Section};
 use serde::{Deserialize, Serialize, Serializer};
 use serde_json::Value;
 use strum::IntoEnumIterator;
@@ -39,21 +40,24 @@ struct MetadataRow {
     metadata: String,
 }
 
-#[derive(Deserialize)]
+#[macro_rules_attribute::apply(request_type)]
 struct AttributeRow {
     key: String,
 }
 
-#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)]
 #[serde(untagged)]
 enum PathPart {
     Key(String),
     Index(usize),
 }
 
-#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, strum::Display)]
+#[macro_rules_attribute::apply(response_type)]
+#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, strum::Display)]
 #[serde(rename_all = "lowercase")]
 #[strum(serialize_all = "lowercase")]
+#[cfg_attr(feature = "schema", schemars(rename = "MetadataValueType"))]
 enum JsonKind {
     Array,
     Boolean,
@@ -78,19 +82,23 @@ impl JsonKind {
     }
 }
 
-#[derive(Clone, Copy, Debug, Serialize, strum::Display)]
+#[macro_rules_attribute::apply(response_type)]
+#[derive(Clone, Copy, Debug, strum::Display)]
 enum MapValueType {
     String,
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryMetadataField"))]
 struct MetadataField {
     path: Vec<PathPart>,
     types: BTreeSet<JsonKind>,
     expression: String,
 }
 
-#[derive(Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryColumn"))]
 struct ColumnSchema {
     name: String,
     #[serde(rename = "type")]
@@ -99,7 +107,9 @@ struct ColumnSchema {
     details: BTreeMap<String, Value>,
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryTable"))]
 struct TableSchema {
     name: TraceTable,
     columns: Vec<ColumnSchema>,
@@ -114,6 +124,38 @@ enum Discovery<T> {
     Unavailable(String),
 }
 
+#[cfg(feature = "schema")]
+impl<T: schemars::JsonSchema> schemars::JsonSchema for Discovery<T> {
+    fn schema_name() -> std::borrow::Cow<'static, str> {
+        format!("Discovery{}", T::schema_name()).into()
+    }
+
+    fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::Schema {
+        let mut schema = T::json_schema(generator);
+        schema
+            .as_object_mut()
+            .unwrap()
+            .get_mut("properties")
+            .unwrap()
+            .as_object_mut()
+            .unwrap()
+            .insert(
+                "error".into(),
+                serde_json::json!({"type": ["string", "null"], "default": null}),
+            );
+        schema
+    }
+}
+
+#[cfg(feature = "schema")]
+pub(crate) fn help_schema() -> schemars::Schema {
+    schemars::generate::SchemaSettings::draft2020_12()
+        .for_serialize()
+        .with_transform(litellm_traces::schema::integer_bounds)
+        .into_generator()
+        .into_root_schema_for::<QueryHelp>()
+}
+
 impl<T: Serialize + Unobserved> Serialize for Discovery<T> {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
         #[derive(Serialize)]
@@ -133,7 +175,7 @@ impl<T: Serialize + Unobserved> Serialize for Discovery<T> {
     }
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
 struct MetadataSample {
     fields: Vec<MetadataField>,
     sampled_rows: usize,
@@ -152,7 +194,9 @@ impl Unobserved for MetadataSample {
     }
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryMetadata"))]
 struct MetadataCatalog {
     table: TraceTable,
     column: &'static str,
@@ -162,7 +206,9 @@ struct MetadataCatalog {
     scope: &'static str,
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryAttributeField"))]
 struct AttributeField {
     key: String,
     #[serde(rename = "type")]
@@ -170,7 +216,7 @@ struct AttributeField {
     expression: String,
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
 struct AttributeSample {
     fields: Vec<AttributeField>,
     truncated: bool,
@@ -185,7 +231,9 @@ impl Unobserved for AttributeSample {
     }
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryAttributes"))]
 struct AttributeCatalog {
     table: TraceTable,
     column: &'static str,
@@ -195,7 +243,9 @@ struct AttributeCatalog {
     scope: &'static str,
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryNormalizedField"))]
 struct NormalizedField {
     table: TraceTable,
     name: &'static str,
@@ -217,7 +267,9 @@ impl From<&NormalizedFieldDefinition> for NormalizedField {
     }
 }
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryRelationship"))]
 struct Relationship {
     left: &'static str,
     right: &'static str,
@@ -228,11 +280,13 @@ struct Relationship {
 const RELATIONSHIPS: [Relationship; 1] = [Relationship {
     left: "otel_traces.LiteLLMRequestId",
     right: "spend_logs.response_id",
-    additional_predicates: "otel_traces.TeamId = spend_logs.team_id AND (otel_traces.TeamId != '' OR (otel_traces.UserId != '' AND otel_traces.UserId = spend_logs.user) OR (otel_traces.ApiKeyHash != '' AND otel_traces.ApiKeyHash = spend_logs.api_key))",
-    meaning: "The normalized ID is the response ID, not request_id. Cached requests can share response_id; joins may return multiple spend rows",
+    additional_predicates: "otel_traces.TeamId = spend_logs.team_id AND ((otel_traces.UserId != '' AND otel_traces.UserId = spend_logs.user) OR (otel_traces.ApiKeyHash != '' AND otel_traces.ApiKeyHash = spend_logs.api_key))",
+    meaning: "LiteLLMRequestId contains the first normalized request or provider response ID. This relationship matches response IDs only; CallKeys retains all typed identifiers. Cached requests can share response_id; joins may return multiple spend rows",
 }];
 
-#[derive(Serialize)]
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceQueryHelp"))]
 pub struct QueryHelp {
     dialect: &'static str,
     access: &'static str,
@@ -242,8 +296,10 @@ pub struct QueryHelp {
     metadata: MetadataCatalog,
     attributes: Vec<AttributeCatalog>,
     relationships: &'static [Relationship],
-    examples: [guide::Example; 5],
-    gotchas: [String; 11],
+    #[cfg_attr(feature = "schema", schemars(with = "Vec<Example>"))]
+    examples: [Example; 12],
+    #[cfg_attr(feature = "schema", schemars(with = "Vec<String>"))]
+    gotchas: [String; 13],
     guide: String,
 }
 
@@ -423,13 +479,33 @@ pub async fn query_help(client: &Client, connection: &Connection) -> Result<Quer
         attributes: &attributes,
         limits: &READER_LIMITS,
     };
+    let bodies = guide.sections()?;
+    let sections = [
+        "Live ClickHouse schema",
+        "Normalized span fields",
+        "Observed LLM call metadata",
+        "Observed span and resource attributes",
+    ]
+    .into_iter()
+    .zip(&bodies)
+    .map(|(title, body)| Section { title, body })
+    .collect::<Vec<_>>();
+    let examples = guide.examples()?;
+    let gotchas = guide.gotchas()?;
+    let rendered = QueryGuide {
+        sections: &sections,
+        examples: &examples,
+        gotchas: &gotchas,
+    }
+    .render()
+    .map_err(|_| Error::InvalidResponse)?;
     Ok(QueryHelp {
         dialect: "ClickHouse SQL",
         access: "Request-log visibility enforced by ClickHouse row policies; proxy admins see all rows, users see their own rows and permitted teams",
         response: "ClickHouse JSON envelope: meta, data, rows, statistics; 64-bit integers may be strings",
-        examples: guide.examples()?,
-        gotchas: guide.gotchas()?,
-        guide: guide::render(&guide)?,
+        examples,
+        gotchas,
+        guide: rendered,
         normalized_fields: NORMALIZED_FIELD_DEFINITIONS
             .iter()
             .map(NormalizedField::from)
@@ -447,6 +523,33 @@ mod tests {
     use rstest::rstest;
     use serde_json::json;
 
+    #[cfg(feature = "schema")]
+    #[rstest]
+    #[case::observed(false)]
+    #[case::unavailable(true)]
+    fn discovery_serialization_matches_its_schema(#[case] unavailable: bool) {
+        let discovery = if unavailable {
+            Discovery::Unavailable("discovery failed".into())
+        } else {
+            Discovery::Observed(MetadataSample::unobserved())
+        };
+        let catalog = MetadataCatalog {
+            table: TraceTable::SpendLogs,
+            column: "metadata",
+            discovery,
+            sample_sql: METADATA_SQL,
+            scope: METADATA_SCOPE,
+        };
+        let schema = schemars::generate::SchemaSettings::draft2020_12()
+            .for_serialize()
+            .into_generator()
+            .into_root_schema_for::<MetadataCatalog>();
+        let serialized = serde_json::to_value(&catalog).unwrap();
+        assert!(jsonschema::is_valid(schema.as_value(), &serialized));
+        assert_eq!(serialized.get("error").is_some(), unavailable);
+        assert!(serialized["fields"].is_array());
+    }
+
     #[rstest]
     fn metadata_discovery_preserves_mixed_types_and_reports_invalid_rows() {
         let sample = [

litellm-rust/crates/traces-clickhouse/src/query/guide.rs

@@ -1,11 +1,15 @@
 use askama::Template;
-use serde::Serialize;
+use litellm_traces::query::guide::Example;
 
 use super::{AttributeCatalog, Discovery, MetadataCatalog, TableSchema};
 use crate::{Error, NormalizedFieldDefinition, query_access::ReaderLimits};
 
 #[derive(Template)]
 #[template(path = "query_help.jinja", escape = "none", blocks = [
+    "live_schema",
+    "normalized_fields",
+    "metadata",
+    "attributes",
     "recent_spans_name",
     "recent_spans_sql",
     "custom_metadata_name",
@@ -16,6 +20,22 @@ use crate::{Error, NormalizedFieldDefinition, query_access::ReaderLimits};
     "correlated_calls_sql",
     "discover_keys_name",
     "discover_keys_sql",
+    "recent_spend_name",
+    "recent_spend_sql",
+    "model_spend_name",
+    "model_spend_sql",
+    "trace_spend_name",
+    "trace_spend_sql",
+    "unmatched_spans_name",
+    "unmatched_spans_sql",
+    "trace_summary_name",
+    "trace_summary_sql",
+    "failed_spans_name",
+    "failed_spans_sql",
+    "metadata_filter_name",
+    "metadata_filter_sql",
+    "missing_spend",
+    "partial_spend",
     "time_window",
     "reader_limits",
     "reader_profile",
@@ -36,14 +56,17 @@ pub(super) struct QueryGuide<'a> {
     pub limits: &'a ReaderLimits,
 }
 
-#[derive(Serialize)]
-pub(super) struct Example {
-    name: String,
-    sql: String,
-}
-
 impl QueryGuide<'_> {
-    pub fn examples(&self) -> Result<[Example; 5], Error> {
+    pub fn sections(&self) -> Result<[String; 4], Error> {
+        Ok([
+            render(&self.as_live_schema())?,
+            render(&self.as_normalized_fields())?,
+            render(&self.as_metadata())?,
+            render(&self.as_attributes())?,
+        ])
+    }
+
+    pub fn examples(&self) -> Result<[Example; 12], Error> {
         Ok([
             Example {
                 name: render(&self.as_recent_spans_name())?,
@@ -65,10 +88,38 @@ impl QueryGuide<'_> {
                 name: render(&self.as_discover_keys_name())?,
                 sql: render(&self.as_discover_keys_sql())?,
             },
+            Example {
+                name: render(&self.as_recent_spend_name())?,
+                sql: render(&self.as_recent_spend_sql())?,
+            },
+            Example {
+                name: render(&self.as_model_spend_name())?,
+                sql: render(&self.as_model_spend_sql())?,
+            },
+            Example {
+                name: render(&self.as_trace_spend_name())?,
+                sql: render(&self.as_trace_spend_sql())?,
+            },
+            Example {
+                name: render(&self.as_unmatched_spans_name())?,
+                sql: render(&self.as_unmatched_spans_sql())?,
+            },
+            Example {
+                name: render(&self.as_trace_summary_name())?,
+                sql: render(&self.as_trace_summary_sql())?,
+            },
+            Example {
+                name: render(&self.as_failed_spans_name())?,
+                sql: render(&self.as_failed_spans_sql())?,
+            },
+            Example {
+                name: render(&self.as_metadata_filter_name())?,
+                sql: render(&self.as_metadata_filter_sql())?,
+            },
         ])
     }
 
-    pub fn gotchas(&self) -> Result<[String; 11], Error> {
+    pub fn gotchas(&self) -> Result<[String; 13], Error> {
         Ok([
             render(&self.as_time_window())?,
             render(&self.as_reader_limits())?,
@@ -79,6 +130,8 @@ impl QueryGuide<'_> {
             render(&self.as_literal_keys())?,
             render(&self.as_time_units())?,
             render(&self.as_spend_totals())?,
+            render(&self.as_missing_spend())?,
+            render(&self.as_partial_spend())?,
             render(&self.as_trace_rollups())?,
             render(&self.as_sampling())?,
         ])

litellm-rust/crates/traces-clickhouse/src/query/lens.rs

@@ -1,27 +1,72 @@
 use litellm_storage_clickhouse::Query;
-use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Deserialize, Serialize)]
+pub const LENS_QUERIES: [litellm_traces::ReadQuery; 5] = [
+    litellm_traces::ReadQuery::Availability,
+    litellm_traces::ReadQuery::Agents,
+    litellm_traces::ReadQuery::Sample,
+    litellm_traces::ReadQuery::Content,
+    litellm_traces::ReadQuery::Evidence,
+];
+
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(rename_all = "lowercase")]
+pub enum ExecutionSource {
+    Traces,
+    Requests,
+    Both,
+}
+
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(rename_all = "lowercase")]
+pub enum ContentSource {
+    Traces,
+    Requests,
+}
+
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[cfg_attr(feature = "schema", schemars(deny_unknown_fields))]
 pub struct LensAccessParams {
-    #[serde(deserialize_with = "super::number::deserialize")]
-    pub all_teams: u8,
+    #[serde(
+        deserialize_with = "super::number::boolean",
+        serialize_with = "litellm_traces::wire::serialize_flag"
+    )]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "litellm_traces::schema::flag")
+    )]
+    pub all_teams: bool,
     pub team: String,
     pub key_hash: String,
 }
 
 pub struct LensAvailability;
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(deny_unknown_fields)]
 pub struct LensAvailabilityParams {
     #[serde(flatten)]
     pub access: LensAccessParams,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[cfg_attr(feature = "schema", schemars(rename = "ActivityAvailability"))]
 pub struct LensAvailabilityRow {
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(default, deserialize_with = "super::number::flag")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::boolean_flag")
+    )]
     pub traces: u8,
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(default, deserialize_with = "super::number::flag")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::boolean_flag")
+    )]
     pub requests: u8,
 }
 
@@ -34,13 +79,17 @@ impl Query for LensAvailability {
 
 pub struct LensAgents;
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(deny_unknown_fields)]
 pub struct LensAgentsParams {
     #[serde(flatten)]
     pub access: LensAccessParams,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[cfg_attr(feature = "schema", schemars(rename = "AgentRow"))]
 pub struct LensAgentsRow {
     pub agent_name: String,
 }
@@ -54,11 +103,13 @@ impl Query for LensAgents {
 
 pub struct LensSample;
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(deny_unknown_fields)]
 pub struct LensSampleParams {
     #[serde(flatten)]
     pub access: LensAccessParams,
-    pub source: String,
+    pub source: ExecutionSource,
     #[serde(deserialize_with = "super::number::deserialize")]
     pub start: u64,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -71,9 +122,14 @@ pub struct LensSampleParams {
     pub execution_ids: Vec<String>,
     #[serde(deserialize_with = "super::number::deserialize")]
     pub sample_cap: u64,
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(deserialize_with = "super::number::percent")]
+    #[cfg_attr(feature = "schema", schemars(range(min = 0, max = 100)))]
     pub sample_percent: f64,
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(deserialize_with = "super::number::flag")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "litellm_traces::schema::flag")
+    )]
     pub preview: u8,
     pub after: String,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -82,26 +138,49 @@ pub struct LensSampleParams {
     pub offset: u64,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[cfg_attr(feature = "schema", schemars(rename = "ExecutionRow"))]
 pub struct LensSampleRow {
-    pub source: String,
+    pub source: ContentSource,
     pub trace_id: String,
     pub team_id: String,
+    #[serde(default)]
     pub trace_ref: String,
     pub name: String,
     pub start_time: String,
     #[serde(deserialize_with = "super::number::deserialize")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::u64_number")
+    )]
     pub span_count: u64,
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(deserialize_with = "super::number::flag")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::flag_number")
+    )]
     pub root_seen: u8,
+    #[serde(default)]
     pub service: String,
+    #[serde(default)]
     pub attributes: Vec<(String, String)>,
     #[serde(deserialize_with = "super::number::deserialize")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::u64_number")
+    )]
     pub eligible: u64,
     #[serde(deserialize_with = "super::number::deserialize")]
+    #[cfg_attr(feature = "schema", schemars(skip))]
     pub position: u64,
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(default, deserialize_with = "super::number::deserialize")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::selected")
+    )]
     pub selected: f64,
+    #[serde(default)]
     pub selection_key: String,
 }
 
@@ -114,11 +193,13 @@ impl Query for LensSample {
 
 pub struct LensContent;
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(deny_unknown_fields)]
 pub struct LensContentParams {
     #[serde(flatten)]
     pub access: LensAccessParams,
-    pub source: String,
+    pub source: ContentSource,
     pub id: String,
     pub record_team: String,
     pub trace_ref: String,
@@ -127,14 +208,20 @@ pub struct LensContentParams {
     pub offset: u32,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[cfg_attr(feature = "schema", schemars(rename = "PartRow"))]
 pub struct LensContentRow {
     pub span_id: String,
     pub parent_span_id: String,
     pub name: String,
     pub kind: String,
     pub content: String,
-    #[serde(deserialize_with = "super::number::deserialize")]
+    #[serde(deserialize_with = "super::number::flag")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::flag_number")
+    )]
     pub truncated: u8,
 }
 
@@ -147,11 +234,13 @@ impl Query for LensContent {
 
 pub struct LensEvidence;
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[serde(deny_unknown_fields)]
 pub struct LensEvidenceParams {
     #[serde(flatten)]
     pub access: LensAccessParams,
-    pub source: String,
+    pub source: ContentSource,
     pub id: String,
     pub record_team: String,
     pub trace_ref: String,
@@ -159,9 +248,15 @@ pub struct LensEvidenceParams {
     pub quote: String,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[macro_rules_attribute::apply(wire_type)]
+#[derive(Debug)]
+#[cfg_attr(feature = "schema", schemars(rename = "CountRow"))]
 pub struct LensEvidenceRow {
     #[serde(deserialize_with = "super::number::deserialize")]
+    #[cfg_attr(
+        feature = "schema",
+        schemars(schema_with = "crate::wire_schema::u64_number")
+    )]
     pub count: u64,
 }
 

litellm-rust/crates/traces-clickhouse/src/query/named.rs

@@ -42,7 +42,8 @@ struct ListTracesRowEncoding {
     pub name: String,
     pub service: String,
     pub input_preview: String,
-    pub status: String,
+    #[serde(serialize_with = "litellm_traces::wire::serialize_status")]
+    pub status: litellm_traces::SpanStatus,
     #[serde(deserialize_with = "super::number::deserialize")]
     pub start_ms: i64,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -79,18 +80,30 @@ pub use contracts::TraceSpansParams;
 #[derive(Deserialize, Serialize)]
 #[serde(remote = "contracts::TraceSpansRow")]
 struct TraceSpansRowEncoding {
+    #[serde(default)]
+    pub trace_id: String,
     pub span_id: String,
     pub parent_span_id: String,
     pub name: String,
     #[serde(rename = "type")]
-    pub kind: String,
+    pub kind: litellm_traces::ObservationType,
+    #[serde(
+        default,
+        deserialize_with = "super::number::boolean",
+        serialize_with = "litellm_traces::wire::serialize_flag"
+    )]
+    pub wrapper_candidate: bool,
     pub agent: String,
     #[serde(default)]
     pub framework: String,
-    pub status: String,
+    #[serde(serialize_with = "litellm_traces::wire::serialize_status")]
+    pub status: litellm_traces::SpanStatus,
     pub status_message: String,
-    #[serde(deserialize_with = "super::number::deserialize")]
-    pub error_truncated: u8,
+    #[serde(
+        deserialize_with = "super::number::boolean",
+        serialize_with = "litellm_traces::wire::serialize_flag"
+    )]
+    pub error_truncated: bool,
     #[serde(deserialize_with = "super::number::deserialize")]
     pub start_ns: i64,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -103,6 +116,16 @@ struct TraceSpansRowEncoding {
     #[serde(deserialize_with = "super::number::deserialize")]
     pub output_tokens: u32,
     pub litellm_request_id: String,
+    #[serde(default)]
+    pub call_keys: Vec<litellm_traces::CallKey>,
+    #[serde(
+        default,
+        deserialize_with = "litellm_traces::wire::evidence",
+        serialize_with = "litellm_traces::wire::serialize_evidence"
+    )]
+    pub call_evidence: Option<litellm_traces::CallEvidenceKind>,
+    #[serde(default)]
+    pub tool_call_id: String,
     pub team_id: String,
     pub api_key_hash: String,
     pub user_id: String,
@@ -158,6 +181,8 @@ struct SpendByResponseIdsParamsEncoding {
     #[serde(flatten)]
     pub access: contracts::ReadAccessParams,
     pub response_ids: Vec<String>,
+    pub request_ids: Vec<String>,
+    pub trace_ids: Vec<String>,
     #[serde(deserialize_with = "super::number::deserialize")]
     pub start_ms: i64,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -180,11 +205,14 @@ impl From<contracts::SpendByResponseIdsParams> for SpendByResponseIdsParams {
 struct SpendByResponseIdsRowEncoding {
     pub request_id: String,
     pub response_id: String,
+    pub upstream_response_id: String,
+    pub trace_id: String,
+    pub span_id: String,
     pub team_id: String,
     pub api_key: String,
     pub user: String,
-    #[serde(deserialize_with = "super::number::deserialize")]
-    pub spend: f64,
+    #[serde(deserialize_with = "super::number::optional_finite")]
+    pub spend: Option<f64>,
     #[serde(deserialize_with = "super::number::deserialize")]
     pub start_ms: i64,
 }
@@ -203,6 +231,38 @@ impl Query for ListTraces {
     const SQL: &'static str = include_str!("../../query/list_traces.sql");
 }
 
+#[derive(Deserialize, Serialize)]
+#[serde(remote = "contracts::TracePageSpansParams")]
+struct TracePageSpansParamsEncoding {
+    #[serde(flatten)]
+    pub access: contracts::ReadAccessParams,
+    pub trace_refs: Vec<String>,
+    #[serde(deserialize_with = "super::number::deserialize")]
+    pub start_ms: i64,
+    #[serde(deserialize_with = "super::number::deserialize")]
+    pub end_ms: i64,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct TracePageSpansParams(
+    #[serde(with = "TracePageSpansParamsEncoding")] pub contracts::TracePageSpansParams,
+);
+
+impl From<contracts::TracePageSpansParams> for TracePageSpansParams {
+    fn from(value: contracts::TracePageSpansParams) -> Self {
+        Self(value)
+    }
+}
+
+pub struct TracePageSpans;
+
+impl Query for TracePageSpans {
+    type Params = TracePageSpansParams;
+    type Row = TraceSpansRow;
+
+    const SQL: &'static str = include_str!("../../query/trace_page_spans.sql");
+}
+
 pub struct TraceSpans;
 
 impl Query for TraceSpans {
@@ -279,11 +339,11 @@ mod tests {
     #[case::quoted(true)]
     fn rows_decode_into_neutral_contracts(#[case] quoted: bool) {
         round_trip::<ListTracesRow>(
-            json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "ok", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["claude-agent-sdk"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}),
+            json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["claude-agent-sdk"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}),
             quoted,
         );
         round_trip::<TraceSpansRow>(
-            json!({"span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "agent": "agent", "framework": "claude-agent-sdk", "status": "error", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
+            json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
             quoted,
         );
         round_trip::<SpanDetailRow>(
@@ -295,7 +355,7 @@ mod tests {
             quoted,
         );
         round_trip::<SpendByResponseIdsRow>(
-            json!({"request_id": "request", "response_id": "response", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
+            json!({"request_id": "request", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
             quoted,
         );
     }
@@ -313,8 +373,36 @@ mod tests {
             quoted,
         );
         round_trip::<SpendByResponseIdsParams>(
-            json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "start_ms": -1, "end_ms": 10}),
+            json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
             quoted,
         );
     }
+    #[rstest]
+    #[case::unknown(json!(null), None)]
+    #[case::free(json!(0), Some(0.0))]
+    #[case::paid(json!("0.125"), Some(0.125))]
+    fn spend_rows_preserve_unknown_and_known_cost(
+        #[case] cost: serde_json::Value,
+        #[case] expected: Option<f64>,
+    ) {
+        let row: SpendByResponseIdsRow = serde_json::from_value(json!({
+            "request_id": "request", "response_id": "response", "upstream_response_id": "",
+            "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key",
+            "user": "user", "spend": cost, "start_ms": 0
+        }))
+        .unwrap();
+        assert_eq!(row.0.spend, expected);
+    }
+    #[rstest]
+    #[case::nan(json!("NaN"))]
+    #[case::infinity(json!("1e999"))]
+    #[case::boolean(json!(true))]
+    fn spend_rows_reject_invalid_cost(#[case] cost: serde_json::Value) {
+        let row = serde_json::from_value::<SpendByResponseIdsRow>(json!({
+            "request_id": "request", "response_id": "response", "upstream_response_id": "",
+            "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key",
+            "user": "user", "spend": cost, "start_ms": 0
+        }));
+        assert!(row.is_err());
+    }
 }

litellm-rust/crates/traces-clickhouse/src/query/number.rs

@@ -18,11 +18,95 @@ where
     .map_err(serde::de::Error::custom)
 }
 
+pub(super) fn optional_finite<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<f64>, D::Error> {
+    let value = Option::<serde_json::Value>::deserialize(deserializer)?;
+    let Some(value) = value else {
+        return Ok(None);
+    };
+    let number: f64 = deserialize(value).map_err(serde::de::Error::custom)?;
+    if number.is_finite() {
+        Ok(Some(number))
+    } else {
+        Err(serde::de::Error::custom("expected finite spend"))
+    }
+}
+
+pub(super) fn flag<'de, D: Deserializer<'de>>(deserializer: D) -> Result<u8, D::Error> {
+    match deserialize(deserializer)? {
+        value @ 0..=1 => Ok(value),
+        _ => Err(serde::de::Error::custom("expected 0 or 1")),
+    }
+}
+
+pub(super) fn percent<'de, D: Deserializer<'de>>(deserializer: D) -> Result<f64, D::Error> {
+    let value: f64 = deserialize(deserializer)?;
+    if value.is_finite() && (0.0..=100.0).contains(&value) {
+        Ok(value)
+    } else {
+        Err(serde::de::Error::custom(
+            "expected a finite percentage between 0 and 100",
+        ))
+    }
+}
+
+pub(super) fn boolean<'de, D: Deserializer<'de>>(deserializer: D) -> Result<bool, D::Error> {
+    flag(deserializer).map(|value| value == 1)
+}
+
 #[cfg(test)]
 mod tests {
     use crate::query::named::SpanErrorRow;
     use rstest::rstest;
 
+    #[rstest]
+    #[case::flag_zero(serde_json::json!(0), true)]
+    #[case::flag_one(serde_json::json!("1"), true)]
+    #[case::invalid_flag(serde_json::json!(2), false)]
+    fn access_rejects_non_boolean_flags(#[case] value: serde_json::Value, #[case] valid: bool) {
+        let parameters = serde_json::json!({"all_teams": value, "team": "team", "key_hash": ""});
+        assert_eq!(
+            serde_json::from_value::<crate::query::lens::LensAccessParams>(parameters).is_ok(),
+            valid
+        );
+    }
+
+    #[rstest]
+    #[case::zero(serde_json::json!(0), true)]
+    #[case::hundred(serde_json::json!("100"), true)]
+    #[case::negative(serde_json::json!(-0.1), false)]
+    #[case::too_large(serde_json::json!(100.1), false)]
+    #[case::nan(serde_json::json!("NaN"), false)]
+    fn sampling_rejects_invalid_percentages(#[case] value: serde_json::Value, #[case] valid: bool) {
+        let parameters = serde_json::json!({
+            "all_teams": 0, "team": "team", "key_hash": "", "source": "both", "start": 0, "end": 1,
+            "agent_name": "", "service": "", "filter_keys": [], "filter_values": [], "selected_team": "",
+            "execution_ids": [], "sample_cap": 0, "sample_percent": value, "preview": 0, "after": "",
+            "limit": 10, "offset": 0
+        });
+        assert_eq!(
+            serde_json::from_value::<crate::query::lens::LensSampleParams>(parameters).is_ok(),
+            valid
+        );
+    }
+
+    #[rstest]
+    #[case::trace("traces", true)]
+    #[case::request("requests", true)]
+    #[case::both("both", false)]
+    #[case::unknown("unknown", false)]
+    fn content_rejects_unsupported_sources(#[case] source: &str, #[case] valid: bool) {
+        let parameters = serde_json::json!({
+            "all_teams": 0, "team": "team", "key_hash": "", "source": source, "id": "id",
+            "record_team": "team", "trace_ref": "", "cursor": "", "offset": 0
+        });
+        assert_eq!(
+            serde_json::from_value::<crate::query::lens::LensContentParams>(parameters).is_ok(),
+            valid
+        );
+    }
+
     #[rstest]
     #[case::quoted_max(serde_json::json!(u64::MAX.to_string()), Some(u64::MAX))]
     #[case::unquoted_max(serde_json::json!(u64::MAX), Some(u64::MAX))]

litellm-rust/crates/traces-clickhouse/src/reads.rs

@@ -0,0 +1,536 @@
+//! Scoped trace reads: the trace list, one trace resolved with its spend, and span payloads.
+
+use std::sync::LazyLock;
+use std::time::Duration;
+
+use base64::{Engine, engine::general_purpose::URL_SAFE};
+use futures_util::{StreamExt, TryStreamExt, stream};
+use itertools::Itertools;
+use litellm_http::Client;
+use litellm_storage_clickhouse::{Query, fetch};
+use litellm_traces::{
+    SpanDetail, SpanErrorPage, SpendLookup, Trace, TracePage, listed_summary,
+    query::named as contracts, resolve_trace, to_ui_content,
+};
+use litellm_traces_cache::{SnapshotCache, SnapshotKey};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    Connection, Error,
+    query::named::{
+        ListTracesParams, ListTracesRow, ReadAccessParams, SpanDetail as SpanDetailQuery,
+        SpanDetailParams, SpanError, SpanErrorParams, SpendByResponseIdsParams, TraceIdentity,
+        TraceIdentityParams, TracePageSpansParams, TraceSpansParams,
+    },
+};
+
+struct RunCandidates;
+
+impl Query for RunCandidates {
+    type Params = ListTracesParams;
+    type Row = ListTracesRow;
+    const SQL: &'static str = concat!(
+        "SELECT * EXCEPT (request_ids), [] AS request_ids FROM (",
+        include_str!("../query/list_traces.sql"),
+        ") ORDER BY start_ms DESC, trace_ref DESC"
+    );
+}
+
+// Cursor pages share a bounded snapshot so advancing does not resolve the whole graph again.
+static TRACE_SNAPSHOTS: LazyLock<SnapshotCache> = LazyLock::new(|| {
+    SnapshotCache::new(
+        crate::span_batches::MAX_GRAPH_BYTES,
+        Duration::from_secs(120),
+    )
+});
+
+const NANOS_PER_MS: i64 = 1_000_000;
+const SPEND_WINDOW_MS: i64 = 30 * 60 * 1000;
+const SPEND_CONCURRENCY: usize = 4;
+
+fn encode_cursor<T: Serialize>(position: &T) -> String {
+    URL_SAFE.encode(serde_json::to_vec(position).unwrap_or_default())
+}
+
+fn decode_cursor<T: for<'de> Deserialize<'de>>(
+    cursor: &str,
+    kind: &'static str,
+) -> Result<T, Error> {
+    URL_SAFE
+        .decode(cursor)
+        .ok()
+        .and_then(|json| serde_json::from_slice(&json).ok())
+        .ok_or(Error::InvalidCursor(kind))
+}
+
+fn trace_position(cursor: Option<&str>) -> Result<(i64, String), Error> {
+    let Some(cursor) = cursor.filter(|cursor| !cursor.is_empty()) else {
+        return Ok((0, String::new()));
+    };
+    match decode_cursor::<(i64, String)>(cursor, "trace")? {
+        (start_ms, trace_ref) if start_ms > 0 && !trace_ref.is_empty() => Ok((start_ms, trace_ref)),
+        _ => Err(Error::InvalidCursor("trace")),
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+struct ErrorPosition {
+    offset: u64,
+    version: String,
+}
+
+fn error_position(cursor: Option<&str>) -> Result<Option<ErrorPosition>, Error> {
+    let Some(cursor) = cursor else {
+        return Ok(None);
+    };
+    let position = decode_cursor::<ErrorPosition>(cursor, "diagnostic")?;
+    let valid_version = position.version.len() == 64
+        && position
+            .version
+            .bytes()
+            .all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte));
+    if i64::try_from(position.offset).is_err() || !valid_version {
+        return Err(Error::InvalidCursor("diagnostic"));
+    }
+    Ok(Some(position))
+}
+
+/// The stored run a trace id names for this caller; ids can repeat across tenants and runs.
+async fn reference(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    trace_id: &str,
+    trace_ref: &str,
+) -> Result<Option<String>, Error> {
+    if !trace_ref.is_empty() {
+        return Ok(Some(trace_ref.to_owned()));
+    }
+    let params = TraceIdentityParams {
+        access: access.clone(),
+        trace_id: trace_id.to_owned(),
+    };
+    let mut identities = fetch::<TraceIdentity>(client, connection, &params).await?;
+    if identities.len() > 1 {
+        return Err(Error::AmbiguousTrace);
+    }
+    Ok(identities.pop().map(|identity| identity.trace_ref))
+}
+
+/// Spend records behind the spans' calls. A failed lookup leaves cost unknown instead of failing
+/// the read.
+async fn spend(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    rows: &[contracts::TraceSpansRow],
+) -> Vec<contracts::SpendByResponseIdsRow> {
+    let lookup = SpendLookup::new(rows);
+    let (Some(start_ns), Some(end_ns)) = (
+        rows.iter().map(|row| row.start_ns).min(),
+        rows.iter()
+            .map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns))
+            .max(),
+    ) else {
+        return Vec::new();
+    };
+    if lookup.is_empty() {
+        return Vec::new();
+    }
+    let params = SpendByResponseIdsParams::from(contracts::SpendByResponseIdsParams {
+        access: access.clone(),
+        response_ids: lookup.response_ids,
+        request_ids: lookup.request_ids,
+        trace_ids: lookup.trace_ids,
+        start_ms: start_ns.div_euclid(NANOS_PER_MS) - SPEND_WINDOW_MS,
+        end_ms: end_ns.div_euclid(NANOS_PER_MS) + SPEND_WINDOW_MS,
+    });
+    match crate::span_batches::read_spend(client, connection, params).await {
+        Ok(rows) => rows,
+        Err(error) => {
+            tracing::warn!(%error, "trace spend lookup unavailable");
+            Vec::new()
+        }
+    }
+}
+
+pub async fn list_traces(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    start_ms: i64,
+    end_ms: i64,
+    cursor: Option<&str>,
+    limit: u32,
+) -> Result<TracePage, Error> {
+    if limit == 0 {
+        return Err(Error::InvalidParameters);
+    }
+    let (cursor_ms, cursor_trace_id) = trace_position(cursor)?;
+    let mut params = ListTracesParams::from(contracts::ListTracesParams {
+        access: access.clone(),
+        start_ms,
+        end_ms,
+        cursor_ms,
+        cursor_trace_id,
+        limit: limit.min(500),
+    });
+    let page: Vec<contracts::ListTracesRow> = loop {
+        match fetch::<RunCandidates>(client, connection, &params).await {
+            Err(litellm_storage_clickhouse::Error::ResponseTooLarge) if params.0.limit > 1 => {
+                params.0.limit /= 2;
+            }
+            Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => {
+                return Err(Error::ReadTooLarge);
+            }
+            result => break result?.into_iter().map(|row| row.0).collect(),
+        }
+    };
+    let next_cursor = page
+        .last()
+        .filter(|_| page.len() == params.0.limit as usize)
+        .map(|last| encode_cursor(&(last.start_ms, &last.trace_ref)));
+    let data = stream::iter(page.chunks(16))
+        .then(|batch| list_summaries(client, connection, access, batch))
+        .try_collect::<Vec<_>>()
+        .await?
+        .into_iter()
+        .flatten()
+        .collect();
+    Ok(TracePage { data, next_cursor })
+}
+
+async fn list_summaries(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    runs: &[contracts::ListTracesRow],
+) -> Result<Vec<litellm_traces::TraceSummary>, Error> {
+    let (Some(start_ms), Some(end_ms)) = (
+        runs.iter().map(|row| row.start_ms).min(),
+        runs.iter()
+            .map(|row| row.start_ms.saturating_add(row.duration_ms))
+            .max(),
+    ) else {
+        return Ok(Vec::new());
+    };
+    let params = TracePageSpansParams::from(contracts::TracePageSpansParams {
+        access: access.clone(),
+        trace_refs: runs.iter().map(|row| row.trace_ref.clone()).collect(),
+        start_ms,
+        end_ms: end_ms.saturating_add(1),
+    });
+    let spans = match crate::span_batches::read_list_spans(client, connection, params).await {
+        Ok(spans) => spans,
+        Err(Error::ReadTooLarge) => {
+            return stream::iter(runs)
+                .then(|row| async move {
+                    match get_trace(client, connection, access, &row.trace_id, &row.trace_ref).await
+                    {
+                        Ok(trace) => {
+                            Ok(trace.map_or_else(|| listed_summary(row), |trace| trace.summary))
+                        }
+                        Err(Error::ReadTooLarge) => Ok(listed_summary(row)),
+                        Err(error) => Err(error),
+                    }
+                })
+                .try_collect()
+                .await;
+        }
+        Err(error) => return Err(error),
+    };
+    let by_trace = spans.into_iter().into_group_map_by(|span| {
+        (
+            span.team_id.clone(),
+            span.api_key_hash.clone(),
+            span.trace_id.clone(),
+        )
+    });
+    let summaries = runs
+        .iter()
+        .map(|row| {
+            let spans = by_trace
+                .get(&(
+                    row.team_id.clone(),
+                    row.api_key_hash.clone(),
+                    row.trace_id.clone(),
+                ))
+                .map(Vec::as_slice)
+                .unwrap_or_default();
+            async move {
+                let spend_rows = spend(client, connection, access, spans).await;
+                resolve_trace(&row.trace_id, &row.trace_ref, spans, &spend_rows)
+                    .map_or_else(|| listed_summary(row), |trace| trace.summary)
+            }
+        })
+        .collect::<Vec<_>>();
+    Ok(stream::iter(summaries)
+        .buffered(SPEND_CONCURRENCY)
+        .collect()
+        .await)
+}
+
+pub async fn get_trace(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    trace_id: &str,
+    trace_ref: &str,
+) -> Result<Option<Trace>, Error> {
+    let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else {
+        return Ok(None);
+    };
+    let params = TraceSpansParams {
+        access: access.clone(),
+        trace_id: trace_id.to_owned(),
+        trace_ref: trace_ref.clone(),
+    };
+    let rows = crate::span_batches::read_spans(client, connection, params, u64::MAX).await?;
+    if rows.is_empty() {
+        return Ok(None);
+    }
+    let spend_rows = spend(client, connection, access, &rows).await;
+    Ok(resolve_trace(trace_id, &trace_ref, &rows, &spend_rows))
+}
+
+#[derive(Deserialize, Serialize)]
+struct SpanPosition {
+    trace_ref: String,
+    snapshot_ms: u64,
+    offset: usize,
+    version: String,
+}
+
+pub async fn get_trace_page(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    trace_id: &str,
+    trace_ref: &str,
+    cursor: Option<&str>,
+    page_size: u32,
+) -> Result<Option<Trace>, Error> {
+    if !(1..=500).contains(&page_size) {
+        return Err(Error::InvalidParameters);
+    }
+    let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else {
+        return Ok(None);
+    };
+    let position = match cursor {
+        Some(cursor) => {
+            let position: SpanPosition = decode_cursor(cursor, "span")?;
+            if position.trace_ref != trace_ref || position.snapshot_ms == 0 {
+                return Err(Error::InvalidCursor("span"));
+            }
+            position
+        }
+        None => SpanPosition {
+            trace_ref: trace_ref.clone(),
+            snapshot_ms: (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000)
+                as u64,
+            offset: 0,
+            version: String::new(),
+        },
+    };
+    let key = SnapshotKey::new(
+        connection.url().as_str(),
+        access,
+        trace_id,
+        &trace_ref,
+        position.snapshot_ms,
+    )
+    .map_err(|_| Error::InvalidParameters)?;
+    let snapshot = match TRACE_SNAPSHOTS.get(&key).await {
+        Some(snapshot) => snapshot,
+        None => {
+            let params = TraceSpansParams {
+                access: access.clone(),
+                trace_id: trace_id.to_owned(),
+                trace_ref: trace_ref.clone(),
+            };
+            let rows =
+                crate::span_batches::read_spans(client, connection, params, position.snapshot_ms)
+                    .await?;
+            let spend_rows = spend(client, connection, access, &rows).await;
+            let Some(trace) = resolve_trace(trace_id, &trace_ref, &rows, &spend_rows) else {
+                return Ok(None);
+            };
+            TRACE_SNAPSHOTS.insert(key, trace).await?
+        }
+    };
+    let spans = &snapshot.trace().spans;
+    if cursor.is_some() && position.version != snapshot.version() {
+        return Err(Error::TraceChanged);
+    }
+    let mut trace = Trace {
+        summary: snapshot.trace().summary.clone(),
+        agents: snapshot.trace().agents.clone(),
+        spans: Vec::new(),
+        next_cursor: None,
+    };
+    if position.offset > spans.len() {
+        return Err(Error::InvalidCursor("span"));
+    }
+    let end = position
+        .offset
+        .saturating_add(page_size as usize)
+        .min(spans.len());
+    trace.next_cursor = (end < spans.len()).then(|| {
+        encode_cursor(&SpanPosition {
+            offset: end,
+            version: snapshot.version().to_owned(),
+            ..position
+        })
+    });
+    trace.spans = spans[position.offset..end].to_vec();
+    while serde_json::to_vec(&trace)
+        .map_err(|_| Error::InvalidResponse)?
+        .len()
+        > litellm_storage_clickhouse::READ_LIMITS.response_bytes
+    {
+        if trace.spans.len() <= 1 {
+            return Err(Error::ReadTooLarge);
+        }
+        trace.spans.truncate(trace.spans.len() / 2);
+        trace.next_cursor = Some(encode_cursor(&SpanPosition {
+            trace_ref: trace_ref.clone(),
+            snapshot_ms: position.snapshot_ms,
+            offset: position.offset + trace.spans.len(),
+            version: snapshot.version().to_owned(),
+        }));
+    }
+    Ok(Some(trace))
+}
+
+pub async fn get_span(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    trace_id: &str,
+    span_id: &str,
+    trace_ref: &str,
+) -> Result<Option<SpanDetail>, Error> {
+    let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else {
+        return Ok(None);
+    };
+    let params = SpanDetailParams {
+        access: access.clone(),
+        trace_id: trace_id.to_owned(),
+        trace_ref,
+        span_id: span_id.to_owned(),
+    };
+    let row = fetch::<SpanDetailQuery>(client, connection, &params)
+        .await?
+        .into_iter()
+        .next();
+    Ok(row.map(|row| SpanDetail {
+        input_ui: to_ui_content(&row.input),
+        output_ui: to_ui_content(&row.output),
+        span_id: row.span_id,
+        input: row.input,
+        output: row.output,
+        attributes: row.attributes,
+    }))
+}
+
+pub async fn get_span_error(
+    client: &Client,
+    connection: &Connection,
+    access: &ReadAccessParams,
+    trace_id: &str,
+    span_id: &str,
+    trace_ref: &str,
+    cursor: Option<&str>,
+) -> Result<Option<SpanErrorPage>, Error> {
+    let position = error_position(cursor)?;
+    let Some(trace_ref) = reference(client, connection, access, trace_id, trace_ref).await? else {
+        return Ok(None);
+    };
+    let offset = position.as_ref().map_or(0, |position| position.offset);
+    let params = SpanErrorParams::from(contracts::SpanErrorParams {
+        access: access.clone(),
+        trace_id: trace_id.to_owned(),
+        trace_ref,
+        span_id: span_id.to_owned(),
+        error_offset: offset,
+        error_version: position
+            .map(|position| position.version)
+            .unwrap_or_default(),
+    });
+    let Some(row) = fetch::<SpanError>(client, connection, &params)
+        .await?
+        .into_iter()
+        .next()
+    else {
+        return Ok(None);
+    };
+    let row = row.0;
+    let next_offset = offset + row.message.chars().count() as u64;
+    let next_cursor = (next_offset < row.total_chars).then(|| {
+        encode_cursor(&ErrorPosition {
+            offset: next_offset,
+            version: row.version,
+        })
+    });
+    Ok(Some(SpanErrorPage {
+        span_id: row.span_id,
+        message: row.message,
+        total_chars: row.total_chars,
+        next_cursor,
+    }))
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    fn trace_cursor_round_trips_the_last_listed_run() {
+        let cursor = encode_cursor(&(1_790_742_989_377_i64, "4bad42b84e9de3ba46fc870185f8f023"));
+        assert_eq!(
+            trace_position(Some(&cursor)).unwrap(),
+            (
+                1_790_742_989_377,
+                "4bad42b84e9de3ba46fc870185f8f023".to_owned()
+            )
+        );
+        assert_eq!(trace_position(None).unwrap(), (0, String::new()));
+        assert_eq!(trace_position(Some("")).unwrap(), (0, String::new()));
+    }
+
+    #[rstest]
+    #[case::not_base64("abc")]
+    #[case::not_json("bm90LWpzb24=")]
+    #[case::numeric_reference("WzEsIDJd")]
+    #[case::zero_start("WzAsICJ0Il0=")]
+    fn malformed_trace_cursors_are_rejected(#[case] cursor: &str) {
+        assert!(matches!(
+            trace_position(Some(cursor)),
+            Err(Error::InvalidCursor("trace"))
+        ));
+    }
+
+    #[rstest]
+    #[case::not_base64("garbage")]
+    #[case::missing_fields("e30=")]
+    #[case::not_an_object("WzEsMl0=")]
+    fn malformed_diagnostic_cursors_are_rejected(#[case] cursor: &str) {
+        assert!(matches!(
+            error_position(Some(cursor)),
+            Err(Error::InvalidCursor("diagnostic"))
+        ));
+    }
+
+    #[rstest]
+    #[case::lowercase_version("a".repeat(64))]
+    #[case::short_version("A".repeat(63))]
+    fn diagnostic_cursor_requires_a_content_version(#[case] version: String) {
+        let cursor = encode_cursor(&ErrorPosition { offset: 1, version });
+        assert!(matches!(
+            error_position(Some(&cursor)),
+            Err(Error::InvalidCursor("diagnostic"))
+        ));
+    }
+}

litellm-rust/crates/traces-clickhouse/src/schema.rs

@@ -78,12 +78,18 @@ pub struct NormalizedFieldDefinition {
     pub meaning: &'static str,
 }
 
-pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 9] = [
+pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 15] = [
     NormalizedFieldDefinition {
         name: "observation_type",
         clickhouse_column: "ObservationType",
         clickhouse_type: "LowCardinality(String)",
-        meaning: "Agent, LLM, tool, chain, or framework span",
+        meaning: "Operation recorded by the span, including agent, model, tool, retrieval and evaluation steps",
+    },
+    NormalizedFieldDefinition {
+        name: "wrapper_candidate",
+        clickhouse_column: "WrapperCandidate",
+        clickhouse_type: "Bool",
+        meaning: "Span may only wrap the operation it names; the trace graph decides",
     },
     NormalizedFieldDefinition {
         name: "agent_name",
@@ -97,12 +103,30 @@ pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 9] = [
         clickhouse_type: "LowCardinality(String)",
         meaning: "Agent framework or SDK that emitted this span, e.g. claude-agent-sdk",
     },
+    NormalizedFieldDefinition {
+        name: "agent_metadata",
+        clickhouse_column: "AgentMetadata",
+        clickhouse_type: "String",
+        meaning: "Typed agent metadata as JSON, including thread, subagent, runtime and repository identity",
+    },
     NormalizedFieldDefinition {
         name: "litellm_request_id",
         clickhouse_column: "LiteLLMRequestId",
         clickhouse_type: "String",
         meaning: "LiteLLM response ID used to link a span to a spend log",
     },
+    NormalizedFieldDefinition {
+        name: "call_keys",
+        clickhouse_column: "CallKeys",
+        clickhouse_type: "Array(String)",
+        meaning: "Model requests the span accounts for, as kind:id (litellm_request, provider_response, transport)",
+    },
+    NormalizedFieldDefinition {
+        name: "call_evidence",
+        clickhouse_column: "CallEvidence",
+        clickhouse_type: "LowCardinality(String)",
+        meaning: "Whether CallKeys are all of the span's requests: complete, partial or unknown",
+    },
     NormalizedFieldDefinition {
         name: "model",
         clickhouse_column: "Model",
@@ -127,10 +151,22 @@ pub const NORMALIZED_FIELD_DEFINITIONS: [NormalizedFieldDefinition; 9] = [
         clickhouse_type: "String",
         meaning: "Normalized input payload",
     },
+    NormalizedFieldDefinition {
+        name: "input_preview",
+        clickhouse_column: "InputPreview",
+        clickhouse_type: "String",
+        meaning: "Latest user message of the input, else the input's first characters",
+    },
     NormalizedFieldDefinition {
         name: "output",
         clickhouse_column: "Output",
         clickhouse_type: "String",
         meaning: "Normalized output payload",
     },
+    NormalizedFieldDefinition {
+        name: "tool_call_id",
+        clickhouse_column: "ToolCallId",
+        clickhouse_type: "String",
+        meaning: "Tool call the span executes, shared by instrumentations recording the same call",
+    },
 ];

litellm-rust/crates/traces-clickhouse/src/span_batches.rs

@@ -0,0 +1,270 @@
+use futures_util::{TryStreamExt, stream};
+use itertools::Itertools;
+use litellm_http::Client;
+use litellm_storage_clickhouse::{Query, fetch};
+use litellm_traces::query::named as contracts;
+use serde::Serialize;
+
+use crate::{Connection, Error, query::named::TraceSpansRow};
+
+const PAGE_SIZE: u32 = 256;
+pub(crate) const MAX_GRAPH_BYTES: usize = 64 * 1024 * 1024;
+const MAX_GRAPH_SPANS: usize = 100_000;
+
+#[derive(Default)]
+struct ReadBudget {
+    bytes: usize,
+    rows: usize,
+}
+
+impl ReadBudget {
+    fn checked_add(&self, bytes: usize, rows: usize) -> Result<Self, Error> {
+        let next = Self {
+            bytes: self.bytes.saturating_add(bytes),
+            rows: self.rows.saturating_add(rows),
+        };
+        if next.bytes > MAX_GRAPH_BYTES || next.rows > MAX_GRAPH_SPANS {
+            return Err(Error::ReadTooLarge);
+        }
+        Ok(next)
+    }
+
+    fn record(&mut self, row: &impl Serialize) -> Result<(), Error> {
+        let bytes = serde_json::to_vec(row).map_err(|_| Error::InvalidResponse)?;
+        *self = self.checked_add(bytes.len(), 1)?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize)]
+struct Parameters {
+    #[serde(flatten)]
+    trace: contracts::TraceSpansParams,
+    after_span_id: String,
+    page_size: u32,
+    snapshot_ms: u64,
+}
+
+struct SpanBatch;
+
+impl Query for SpanBatch {
+    type Params = Parameters;
+    type Row = TraceSpansRow;
+
+    const SQL: &'static str = include_str!("../query/trace_span_batch.sql");
+}
+
+pub(crate) async fn read_spans(
+    client: &Client,
+    connection: &Connection,
+    trace: contracts::TraceSpansParams,
+    snapshot_ms: u64,
+) -> Result<Vec<contracts::TraceSpansRow>, Error> {
+    let mut parameters = Parameters {
+        trace,
+        after_span_id: String::new(),
+        page_size: PAGE_SIZE,
+        snapshot_ms,
+    };
+    let mut spans = Vec::new();
+    let mut budget = ReadBudget::default();
+    loop {
+        let page = match fetch::<SpanBatch>(client, connection, &parameters).await {
+            Err(litellm_storage_clickhouse::Error::ResponseTooLarge)
+                if parameters.page_size > 1 =>
+            {
+                parameters.page_size /= 2;
+                continue;
+            }
+            Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => {
+                return Err(Error::ReadTooLarge);
+            }
+            result => result?,
+        };
+        let complete = page.len() < parameters.page_size as usize;
+        if let Some(last) = page.last() {
+            parameters.after_span_id.clone_from(&last.0.span_id);
+        }
+        for row in page {
+            budget.record(&row)?;
+            spans.push(row.0);
+        }
+        if complete {
+            spans.sort_by_key(|row| row.start_ns);
+            return Ok(spans);
+        }
+        parameters.page_size = (parameters.page_size * 2).min(PAGE_SIZE);
+    }
+}
+
+#[derive(Serialize)]
+struct ListParameters {
+    #[serde(flatten)]
+    runs: crate::query::named::TracePageSpansParams,
+    after_team: String,
+    after_key: String,
+    after_trace: String,
+    after_span: String,
+    page_size: u32,
+    snapshot_ms: u64,
+}
+
+struct ListSpanBatch;
+
+impl Query for ListSpanBatch {
+    type Params = ListParameters;
+    type Row = TraceSpansRow;
+
+    const SQL: &'static str = include_str!("../query/trace_list_span_batch.sql");
+}
+
+pub(crate) async fn read_list_spans(
+    client: &Client,
+    connection: &Connection,
+    runs: crate::query::named::TracePageSpansParams,
+) -> Result<Vec<contracts::TraceSpansRow>, Error> {
+    let parameters = ListParameters {
+        runs,
+        after_team: String::new(),
+        after_key: String::new(),
+        after_trace: String::new(),
+        after_span: String::new(),
+        page_size: PAGE_SIZE,
+        snapshot_ms: (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as u64,
+    };
+    let pages = stream::try_unfold(
+        (Some(parameters), ReadBudget::default()),
+        |(parameters, budget)| async move {
+            let Some(parameters) = parameters else {
+                return Ok(None);
+            };
+            let page = match fetch::<ListSpanBatch>(client, connection, &parameters).await {
+                Err(litellm_storage_clickhouse::Error::ResponseTooLarge)
+                    if parameters.page_size > 1 =>
+                {
+                    let retry = ListParameters {
+                        page_size: parameters.page_size / 2,
+                        ..parameters
+                    };
+                    return Ok(Some((Vec::new(), (Some(retry), budget))));
+                }
+                Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => {
+                    return Err(Error::ReadTooLarge);
+                }
+                result => result?,
+            };
+            let next = page
+                .last()
+                .filter(|_| page.len() == parameters.page_size as usize)
+                .map(|last| ListParameters {
+                    after_team: last.0.team_id.clone(),
+                    after_key: last.0.api_key_hash.clone(),
+                    after_trace: last.0.trace_id.clone(),
+                    after_span: last.0.span_id.clone(),
+                    page_size: (parameters.page_size * 2).min(PAGE_SIZE),
+                    ..parameters
+                });
+            let next_budget = page.iter().try_fold(budget, |budget, row| {
+                let bytes = serde_json::to_vec(row).map_err(|_| Error::InvalidResponse)?;
+                budget.checked_add(bytes.len(), 1)
+            })?;
+            Ok(Some((page, (next, next_budget))))
+        },
+    )
+    .try_collect::<Vec<_>>()
+    .await?;
+    Ok(pages
+        .into_iter()
+        .flatten()
+        .map(|row| row.0)
+        .sorted_by_key(|row| row.start_ns)
+        .collect())
+}
+
+#[derive(Serialize)]
+struct SpendParameters {
+    #[serde(flatten)]
+    lookup: crate::query::named::SpendByResponseIdsParams,
+    has_cursor: u8,
+    after_team: String,
+    after_ms: i64,
+    after_id: String,
+    page_size: u32,
+}
+
+struct SpendBatch;
+
+impl Query for SpendBatch {
+    type Params = SpendParameters;
+    type Row = crate::query::named::SpendByResponseIdsRow;
+
+    const SQL: &'static str = include_str!("../query/spend_batch.sql");
+}
+
+pub(crate) async fn read_spend(
+    client: &Client,
+    connection: &Connection,
+    lookup: crate::query::named::SpendByResponseIdsParams,
+) -> Result<Vec<contracts::SpendByResponseIdsRow>, Error> {
+    let mut parameters = SpendParameters {
+        lookup,
+        has_cursor: 0,
+        after_team: String::new(),
+        after_ms: 0,
+        after_id: String::new(),
+        page_size: PAGE_SIZE,
+    };
+    let mut rows = Vec::new();
+    let mut budget = ReadBudget::default();
+    loop {
+        let page = match fetch::<SpendBatch>(client, connection, &parameters).await {
+            Err(litellm_storage_clickhouse::Error::ResponseTooLarge)
+                if parameters.page_size > 1 =>
+            {
+                parameters.page_size /= 2;
+                continue;
+            }
+            Err(litellm_storage_clickhouse::Error::ResponseTooLarge) => {
+                return Err(Error::ReadTooLarge);
+            }
+            result => result?,
+        };
+        let complete = page.len() < parameters.page_size as usize;
+        if let Some(last) = page.last() {
+            parameters.has_cursor = 1;
+            parameters.after_team.clone_from(&last.0.team_id);
+            parameters.after_ms = last.0.start_ms;
+            parameters.after_id.clone_from(&last.0.request_id);
+        }
+        for row in page {
+            budget.record(&row)?;
+            rows.push(row.0);
+        }
+        if complete {
+            return Ok(rows);
+        }
+        parameters.page_size = (parameters.page_size * 2).min(PAGE_SIZE);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case::byte_boundary(MAX_GRAPH_BYTES - 1, 0, 1, false)]
+    #[case::byte_overflow(MAX_GRAPH_BYTES - 1, 0, 2, true)]
+    #[case::integer_overflow(MAX_GRAPH_BYTES, 0, usize::MAX, true)]
+    #[case::row_boundary(0, MAX_GRAPH_SPANS - 1, 1, false)]
+    #[case::row_overflow(0, MAX_GRAPH_SPANS, 1, true)]
+    fn accumulation_stops_at_the_graph_budget(
+        #[case] bytes: usize,
+        #[case] rows: usize,
+        #[case] next: usize,
+        #[case] rejected: bool,
+    ) {
+        let budget = ReadBudget { bytes, rows };
+        assert_eq!(budget.checked_add(next, 1).is_err(), rejected);
+    }
+}

litellm-rust/crates/traces-clickhouse/src/span_row.rs

@@ -0,0 +1,217 @@
+//! Decoded spans as `otel_traces` rows: payloads capped, the sending tenant stamped over whatever
+//! the export claimed, and resource maps shared across the rows that came from one resource.
+
+use std::collections::{BTreeMap, HashMap};
+
+use litellm_traces::{
+    CallEvidence, CallKey, DecodedEvent, DecodedSpan, Shared, SharedIdentity, Tenant,
+    truncate_messages, truncate_value,
+};
+use serde::Serialize;
+use serde_json::{Map, Value};
+
+use crate::InsertRow;
+
+/// Converts each distinct shared source once; keeping the source pins its identity.
+struct SharedValues<T>(HashMap<SharedIdentity, (Shared<T>, Shared<Value>)>);
+
+impl<T: Clone> SharedValues<T> {
+    fn new() -> Self {
+        Self(HashMap::new())
+    }
+
+    fn get(&mut self, source: &Shared<T>, convert: impl FnOnce(&T) -> Value) -> Shared<Value> {
+        self.0
+            .entry(source.identity())
+            .or_insert_with(|| (source.clone(), Shared::new(convert(source))))
+            .1
+            .clone()
+    }
+}
+
+fn stamped(attributes: &BTreeMap<String, String>, tenant: &Tenant) -> Value {
+    let mut stamped: Map<String, Value> = attributes
+        .iter()
+        .map(|(key, value)| (key.clone(), Value::from(value.as_str())))
+        .collect();
+    for (key, value) in [
+        ("litellm.team_id", &tenant.team_id),
+        ("litellm.api_key_hash", &tenant.api_key_hash),
+        ("litellm.org_id", &tenant.org_id),
+        ("litellm.user_id", &tenant.user_id),
+    ] {
+        stamped.insert(key.to_owned(), Value::from(value.as_str()));
+    }
+    Value::Object(stamped)
+}
+
+fn exception_message(events: &[DecodedEvent]) -> String {
+    events
+        .iter()
+        .find(|event| event.name == "exception")
+        .and_then(|event| {
+            event
+                .attributes
+                .get("exception.message")
+                .filter(|message| !message.is_empty())
+                .or_else(|| event.attributes.get("exception.type"))
+        })
+        .cloned()
+        .unwrap_or_default()
+}
+
+fn json<T: Serialize>(value: T) -> Value {
+    serde_json::to_value(value).unwrap_or(Value::Null)
+}
+
+fn present_fields<T: Serialize>(value: &T) -> String {
+    match json(value) {
+        Value::Object(fields) => Value::Object(
+            fields
+                .into_iter()
+                .filter(|(_, value)| !value.is_null())
+                .collect(),
+        )
+        .to_string(),
+        other => other.to_string(),
+    }
+}
+
+pub fn span_rows(
+    spans: Vec<DecodedSpan>,
+    tenant: &Tenant,
+    max_value_bytes: usize,
+) -> Vec<InsertRow> {
+    let mut resources = SharedValues::new();
+    let mut scopes = SharedValues::new();
+    spans
+        .into_iter()
+        .map(|span| {
+            let normalized = span.normalized;
+            let service = span
+                .resource_attributes
+                .get("service.name")
+                .cloned()
+                .unwrap_or_default();
+            let status_message = if span.status_message.is_empty() {
+                exception_message(&span.events)
+            } else {
+                span.status_message
+            };
+            let attributes: Map<String, Value> = span
+                .attributes
+                .into_iter()
+                .filter(|(key, _)| !span.consumed_attributes.contains(&key.as_str()))
+                .map(|(key, value)| (key, Value::String(truncate_value(value, max_value_bytes))))
+                .collect();
+            let shared = [
+                (
+                    "ResourceAttributes",
+                    resources.get(&span.resource_attributes, |attributes| {
+                        stamped(attributes, tenant)
+                    }),
+                ),
+                (
+                    "ScopeName",
+                    scopes.get(&span.scope_name, |name| Value::from(name.as_str())),
+                ),
+                (
+                    "ScopeVersion",
+                    scopes.get(&span.scope_version, |version| Value::from(version.as_str())),
+                ),
+            ];
+            let owned = [
+                ("Timestamp", json(span.start_ns)),
+                ("TraceId", Value::String(span.trace_id)),
+                ("SpanId", Value::String(span.span_id)),
+                ("ParentSpanId", Value::String(span.parent_span_id)),
+                ("TraceState", Value::String(span.trace_state)),
+                ("SpanName", Value::String(span.name)),
+                ("SpanKind", Value::String(span.kind)),
+                ("ServiceName", Value::String(service)),
+                ("SpanAttributes", Value::Object(attributes)),
+                ("Duration", json(span.end_ns - span.start_ns)),
+                ("StatusCode", Value::String(span.status_code)),
+                ("StatusMessage", Value::String(status_message)),
+                ("TeamId", Value::from(tenant.team_id.as_str())),
+                ("ApiKeyHash", Value::from(tenant.api_key_hash.as_str())),
+                ("UserId", Value::from(tenant.user_id.as_str())),
+                ("ObservationType", json(normalized.observation_type)),
+                (
+                    "WrapperCandidate",
+                    Value::Bool(normalized.wrapper_candidate),
+                ),
+                (
+                    "AgentName",
+                    Value::String(normalized.agent_name.unwrap_or_default()),
+                ),
+                (
+                    "Framework",
+                    Value::String(
+                        normalized
+                            .framework
+                            .map(|integration| integration.to_string())
+                            .unwrap_or_default(),
+                    ),
+                ),
+                (
+                    "AgentMetadata",
+                    Value::String(present_fields(&normalized.agent_metadata)),
+                ),
+                (
+                    "LiteLLMRequestId",
+                    Value::String(request_id(&normalized.calls).to_owned()),
+                ),
+                (
+                    "CallKeys",
+                    json(
+                        normalized
+                            .calls
+                            .key_set()
+                            .into_iter()
+                            .flatten()
+                            .collect::<Vec<_>>(),
+                    ),
+                ),
+                ("CallEvidence", json(normalized.calls.kind())),
+                ("Model", Value::String(normalized.model.unwrap_or_default())),
+                ("InputTokens", Value::from(normalized.input_tokens)),
+                ("OutputTokens", Value::from(normalized.output_tokens)),
+                (
+                    "Input",
+                    Value::String(truncate_messages(normalized.input, max_value_bytes)),
+                ),
+                ("InputPreview", Value::String(normalized.input_preview)),
+                (
+                    "Output",
+                    Value::String(truncate_value(normalized.output, max_value_bytes)),
+                ),
+                (
+                    "ToolCallId",
+                    Value::String(normalized.tool_call_id.unwrap_or_default()),
+                ),
+            ];
+            shared
+                .into_iter()
+                .chain(
+                    owned
+                        .into_iter()
+                        .map(|(column, value)| (column, Shared::new(value))),
+                )
+                .map(|(column, value)| (column.to_owned(), value))
+                .collect()
+        })
+        .collect()
+}
+
+fn request_id(evidence: &CallEvidence) -> &str {
+    evidence
+        .key_set()
+        .into_iter()
+        .flatten()
+        .find_map(|key| match key {
+            CallKey::LiteLlmRequest(id) | CallKey::ProviderResponse(id) => Some(id.as_str()),
+            CallKey::Transport => None,
+        })
+        .unwrap_or_default()
+}

litellm-rust/crates/traces-clickhouse/src/sql.rs

@@ -19,6 +19,9 @@ pub async fn execute_named_read(
             named_json::<TraceIdentity>(client, connection, parameters).await
         }
         ReadQuery::TraceSpans => named_json::<TraceSpans>(client, connection, parameters).await,
+        ReadQuery::TracePageSpans => {
+            named_json::<TracePageSpans>(client, connection, parameters).await
+        }
         ReadQuery::SpanDetail => named_json::<SpanDetail>(client, connection, parameters).await,
         ReadQuery::SpanError => named_json::<SpanError>(client, connection, parameters).await,
         ReadQuery::SpendByResponseIds => {

litellm-rust/crates/traces-clickhouse/src/table.rs

@@ -1,12 +1,7 @@
+#[macro_rules_attribute::apply(response_type)]
+#[cfg_attr(feature = "schema", schemars(rename = "TraceTableName"))]
 #[derive(
-    Clone,
-    Copy,
-    Debug,
-    serde::Serialize,
-    strum::Display,
-    strum::AsRefStr,
-    strum::EnumIter,
-    strum::IntoStaticStr,
+    Clone, Copy, Debug, strum::Display, strum::AsRefStr, strum::EnumIter, strum::IntoStaticStr,
 )]
 #[serde(rename_all = "snake_case")]
 #[strum(serialize_all = "snake_case")]

litellm-rust/crates/traces-clickhouse/src/wire_schema.rs

@@ -0,0 +1,119 @@
+use std::collections::BTreeMap;
+
+use schemars::{JsonSchema, Schema, SchemaGenerator, generate::SchemaSettings};
+use serde_json::json;
+
+use crate::query::lens;
+
+fn quoted_u64() -> Schema {
+    let upper = u64::MAX.to_string();
+    let alternatives = upper
+        .char_indices()
+        .filter_map(|(index, digit)| {
+            let lower = if index == 0 { '1' } else { '0' };
+            if digit <= lower {
+                return None;
+            }
+            Some(format!(
+                "{}[{}-{}][0-9]{{{}}}",
+                &upper[..index],
+                lower,
+                char::from(digit as u8 - 1),
+                upper.len() - index - 1
+            ))
+        })
+        .collect::<Vec<_>>()
+        .join("|");
+    json!({
+        "type": "string",
+        "pattern": format!("^(?:0|[1-9][0-9]{{0,{}}}|{alternatives}|{upper})$", upper.len() - 2),
+    })
+    .try_into()
+    .unwrap()
+}
+
+fn numeric_wire(normalized: Schema, python_type: String) -> Schema {
+    json!({
+        "anyOf": [normalized, quoted_u64()],
+        "x-python-normalized": {"type": python_type, "minimum": 0, "maximum": u64::MAX},
+    })
+    .try_into()
+    .unwrap()
+}
+
+pub(crate) fn u64_number(generator: &mut SchemaGenerator) -> Schema {
+    numeric_wire(u64::json_schema(generator), "int".to_owned())
+}
+
+pub(crate) fn flag_number(_: &mut SchemaGenerator) -> Schema {
+    json!({
+        "anyOf": [{"type": "integer", "enum": [0, 1]}, {"type": "string", "enum": ["0", "1"]}],
+        "x-python-normalized": {"type": "int", "minimum": 0, "maximum": 1}
+    })
+    .try_into()
+    .unwrap()
+}
+
+pub(crate) fn boolean_flag(_: &mut SchemaGenerator) -> Schema {
+    json!({
+        "anyOf": [{"type": "boolean"}, {"type": "integer", "enum": [0, 1]}, {"type": "string", "enum": ["0", "1"]}],
+        "default": false,
+        "x-python-normalized": {"type": "bool"}
+    }).try_into().unwrap()
+}
+
+pub(crate) fn selected(generator: &mut SchemaGenerator) -> Schema {
+    u64_number(generator)
+}
+
+fn received<T: JsonSchema>() -> Schema {
+    SchemaSettings::draft2020_12()
+        .for_deserialize()
+        .with_transform(litellm_traces::schema::integer_bounds)
+        .into_generator()
+        .into_root_schema_for::<T>()
+}
+
+pub fn schemas() -> BTreeMap<&'static str, Schema> {
+    BTreeMap::from([
+        ("ReadQueryName", json!({"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "ReadQueryName", "type": "string", "enum": lens::LENS_QUERIES.map(|query| query.to_string())}).try_into().unwrap()),
+        ("LensAccessParams", received::<lens::LensAccessParams>()),
+        ("LensSampleParams", received::<lens::LensSampleParams>()),
+        ("LensContentParams", received::<lens::LensContentParams>()),
+        ("LensEvidenceParams", received::<lens::LensEvidenceParams>()),
+        (
+            "ActivityAvailability",
+            received::<lens::LensAvailabilityRow>(),
+        ),
+        ("ExecutionRow", received::<lens::LensSampleRow>()),
+        ("PartRow", received::<lens::LensContentRow>()),
+        ("CountRow", received::<lens::LensEvidenceRow>()),
+        ("AgentRow", received::<lens::LensAgentsRow>()),
+        ("TraceQueryHelp", crate::query::help_schema()),
+    ])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case::zero(json!(0), true)]
+    #[case::quoted_zero(json!("0"), true)]
+    #[case::maximum(json!(u64::MAX), true)]
+    #[case::quoted_maximum(json!(u64::MAX.to_string()), true)]
+    #[case::negative(json!(-1), false)]
+    #[case::overflow(json!((u128::from(u64::MAX) + 1).to_string()), false)]
+    #[case::fraction(json!(1.5), false)]
+    fn count_schema_enforces_the_native_range(
+        #[case] value: serde_json::Value,
+        #[case] valid: bool,
+    ) {
+        let schema = received::<lens::LensEvidenceRow>();
+        assert_eq!(
+            jsonschema::is_valid(schema.as_value(), &json!({"count": value})),
+            valid
+        );
+    }
+}

litellm-rust/crates/traces-clickhouse/templates/query_help.jinja

@@ -1,65 +1,204 @@
-Trace SQL query guide
-
-Live ClickHouse schema
-{% for table in tables %}
+{% block live_schema -%}
+{% for table in tables -%}
 {{ table.name }}
-{% for column in table.columns %}{{ column.name }}: {{ column.kind }}
-{% endfor %}{% endfor %}
-Normalized span fields
-{% for field in normalized_fields %}{{ field.name }}: otel_traces.{{ field.clickhouse_column }} ({{ field.clickhouse_type }})
-{{ field.meaning }}
+{% for column in table.columns -%}
+{{ column.name }}: {{ column.kind }}
 {% endfor %}
-Observed LLM call metadata
+{% endfor -%}
+{%- endblock %}
+
+{% block normalized_fields -%}
+{% for field in normalized_fields -%}
+{{ field.name }}: otel_traces.{{ field.clickhouse_column }} ({{ field.clickhouse_type }})
+{{ field.meaning }}
+{% endfor -%}
+{%- endblock %}
+
+{% block metadata -%}
 {{ metadata.scope }}
-{% match metadata.discovery %}{% when Discovery::Unavailable(error) %}Metadata discovery unavailable: {{ error }}
-{% when Discovery::Observed(sample) %}Sampled rows: {{ sample.sampled_rows }}; invalid JSON rows: {{ sample.invalid_json_rows }}; truncated: {{ sample.truncated }}
-{% if sample.fields.is_empty() %}No metadata paths found in the sampled rows
-{% else %}{% for field in sample.fields %}{{ field.expression }}: {% for kind in field.types %}{{ kind }} {% endfor %}
-{% endfor %}{% endif %}{% endmatch %}
-Observed span and resource attributes
-{% for catalog in attributes %}{{ catalog.table }}.{{ catalog.column }}
+Sampling SQL:
+{{ metadata.sample_sql }}
+{% match metadata.discovery -%}
+{% when Discovery::Unavailable(error) -%}
+Metadata discovery unavailable: {{ error }}
+{% when Discovery::Observed(sample) -%}
+Sampled rows: {{ sample.sampled_rows }}; invalid JSON rows: {{ sample.invalid_json_rows }}; truncated: {{ sample.truncated }}
+{% if sample.fields.is_empty() -%}
+No metadata paths found in the sampled rows
+{% else -%}
+{% for field in sample.fields -%}
+{{ field.expression }}: {{ field.types|join(", ") }}
+{% endfor -%}
+{% endif -%}
+{% endmatch -%}
+{%- endblock %}
+
+{% block attributes -%}
+{% for catalog in attributes -%}
+{{ catalog.table }}.{{ catalog.column }}
 {{ catalog.scope }}
-{% match catalog.discovery %}{% when Discovery::Unavailable(error) %}Attribute discovery unavailable: {{ error }}
-{% when Discovery::Observed(sample) %}{% if sample.fields.is_empty() %}No attribute keys found in the sampled spans
-{% else %}{% for field in sample.fields %}{{ field.expression }}: {{ field.kind }}
-{% endfor %}{% endif %}{% endmatch %}{% endfor %}
-Examples
+Discovery SQL:
+{{ catalog.discovery_sql }}
+{% match catalog.discovery -%}
+{% when Discovery::Unavailable(error) -%}
+Attribute discovery unavailable: {{ error }}
+{% when Discovery::Observed(sample) -%}
+Truncated: {{ sample.truncated }}
+{% if sample.fields.is_empty() -%}
+No attribute keys found in the sampled spans
+{% else -%}
+{% for field in sample.fields -%}
+{{ field.expression }}: {{ field.kind }}
+{% endfor -%}
+{% endif -%}
+{% endmatch %}
+{% endfor -%}
+{%- endblock %}
 
-{% block recent_spans_name %}Recent normalized LLM spans{% endblock %}
-{% block recent_spans_sql %}SELECT TraceId, SpanId, Model, InputTokens, OutputTokens, Duration / 1000000 AS duration_ms FROM otel_traces WHERE Timestamp >= now() - INTERVAL 1 DAY AND ObservationType = 'llm' ORDER BY Timestamp DESC LIMIT 100{% endblock %}
+{% block recent_spans_name -%}
+Recent normalized LLM spans
+{%- endblock %}
 
-{% block custom_metadata_name %}Find calls by custom metadata{% endblock %}
-{% block custom_metadata_sql %}SELECT request_id, response_id, model, spend, JSONExtractString(metadata, 'project') AS project FROM spend_logs FINAL WHERE start_time >= now() - INTERVAL 1 DAY AND JSONHas(metadata, 'project') AND JSONExtractString(metadata, 'project') = 'example' ORDER BY start_time DESC LIMIT 100{% endblock %}
+{% block recent_spans_sql -%}
+{% include "../query/help/recent_spans.sql" %}
+{%- endblock %}
 
-{% block nested_metadata_name %}Nested metadata with unknown types{% endblock %}
-{% block nested_metadata_sql %}SELECT request_id, JSONType(metadata, 'labels', 'priority') AS type, JSONExtractRaw(metadata, 'labels', 'priority') AS value FROM spend_logs FINAL WHERE start_time >= now() - INTERVAL 1 DAY AND JSONHas(metadata, 'labels', 'priority') LIMIT 100{% endblock %}
+{% block custom_metadata_name -%}
+Find calls by custom metadata
+{%- endblock %}
 
-{% block correlated_calls_name %}Traces correlated with LLM call metadata{% endblock %}
-{% block correlated_calls_sql %}SELECT t.TraceId, t.SpanId, s.request_id, s.spend, s.metadata FROM otel_traces AS t INNER JOIN (SELECT * FROM spend_logs FINAL WHERE start_time >= now() - INTERVAL 1 DAY) AS s ON t.LiteLLMRequestId = s.response_id AND t.TeamId = s.team_id AND (t.TeamId != '' OR (t.UserId != '' AND t.UserId = s.user) OR (t.ApiKeyHash != '' AND t.ApiKeyHash = s.api_key)) WHERE t.Timestamp >= now() - INTERVAL 1 DAY AND t.LiteLLMRequestId != '' AND JSONExtractString(s.metadata, 'project') = 'example' LIMIT 100{% endblock %}
+{% block custom_metadata_sql -%}
+{% include "../query/help/custom_metadata.sql" %}
+{%- endblock %}
 
-{% block discover_keys_name %}Discover metadata keys over a different window{% endblock %}
-{% block discover_keys_sql %}SELECT DISTINCT arrayJoin(JSONExtractKeys(metadata)) AS key FROM spend_logs FINAL WHERE start_time >= now() - INTERVAL 30 DAY ORDER BY key LIMIT 200{% endblock %}
+{% block nested_metadata_name -%}
+Nested metadata with unknown types
+{%- endblock %}
 
-Gotchas
+{% block nested_metadata_sql -%}
+{% include "../query/help/nested_metadata.sql" %}
+{%- endblock %}
 
-{% block time_window %}Always bound Timestamp or start_time and use LIMIT; add TeamId/ApiKeyHash or team_id/api_key filters when investigating one tenant{% endblock %}
+{% block correlated_calls_name -%}
+Traces correlated with LLM call metadata
+{%- endblock %}
 
-{% block reader_limits %}The reader enforces {{ limits.result_rows }} result rows, {{ limits.result_mib() }} MiB response bytes, {{ limits.memory_mib() }} MiB memory and a {{ limits.execution_seconds }} second query limit; exceeding limits fails instead of returning partial results{% endblock %}
+{% block correlated_calls_sql -%}
+{% include "../query/help/correlated_calls.sql" %}
+{%- endblock %}
 
-{% block reader_profile %}LiteLLM provisions SELECT-only readers from the configured ClickHouse connection and enforces request-log visibility through row policies. Callers see their own user rows and permitted teams. Provisioning requires CREATE USER, ALTER USER, CREATE ROW POLICY, and GRANT SELECT permissions{% endblock %}
+{% block discover_keys_name -%}
+Discover metadata keys over a different window
+{%- endblock %}
 
-{% block output_format %}Do not add FORMAT clauses; the endpoint requires ClickHouse JSON output{% endblock %}
+{% block discover_keys_sql -%}
+{% include "../query/help/discover_keys.sql" %}
+{%- endblock %}
 
-{% block json_values %}metadata is a JSON-encoded String; use JSONHas before typed extraction to distinguish missing values from empty strings, zero and false{% endblock %}
+{% block recent_spend_name -%}
+Recent spend records
+{%- endblock %}
 
-{% block map_values %}SpanAttributes and ResourceAttributes are Map(String, String); missing map keys return an empty string, so use mapContains for existence checks{% endblock %}
+{% block recent_spend_sql -%}
+{% include "../query/help/recent_spend.sql" %}
+{%- endblock %}
 
-{% block literal_keys %}Use the discovered path components as separate JSONExtract arguments; a dot inside a key is literal, not a path separator{% endblock %}
+{% block model_spend_name -%}
+Spend and tokens by model
+{%- endblock %}
 
-{% block time_units %}Duration is nanoseconds; Timestamp has nanosecond precision, spend start_time has millisecond precision{% endblock %}
+{% block model_spend_sql -%}
+{% include "../query/help/model_spend.sql" %}
+{%- endblock %}
 
-{% block spend_totals %}Use spend_logs FINAL to collapse replacement rows before totals. Shared response IDs and multiple spans can multiply costs in joins; require one spend match per response ID and ownership before aggregating. Missing IDs or costs leave totals unknown{% endblock %}
+{% block trace_spend_name -%}
+Recorded spend by trace
+{%- endblock %}
 
-{% block trace_rollups %}agent_traces_by_key uses SimpleAggregateFunction columns; group by TeamId, ApiKeyHash and TraceId, using min(StartTs), max(EndTs), sum(SpanCount) and groupUniqArrayArray(Models). Do not use Merge combinators{% endblock %}
+{% block trace_spend_sql -%}
+{% include "../query/help/trace_spend.sql" %}
+{%- endblock %}
 
-{% block sampling %}Discovery is sampled, contains no metadata values, and is not an exhaustive schema. Edit the supplied discovery SQL for older data or nested JSONExtractKeys(metadata, 'parent'){% endblock %}
+{% block unmatched_spans_name -%}
+LLM spans without a direct spend match
+{%- endblock %}
+
+{% block unmatched_spans_sql -%}
+{% include "../query/help/unmatched_spans.sql" %}
+{%- endblock %}
+
+{% block trace_summary_name -%}
+Trace summaries with tokens and errors
+{%- endblock %}
+
+{% block trace_summary_sql -%}
+{% include "../query/help/trace_summary.sql" %}
+{%- endblock %}
+
+{% block failed_spans_name -%}
+Recent failed spans
+{%- endblock %}
+
+{% block failed_spans_sql -%}
+{% include "../query/help/failed_spans.sql" %}
+{%- endblock %}
+
+{% block metadata_filter_name -%}
+Filter calls by nested metadata
+{%- endblock %}
+
+{% block metadata_filter_sql -%}
+{% include "../query/help/metadata_filter.sql" %}
+{%- endblock %}
+
+{% block time_window -%}
+Always bound Timestamp or start_time and use LIMIT; add TeamId/ApiKeyHash or team_id/api_key filters when investigating one tenant
+{%- endblock %}
+
+{% block reader_limits -%}
+The reader enforces {{ limits.result_rows }} result rows, {{ limits.result_mib() }} MiB response bytes, {{ limits.memory_mib() }} MiB memory and a {{ limits.execution_seconds }} second query limit; exceeding limits fails instead of returning partial results
+{%- endblock %}
+
+{% block reader_profile -%}
+LiteLLM provisions SELECT-only readers from the configured ClickHouse connection and enforces request-log visibility through row policies. Callers see their own user rows and permitted teams. Provisioning requires CREATE USER, ALTER USER, CREATE ROW POLICY, and GRANT SELECT permissions
+{%- endblock %}
+
+{% block output_format -%}
+Do not add FORMAT clauses; the endpoint requires ClickHouse JSON output
+{%- endblock %}
+
+{% block json_values -%}
+metadata is a JSON-encoded String; use JSONHas before typed extraction to distinguish missing values from empty strings, zero and false
+{%- endblock %}
+
+{% block map_values -%}
+SpanAttributes and ResourceAttributes are Map(String, String); missing map keys return an empty string, so use mapContains for existence checks
+{%- endblock %}
+
+{% block literal_keys -%}
+Use the discovered path components as separate JSONExtract arguments; a dot inside a key is literal, not a path separator
+{%- endblock %}
+
+{% block time_units -%}
+Duration is nanoseconds; Timestamp has nanosecond precision, spend start_time has millisecond precision
+{%- endblock %}
+
+{% block missing_spend -%}
+Token usage does not establish billed spend. OTLP exports without companion spend_logs rows have unknown cost; synthetic fixture spend is marked by metadata.synthetic_spend
+{%- endblock %}
+
+{% block partial_spend -%}
+Recorded spend by trace totals only requests whose spend_logs.trace_id is populated. Direct ID joins do not resolve every CallKeys entry, managed Responses IDs, or transport correlation. Use the trace detail API for resolved totals; unmatched spans are a starting point for investigation
+{%- endblock %}
+
+{% block spend_totals -%}
+Use spend_logs FINAL to collapse replacement rows before totals. Shared response IDs and multiple spans can multiply costs in joins; require one spend match per response ID and ownership before aggregating. Missing IDs or costs leave totals unknown
+{%- endblock %}
+
+{% block trace_rollups -%}
+agent_traces_by_key uses SimpleAggregateFunction columns; group by TeamId, ApiKeyHash and TraceId, using min(StartTs), max(EndTs), sum(SpanCount) and groupUniqArrayArray(Models). Do not use Merge combinators
+{%- endblock %}
+
+{% block sampling -%}
+Discovery is sampled, contains no metadata values, and is not an exhaustive schema. Edit the supplied discovery SQL for older data or nested JSONExtractKeys(metadata, 'parent')
+{%- endblock %}

litellm-rust/crates/traces-clickhouse/tests/admin_sql.rs

@@ -174,7 +174,7 @@ async fn admin_sql_enforces_result_row_limit(
         matches!(
             result,
             Err(Error::Storage(
-                litellm_storage_clickhouse::Error::QueryFailed(_)
+                litellm_storage_clickhouse::Error::ResponseTooLarge
             ))
         ),
         "{result:?}"

litellm-rust/crates/traces-clickhouse/tests/fixtures/README.md

@@ -8,6 +8,18 @@ Raw OTLP exports live in `crates/traces/tests/fixtures/query_*.json`. The seeded
 
 The swarm capture has handoff spans marked ERROR with `ParentCommand` exception events and a root with UNSET status. These are exported diagnostic statuses, which do not establish a failed execution. The tests preserve incoming statuses and check root status separately from the count of error spans, deriving both from the decoded export. They do not infer an execution outcome from exception text, framework names, successful model calls, or output presence. Framework-specific interpretation of control-flow exceptions belongs in the instrumentation integration
 
+For a local dashboard with linked requests and traces, run `bash scripts/run_tracing_proxy_local.sh --seed` from the repository root and open `http://127.0.0.1:4002/ui/`. Log in as `admin` with password `sk-1234`, matching the UI E2E harness. The launcher keeps the proxy running until Ctrl-C and leaves the database volumes intact
+
+`deeplite_swarm_spend_logs.jsonl` pairs every LLM span in the swarm export with a ClickHouse spend row. Response IDs, trace and span IDs, token counts, input, output, and timestamps come from the export. Messages and responses use the chat completion format supported by the request viewer. Spend is synthetic, set to $0.01 per request and marked in metadata, because the export does not include actual billed costs. These rows are stored here because `traces-clickhouse` owns the spend row schema
+
+The simple and swarm exports for all twelve SDK examples were captured on 2026-10-03 against port 4002 using `openai/gpt-6-luna`. Each export has a matching `<name>_spend_logs.jsonl` with actual proxy spend, usage, request and response IDs, messages, and timestamps. Authorization headers, provider cookies, organization and project identifiers, and local paths were redacted. OTLP identifiers and enums use their canonical JSON encodings. `metadata.fixture_capture` identifies the associated export and whether model spans contain sufficient identity to join spend
+
+The LlamaIndex captures contain provider IDs inside `output.value.raw.id`. Regression tests require normalization to retain those call keys and trace cost resolution to count nested model spans once. The Claude captures use the SDK example's local gateway adapter, which supplies the actual Anthropic message ID in the `request-id` response header. The two `claude_agent_sdk_missing_request_id_*` exports retain the earlier behavior: real spend rows exist, but model spans contain no matching call IDs, so trace spend remains unknown
+
+`scripts/seed_tracing_fixtures.py` replays every JSON export in `crates/traces/tests/fixtures` through `POST /v1/traces`, then inserts all companion spend rows into ClickHouse through the production storage API and into Postgres through Prisma. The Requests table reads Postgres, while trace costs and Lens read ClickHouse. It shifts each capture into the current time window, keeping span, event, and paired spend timestamps aligned. The split `query_*.json` exports share a time shift and ID namespace to preserve cross-file parent links. Other captures get separate ID namespaces to avoid collisions between fixtures. It assigns fresh linked IDs for each run, including provider IDs inside managed response IDs, and reads the authenticated tenant from the ingested spans before stamping spend rows. The command exits unsuccessfully if any trace detail API result differs from its captured spend total or expected unknown cost. Exports without companion spend rows retain missing costs and do not create Requests entries
+
+`tests/test_litellm_rust/test_traces.py` ingests these exports and spend rows into an isolated ClickHouse container, then checks trace detail costs and spend queries through the real FastAPI endpoints. Every SQL example returned by `/v1/traces/query/help` is executed through `/v1/traces/query`, including missing costs, free requests, replacement rows, and tenant ownership cases
+
 `spend_logs.jsonl` contains spend insert rows with millisecond timestamps, including two versions of one request. Replace this small placeholder dataset when the actual data is available. The query fixture applies production migrations, then removes TTL from its isolated database so fixed timestamps do not expire. Background merges are stopped so rollup aggregation and `FINAL` deduplication are exercised on unmerged data. Retention behavior stays covered by the migration tests
 
 Curated SQL lives in `tests/queries/*.sql`. Each query has a matching `.expected.json` containing ordered result rows for `admin`, `team`, `key`, and `other_team` readers. Update the exports and expected results together. Add a named case in `tests/queries.rs` for each new query. Assertions compare only result data, excluding server statistics and execution timing