Merge remote-tracking branch 'origin/litellm_mcp_listed_tool_metadata' into litellm_mcp_gateway_sign_in_provider_pr

.circleci/scripts/unit_selection.sh

@@ -107,6 +107,7 @@ legacy_paths() {
       echo tests/unit/proxy/test_update_spend.py
       echo tests/unit/skills/test_skills_db.py ;;
     proxy-db-endpoints-and-responses)
+      echo tests/unit/proxy/engine
       echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
       echo tests/unit/proxy/common_utils/test_check_batch_cost.py
       echo tests/unit/proxy/common_utils/test_check_responses_cost.py
@@ -148,7 +149,10 @@ legacy_paths() {
       echo tests/unit/proxy/test_proxy_server.py ;;
     proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
     proxy-extras) echo tests/unit/litellm_proxy_extras ;;
-    proxy-infra) echo tests/unit/gateway ;;
+    proxy-infra)
+      echo tests/unit/gateway
+      echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
+      echo tests/unit/proxy/roi_calculator ;;
     responses-caching-types)
       find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
       echo tests/unit/types ;;

.github/merge-smoke-tests.json

@@ -3,8 +3,8 @@
     "CHAT-JSON": "tests/unit/llms/openai/test_openai.py::test_acompletion_returns_json_reply_over_injected_transport",
     "CHAT-TEXT-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_text_deltas_over_injected_transport",
     "CHAT-TOOL-STREAM": "tests/unit/llms/openai/test_openai.py::test_acompletion_streams_tool_call_arguments_over_injected_transport",
-    "MODEL-ALLOW": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_allows_listed_model_for_key",
-    "MODEL-DENY": "tests/test_litellm/proxy/auth/test_auth_checks.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
+    "MODEL-ALLOW": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_allows_listed_model_for_key",
+    "MODEL-DENY": "tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py::test_can_object_call_model_denials_return_forbidden[key-key_model_access_denied]",
     "COST-EXPLICIT": "tests/unit/test_cost_calculator.py::test_completion_cost_charges_explicit_per_token_rates_over_registered_ones",
     "COST-ZERO": "tests/unit/test_cost_calculator.py::test_completion_cost_is_zero_when_explicit_rates_are_zero",
     "LOG-CONTENT-ON": "tests/unit/litellm_core_utils/test_litellm_logging.py::test_standard_logging_payload_keeps_message_content_when_message_logging_is_on",

.github/workflows/lens-worker.yml

@@ -0,0 +1,66 @@
+name: Lens Worker Image
+
+on:
+  pull_request:
+    branches: [main, litellm_oss_branch, "litellm_**"]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/engine/**
+      - .github/workflows/lens-worker.yml
+  push:
+    branches: [main, litellm_agent_engine]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/engine/**
+      - .github/workflows/lens-worker.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lens-worker-image:
+    permissions:
+      contents: read
+      packages: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Build Lens worker
+        run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+      - name: Verify standalone imports with a read-only filesystem
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \
+            --security-opt no-new-privileges --entrypoint python \
+            lens-worker:${{ github.sha }} -c '
+          import os
+          import engine.worker
+          from engine.trace_store import trace_store
+          assert os.getuid() == 65532
+          with trace_store() as store:
+              assert store.count() == 0
+          '
+      - name: Verify recovery after temporary storage fills
+        run: |
+          docker run --rm --network none --read-only --cap-drop ALL \
+            --tmpfs /tmp:rw,noexec,nosuid,size=64k --security-opt no-new-privileges \
+            -v "$PWD/tests/proxy_behavior/lens/worker_storage_smoke.py:/app/storage_smoke.py:ro" \
+            --entrypoint python lens-worker:${{ github.sha }} /app/storage_smoke.py
+      - name: Publish versioned Lens worker
+        if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
+        env:
+          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }}
+        run: |
+          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
+          docker tag lens-worker:${{ github.sha }} "$IMAGE"
+          docker push "$IMAGE"
+          printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test-code-quality.yml

@@ -80,6 +80,11 @@ jobs:
       - name: test_e2e_changed_gate
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
 
+      - name: test_e2e_metadata
+        env:
+          PYTHONPATH: tests/e2e
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_metadata.py tests/code_coverage_tests/test_e2e_junit_report.py
+
       - name: Check merge smoke harness
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_merge_smoke.py
 

.github/workflows/test-postgres.yml

@@ -24,6 +24,7 @@ jobs:
     timeout-minutes: ${{ matrix.job-timeout-minutes }}
     permissions:
       contents: read
+      id-token: write
 
     services:
       postgres:
@@ -134,9 +135,21 @@ jobs:
         env:
           TEST_PATH: ${{ matrix.test-path }}
           WORKERS: ${{ matrix.workers }}
+          PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=./litellm --cov-report=xml:coverage-lens-postgres.xml' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
           else
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
           fi
+
+      - name: Upload Lens database coverage
+        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
+        uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
+        with:
+          use_oidc: true
+          version: v11.3.1
+          root_dir: ${{ github.workspace }}
+          files: coverage-lens-postgres.xml
+          flags: lens-postgres
+          fail_ci_if_error: true

.github/workflows/test-unit.yml

@@ -79,7 +79,9 @@ jobs:
 
           - shard: integrations
             artifact-name: integrations
-            test-path: ""
+            test-path: >-
+              tests/test_litellm/integrations
+              tests/test_litellm/tracing
             unit-flag: integrations
             workers: 2
             reruns: 3
@@ -117,10 +119,19 @@ jobs:
           - shard: proxy-auth
             artifact-name: proxy-auth
             test-path: >-
-              tests/test_litellm/proxy/auth
-              tests/test_litellm/proxy/hooks
-              tests/test_litellm/proxy/policy_engine
-              tests/test_litellm/proxy/client
+              tests/unit/proxy/auth
+              tests/unit/proxy/hooks
+              tests/unit/proxy/policy_engine
+              tests/unit/proxy/client
+              --ignore=tests/unit/proxy/auth/test_auth_checks.py
+              --ignore=tests/unit/proxy/auth/test_user_api_key_auth.py
+              --ignore=tests/unit/proxy/auth/test_default_end_user_budget_simple.py
+              --ignore=tests/unit/proxy/auth/test_jwt.py
+              --ignore=tests/unit/proxy/auth/test_models_fallback_endpoint.py
+              --ignore=tests/unit/proxy/auth/test_multipart_bypass_repro.py
+              --ignore=tests/unit/proxy/auth/test_proxy_routes.py
+              --ignore=tests/unit/proxy/hooks/test_banned_keyword_list.py
+              --ignore=tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
@@ -188,6 +199,8 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
+              tests/unit/proxy/test_proxy_server_endpoints_and_startup.py
+              tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py
             unit-flag: proxy-infra
             workers: 4
             reruns: 2

Makefile

@@ -324,10 +324,10 @@ test-unit-proxy-guardrails: install-test-deps
 	$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-core: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/unit/proxy/auth tests/unit/proxy/client tests/test_litellm/proxy/db tests/unit/proxy/hooks tests/unit/proxy/policy_engine --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-misc: install-test-deps
-	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py tests/unit/proxy/test_proxy_server_endpoints_and_startup.py tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py tests/unit/proxy/_experimental/mcp_server/test_mcp_server_tool_calls_and_headers.py --tb=short -vv -n 4 --durations=20
 
 test-unit-integrations: install-test-deps
 	$(UV_RUN) pytest tests/unit/integrations --tb=short -vv -n 4 --durations=20

backend/routes/allowlist.py

@@ -81,6 +81,8 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Spend / analytics
     "/spend/",
     "/analytics/",
+    "/engine/",
+    "/v1/traces",
     "/global/",
     "/user_agent",
     "/usage/",
@@ -144,6 +146,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
     {
         "/",
         "/routes",
+        "/engine",
         "/openapi.json",
         "/docs",
         "/docs/oauth2-redirect",

cookbook/misc/test_responses_api.py

@@ -12,7 +12,7 @@ def encode_image(image_path):
 
 
 # Path to your image
-image_path = "litellm/proxy/logo.jpg"
+image_path = "litellm/proxy/logo.png"
 
 # Getting the Base64 string
 base64_image = encode_image(image_path)
@@ -27,7 +27,7 @@ response = client.responses.create(
                 {"type": "input_text", "text": "what color is the image"},
                 {
                     "type": "input_image",
-                    "image_url": f"data:image/jpeg;base64,{base64_image}",
+                    "image_url": f"data:image/png;base64,{base64_image}",
                 },
             ],
         }

deploy/lens/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
+COPY litellm/proxy/engine/__init__.py litellm/proxy/engine/models.py litellm/proxy/engine/trace_store.py litellm/proxy/engine/analysis.py litellm/proxy/engine/worker.py /app/engine/
+USER 65532:65532
+CMD ["python", "-m", "engine.worker"]

deploy/lens/Dockerfile.dockerignore

@@ -0,0 +1,8 @@
+**
+!litellm/
+!litellm/proxy/
+!litellm/proxy/engine/
+!litellm/proxy/engine/__init__.py
+!litellm/proxy/engine/models.py
+!litellm/proxy/engine/analysis.py
+!litellm/proxy/engine/worker.py

deploy/lens/README.md

@@ -0,0 +1,109 @@
+# Lens worker
+
+Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
+
+## Start a worker
+
+Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
+
+In Lens, click **Set up analysis**, choose an existing virtual key or **Create worker key**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Analyzer connected** when the container checks in
+
+The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
+
+The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds
+
+For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
+
+```bash
+docker compose --env-file /path/to/lens.env -f compose.yaml up -d
+```
+
+Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
+
+The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
+
+The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its normal virtual-key authorization and inference pipeline; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential
+
+If your deployment restricts `allowed_ips`, allow the worker's address. For workers behind a reverse proxy with `use_x_forwarded_for: true`, also configure `mcp_trusted_proxy_ranges` with that proxy's CIDRs and, when needed, `mcp_xff_num_trusted_hops`. Lens reuses these existing trusted-proxy settings. Forwarded addresses without an established trust boundary are rejected by the allowlist; accepting them would let a worker impersonate an allowed address
+
+V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Proxy-admin viewers can inspect results. Regular user and team keys cannot access the Lens API. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
+
+## Configure a lens
+
+Choose agent runs, individual LLM requests, or both. The matching-activity preview updates as you choose an application (the recorded OpenTelemetry service.name) or, for request activity, a LiteLLM model group and add metadata conditions. It shows run names, timestamps, and trace IDs; open a run to inspect its original steps before starting analysis. Suggestions come from up to 100 recent executions and may not include every recorded attribute. You can enter other exact keys and values. Leave service and filters blank for all activity your account can access. Filters are exact key/value matches, combined with AND. Trace filters match span or resource attributes on the same span. Request filters match logged metadata, including caller metadata stored under `requester_metadata`; `tag=value` matches request tags. `swarm=research` works only if your instrumentation records that attribute
+
+Describe how the agent should behave and optionally add specific checks. Select the lookback window, team and metadata, then choose the percentage to review and an optional maximum. **100% with no maximum selects every matching run**. The preview pages through all matching activity and lets you select particular runs. Percentage sampling uses a stable hash order, rounds up, and applies the optional maximum after the percentage
+
+Choose your analysis model, parallelism and monthly budget. Parallelism controls simultaneous model calls, not the number of runs selected. New lenses run once by default. Turn on monitoring to repeat the same setup at a custom interval. **Run now** uses the same saved settings immediately, including the same lookback window and sampling. Every scan recalculates the window, so overlapping windows can review the same activity again. Duplicate a lens when you want a separate investigation without changing an existing monitor
+
+Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Run now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
+
+## Read the results
+
+Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step
+
+Use the batch selector or Scans tab to reopen previous results. Each batch keeps its own findings, settings, selected runs, coverage and cost. Older batches created before snapshot support remain available through accumulated findings. The Runs tab lists the selected batch's sample and can filter per-run observations, including runs without an observed issue and runs with insufficient evidence. These observations precede the final evidence investigation. Linked-run counts on findings include cited counterexamples, so they are not failure counts
+
+Choose **This is expected** and explain why to teach later scans about acceptable behavior. Feedback is kept with the lens and included in subsequent reviews. It does not alter historical evidence or exempt different problems
+
+## What a scan does
+
+The proxy selects executions received or updated within the configured lookback window, with a two-minute settling period. Older rows without receipt timestamps use execution end time. Overlapping scans do not increment a finding's occurrence count for the same execution ID
+
+A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
+
+The worker reviews the selected executions in parallel. It pages through their recorded spans and gives the first reviewer a catalog, task and outcome excerpts. The reviewer can read more original content to resolve uncertainties. Large catalogs and groups of observations are processed in bounded context windows, with every page available. Grouping retains supporting run IDs in code, so a pattern occurring thousands of times does not require a model to repeat thousands of IDs. Candidate investigators can page through supporting observations, other runs and original evidence
+
+There is no fixed total run, span, candidate or investigation-turn cutoff. Repeated or empty evidence requests stop a stalled investigation. Context windows, the configured budget, available model capacity and recorded evidence still bound practical work. The dashboard reports completed work and gaps. The investigator has no shell, browsing, code-editing or production-action tools
+
+Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
+
+Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
+
+## Operations and limits
+
+PostgreSQL stores configurations, findings and all scan history, returned in pages of 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost
+
+Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Both the Lens budget and the selected virtual key’s budgets, model permissions, and rate limits apply. Analysis spend appears under that key in Virtual Keys and normal request logs, with Lens, scan, and worker IDs in request metadata. Analysis prompts and responses are redacted from spend logs; source traces and findings remain available through the administrator-only Lens API. Existing workers need a billing key assigned in **Set up analysis** before they can resume
+
+V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them
+
+
+## API access
+
+The UI and API use the same scan lifecycle. Authenticate with a proxy administrator credential for writes, or a proxy-admin viewer credential for reads. Worker credentials are only for worker operations
+
+```bash
+curl "$LITELLM_URL/engine" -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -H 'Content-Type: application/json' -d '{
+    "name": "Research quality", "model": "your-model-alias",
+    "context": "Answer the requested question using cited, retrieved evidence.",
+    "source": "traces", "lookback_hours": 24,
+    "sample_percent": 100, "sample_size": null, "concurrency": 8,
+    "enabled": true, "interval_minutes": 1440, "monthly_budget": 50
+  }'
+
+curl "$LITELLM_URL/engine/$LENS_ID/runs" -X POST \
+  -H "Authorization: Bearer $LITELLM_API_KEY" -H 'Content-Type: application/json' -d '{}'
+
+curl "$LITELLM_URL/engine/$LENS_ID/runs?offset=0" -H "Authorization: Bearer $LITELLM_API_KEY"
+curl "$LITELLM_URL/engine/$LENS_ID/runs/$BATCH_ID" -H "Authorization: Bearer $LITELLM_API_KEY"
+```
+
+Creation queues the first batch. Posting to `/engine/{id}/runs` queues another, or returns the existing active batch. The run response contains its ID under `jobs[0].id`. Poll the batch URL for status, findings and assessments. List responses omit large result payloads; request a batch to retrieve them. Supply an optional complete `settings` object on the runs POST for a one-off override; the saved lens stays unchanged. Selection accepts `team_id`, exact `filters`, and opaque `execution_ids` returned by `/engine/preview/sample`. Preview accepts `offset` and `as_of` to keep the time window fixed while paging. Feedback uses `PATCH /engine/{id}/findings/{finding_id}` with `status` and `reason`
+
+## Quality evaluation
+
+Run the checked-in cases against a configured real model. Expected labels are used only for scoring, never passed to the model. Dev and held-out cases include missing outcomes, failed tools, recovery, handoffs, unsupported claims, repeated work, long evidence and prompt injection. The background option adds clean arithmetic traces to test rare-issue discovery at scale; those repeated synthetic cases do not establish accuracy on every production workload
+
+```bash
+python -m tests.proxy_behavior.lens.evaluate --api-base "$LITELLM_URL" \
+  --model your-model-alias --split all --background 1000 --concurrency 16 \
+  --output /tmp/lens-quality.json
+```
+
+Set `LITELLM_API_KEY` privately. This makes paid model calls. Inspect missed and unexpected per-run labels, final findings and coverage; do not equate a passing dataset with guaranteed detection on arbitrary traces
+
+The worker uses temporary disk space for trace content while reviewing it, and removes those files after each review. The Docker command supplies a writable temporary mount while keeping the application filesystem read-only
+
+To check that accepted behavior stays accepted without hiding new problems, run the evaluator with `--dataset tests/proxy_behavior/lens/feedback_cases.json`. Reports include elapsed time, model call count, reported cost when the proxy provides it, missed checks, unexpected checks, and inconclusive candidates

deploy/lens/compose.build.yaml

@@ -0,0 +1,6 @@
+services:
+  lens-worker:
+    build:
+      context: ../..
+      dockerfile: deploy/lens/Dockerfile
+    image: litellm-lens-worker:local

deploy/lens/compose.yaml

@@ -0,0 +1,12 @@
+services:
+  lens-worker:
+    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:c41e932eaf3e4efbcaf8cc5027c7e93021e5b2823f21cb8785cd107e37b91c9a}
+    environment:
+      LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
+      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
+    restart: unless-stopped
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]

docker/Dockerfile.non_root

@@ -103,7 +103,7 @@ ENV LITELLM_NON_ROOT=true
 
 RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
     cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
-    cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
+    cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \
     touch /var/lib/litellm/ui/.litellm_ui_ready
 
 RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \

docker/docker-compose.tracing.yml

@@ -0,0 +1,62 @@
+name: litellm-tracing
+
+services:
+  litellm:
+    build:
+      context: ..
+      target: runtime
+    command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
+    environment:
+      LITELLM_MASTER_KEY: local-tracing-master-key
+      LITELLM_SALT_KEY: sk-local-tracing-salt-key
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+      CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
+      CLICKHOUSE_READER_URL: http://default:local-tracing@clickhouse:8123
+      CLICKHOUSE_DATABASE: litellm
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
+    volumes:
+      - ./tracing-config.yaml:/app/tracing-config.yaml:ro
+    ports:
+      - "127.0.0.1:4002:4000"
+    depends_on:
+      db:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "127.0.0.1:15432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:26.9.6.6
+    environment:
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_PASSWORD: local-tracing
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    ports:
+      - "127.0.0.1:18123:8123"
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-tracing", "--query", "SELECT 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+volumes:
+  postgres_data:
+  clickhouse_data:

docker/tracing-config.yaml

@@ -0,0 +1,10 @@
+model_list:
+  - model_name: gpt-6.1-sol
+    litellm_params:
+      model: openai/gpt-6.1-sol
+      api_key: os.environ/OPENAI_API_KEY
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  tracing:
+    store: clickhouse

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -22,10 +22,8 @@ from litellm._uuid import uuid
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.management_endpoints.common_utils import (
-    _is_user_team_admin,  # pyright: ignore[reportPrivateUsage]  # shared owner of team-admin membership
-    _set_object_metadata_field,
-)
+from litellm.proxy.management.teams.access import is_team_admin
+from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
 from litellm.proxy.management_helpers.utils import (
     management_endpoint_wrapper,
@@ -117,7 +115,7 @@ async def _check_user_permission_for_project(
         return False
 
     team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
-    return _is_user_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
+    return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
 
 
 async def _validate_team_exists(

gateway/routes/allowlist.py

@@ -73,6 +73,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/v1/containers",
     "/containers",
     "/v1/evals",
+    "/v1/traces",
     "/v1/memory",
     "/queue/chat/",
     # Google data plane (v1beta is the Google AI Studio version)

litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_agent_engine/migration.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_Engine" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "version" INTEGER NOT NULL DEFAULT 0,
+    "data" JSONB NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "LiteLLM_EngineWorker" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "token_hash" TEXT NOT NULL UNIQUE,
+    "data" JSONB NOT NULL
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20261001000000_lens_run_history/migration.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_EngineRun" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "engine_id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL,
+    "data" JSONB NOT NULL
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_EngineRun_engine_id_created_at_idx" ON "LiteLLM_EngineRun"("engine_id", "created_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1894,3 +1894,24 @@ model LiteLLM_WorkflowMessage {
   @@unique([run_id, sequence_number])
   @@index([run_id])
 }
+
+model LiteLLM_Engine {
+  id      String @id
+  version Int @default(0)
+  data    Json
+}
+
+model LiteLLM_EngineRun {
+  id         String @id
+  engine_id  String
+  created_at DateTime
+  data       Json
+
+  @@index([engine_id, created_at])
+}
+
+model LiteLLM_EngineWorker {
+  id         String @id
+  token_hash String @unique
+  data       Json
+}

litellm-rust/Cargo.lock

@@ -1274,6 +1274,18 @@ version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
 
+[[package]]
+name = "const-hex"
+version = "1.19.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.2.17",
+ "proptest",
+ "serde_core",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -2372,9 +2384,9 @@ dependencies = [
  "http-body-util",
  "hyper 1.10.1",
  "lazy_static",
- "opentelemetry",
+ "opentelemetry 0.32.0",
  "opentelemetry-semantic-conventions",
- "opentelemetry_sdk",
+ "opentelemetry_sdk 0.32.1",
  "percent-encoding",
  "pin-project",
  "prost",
@@ -4075,6 +4087,7 @@ dependencies = [
  "litellm-secrets-aws",
  "litellm-secrets-types",
  "litellm-token-counter",
+ "litellm-traces",
  "litellm-tracing",
  "pyo3",
  "pyo3-async-runtimes",
@@ -4351,6 +4364,26 @@ dependencies = [
  "tiktoken-rs",
 ]
 
+[[package]]
+name = "litellm-traces"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "flate2",
+ "litellm-http",
+ "opentelemetry-proto",
+ "prost",
+ "rstest",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "testcontainers-modules",
+ "thiserror 2.0.19",
+ "time",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "litellm-tracing"
 version = "0.1.0"
@@ -4760,6 +4793,33 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "opentelemetry"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "js-sys",
+ "pin-project-lite",
+ "thiserror 2.0.19",
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d"
+dependencies = [
+ "base64 0.22.1",
+ "const-hex",
+ "opentelemetry 0.33.0",
+ "opentelemetry_sdk 0.33.0",
+ "prost",
+ "serde",
+]
+
 [[package]]
 name = "opentelemetry-semantic-conventions"
 version = "0.32.1"
@@ -4775,7 +4835,23 @@ dependencies = [
  "futures-channel",
  "futures-executor",
  "futures-util",
- "opentelemetry",
+ "opentelemetry 0.32.0",
+ "percent-encoding",
+ "portable-atomic",
+ "rand 0.9.5",
+ "thiserror 2.0.19",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520"
+dependencies = [
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "opentelemetry 0.33.0",
  "percent-encoding",
  "portable-atomic",
  "rand 0.9.5",
@@ -5704,6 +5780,7 @@ checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
 dependencies = [
  "base64 0.23.1",
  "bytes",
+ "encoding_rs",
  "futures-core",
  "futures-util",
  "h2 0.4.15",
@@ -5715,6 +5792,7 @@ dependencies = [
  "hyper-util",
  "js-sys",
  "log",
+ "mime",
  "percent-encoding",
  "pin-project-lite",
  "quinn",
@@ -6945,6 +7023,7 @@ dependencies = [
  "memchr",
  "parse-display",
  "pin-project-lite",
+ "reqwest 0.13.5",
  "serde",
  "serde_json",
  "serde_with",
@@ -7505,7 +7584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
 dependencies = [
  "js-sys",
- "opentelemetry",
+ "opentelemetry 0.32.0",
  "tracing",
  "tracing-core",
  "tracing-subscriber",

litellm-rust/Cargo.toml

@@ -12,6 +12,7 @@ repository = "https://github.com/BerriAI/litellm"
 litellm-config = { path = "crates/config" }
 litellm-router = { path = "crates/router" }
 litellm-tracing = { path = "crates/tracing" }
+litellm-traces = { path = "crates/traces" }
 litellm-core = { path = "crates/core" }
 litellm-gateway-mcp = { path = "crates/gateway-mcp" }
 litellm-gateway = { path = "crates/gateway" }

litellm-rust/crates/gateway-ui/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-axum = { workspace = true, features = ["json", "original-uri"] }
+axum = { workspace = true, features = ["json", "original-uri", "query"] }
 axum-login.workspace = true
 base64.workspace = true
 governor = { version = "0.10.4", default-features = false, features = ["std"] }
@@ -16,6 +16,7 @@ rand.workspace = true
 serde.workspace = true
 thiserror.workspace = true
 time.workspace = true
+tower = { version = "0.5", features = ["util"] }
 tower-cookies = "0.11.0"
 tower-http = { version = "0.6.11", features = ["fs", "set-header"] }
 tower-sessions.workspace = true

litellm-rust/crates/gateway-ui/src/dashboard.rs

@@ -1,7 +1,12 @@
 use std::path::Path;
 
-use axum::{Router, routing::get};
-use serde::Serialize;
+use axum::{
+    Router,
+    extract::{Query, Request},
+    routing::get,
+};
+use serde::{Deserialize, Serialize};
+use tower::ServiceExt;
 use tower_http::services::{ServeDir, ServeFile};
 
 #[derive(Serialize)]
@@ -9,9 +14,39 @@ struct Logo {
     logo_url: &'static str,
 }
 
+#[derive(Clone, Copy, Deserialize)]
+#[serde(rename_all = "lowercase")]
+enum Theme {
+    Light,
+    Dark,
+}
+
+#[derive(Clone, Copy, Deserialize)]
+#[serde(rename_all = "lowercase")]
+enum Variant {
+    Full,
+    Monogram,
+}
+
+#[derive(Deserialize)]
+struct LogoQuery {
+    theme: Option<Theme>,
+    variant: Option<Variant>,
+}
+
+fn logo_file(query: &LogoQuery) -> &'static str {
+    match (query.variant, query.theme) {
+        (Some(Variant::Monogram), Some(Theme::Dark)) => "assets/logos/litellm_monogram_dark.svg",
+        (Some(Variant::Monogram), _) => "assets/logos/litellm_monogram.svg",
+        (_, Some(Theme::Dark)) => "assets/logos/litellm_logo_dark.png",
+        _ => "assets/logos/litellm_logo.png",
+    }
+}
+
 pub fn dashboard_assets(directory: impl AsRef<Path>) -> Router {
     let directory = directory.as_ref();
     let assets = ServeDir::new(directory.join("_next")).append_index_html_on_directories(false);
+    let logos = directory.to_path_buf();
 
     crate::static_assets(directory)
         .route(
@@ -22,9 +57,11 @@ pub fn dashboard_assets(directory: impl AsRef<Path>) -> Router {
                 })
             }),
         )
-        .route_service(
+        .route(
             "/get_image",
-            ServeFile::new(directory.join("assets/logos/litellm_logo.jpg")),
+            get(move |Query(query): Query<LogoQuery>, request: Request| {
+                ServeFile::new(logos.join(logo_file(&query))).oneshot(request)
+            }),
         )
         .route_service(
             "/get_favicon",

litellm-rust/crates/gateway-ui/tests/assets.rs

@@ -40,7 +40,14 @@ fn dashboard(directory: TempDir) -> App {
     let export = directory.path().join("public");
     std::fs::create_dir_all(export.join("_next/static")).unwrap();
     std::fs::create_dir_all(export.join("assets/logos")).unwrap();
-    std::fs::write(export.join("assets/logos/litellm_logo.jpg"), "logo bytes").unwrap();
+    for (file, bytes) in [
+        ("litellm_logo.png", "logo bytes"),
+        ("litellm_logo_dark.png", "dark logo bytes"),
+        ("litellm_monogram.svg", "monogram bytes"),
+        ("litellm_monogram_dark.svg", "dark monogram bytes"),
+    ] {
+        std::fs::write(export.join("assets/logos").join(file), bytes).unwrap();
+    }
     std::fs::write(export.join("favicon.ico"), "icon bytes").unwrap();
     std::fs::write(export.join("_next/static/app.js"), "window.app = true;").unwrap();
     App {
@@ -130,7 +137,15 @@ async fn missing_paths_never_fall_back_to_dashboard(app: App, #[case] path: &str
 )]
 #[case::root_assets("/_next/static/app.js", "window.app = true;", "text/javascript")]
 #[case::nested_assets("/ui/_next/static/app.js", "window.app = true;", "text/javascript")]
-#[case::logo("/get_image", "logo bytes", "image/jpeg")]
+#[case::logo("/get_image", "logo bytes", "image/png")]
+#[case::logo_light("/get_image?theme=light", "logo bytes", "image/png")]
+#[case::logo_dark("/get_image?theme=dark", "dark logo bytes", "image/png")]
+#[case::monogram("/get_image?variant=monogram", "monogram bytes", "image/svg+xml")]
+#[case::monogram_dark(
+    "/get_image?theme=dark&variant=monogram",
+    "dark monogram bytes",
+    "image/svg+xml"
+)]
 #[case::favicon("/get_favicon", "icon bytes", "image/x-icon")]
 #[tokio::test]
 async fn dashboard_adapter_preserves_existing_urls(
@@ -184,3 +199,19 @@ async fn logo_discovery_points_to_served_image(dashboard: App) {
         "logo bytes"
     );
 }
+
+#[rstest]
+#[case::logo("/get_image")]
+#[case::logo_dark("/get_image?theme=dark")]
+#[case::monogram("/get_image?variant=monogram")]
+#[case::monogram_dark("/get_image?theme=dark&variant=monogram")]
+#[tokio::test]
+async fn committed_dashboard_export_serves_every_logo(#[case] path: &str) {
+    let export = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../../litellm/proxy/_experimental/out");
+    let response = litellm_gateway_ui::dashboard_assets(export)
+        .oneshot(Request::get(path).body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::OK);
+}

litellm-rust/crates/python-bridge/Cargo.toml

@@ -21,6 +21,7 @@ tiktoken = ["litellm-token-counter/tiktoken"]
 [dependencies]
 fancy-regex.workspace = true
 litellm-tracing.workspace = true
+litellm-traces.workspace = true
 litellm-host.workspace = true
 bytes.workspace = true
 futures-util.workspace = true

litellm-rust/crates/python-bridge/src/lib.rs

@@ -43,6 +43,8 @@ mod _native {
     use crate::routes::responses::{ResponsesWebSocketConnection, aresponses, responses};
     #[pymodule_export]
     use crate::routes::token_counter::TokenCounter;
+    #[pymodule_export]
+    use crate::routes::traces::{NativeTraceStorage, trace_decode_otlp};
     #[cfg(feature = "huggingface")]
     #[pymodule_export]
     use crate::tokenizer::HuggingFaceEncoding;
@@ -107,6 +109,8 @@ mod tests {
                 "aresponses",
                 "ResponsesWebSocketConnection",
                 "NativeDiagnosticProcessor",
+                "NativeTraceStorage",
+                "trace_decode_otlp",
                 "TokenCounter",
                 "Tokenizer",
                 "gil_stats",

litellm-rust/crates/python-bridge/src/routes/mod.rs

@@ -6,6 +6,7 @@ pub(crate) mod messages;
 pub(crate) mod ocr;
 pub(crate) mod responses;
 pub(crate) mod token_counter;
+pub(crate) mod traces;
 
 use litellm_callbacks_legacy_python::LoggingOperation;
 use litellm_callbacks_legacy_python::{LegacyLogging, PublicCall};

litellm-rust/crates/python-bridge/src/routes/traces.rs

@@ -0,0 +1,166 @@
+use std::collections::BTreeMap;
+
+use litellm_http::ClientVariant;
+use litellm_traces::{Connection, Error, InsertTable, Parameter, ReadQuery};
+use pyo3::{
+    exceptions::{PyOverflowError, PyRuntimeError, PyValueError},
+    prelude::*,
+};
+
+fn map_error(error: Error) -> PyErr {
+    match error {
+        Error::InvalidRow
+        | Error::InvalidTable
+        | Error::InvalidSchema
+        | Error::EmptySql
+        | Error::InvalidQuery => PyValueError::new_err(error.to_string()),
+        Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()),
+        Error::InvalidUrl
+        | Error::QueryFailed(_)
+        | Error::InsertFailed(_)
+        | Error::SchemaFailed(_)
+        | Error::ResponseTooLarge
+        | Error::InvalidResponse
+        | Error::Transport => PyRuntimeError::new_err(error.to_string()),
+    }
+}
+
+#[pyclass]
+pub struct NativeTraceStorage {
+    database: String,
+    writer: Connection,
+    reader: Option<Connection>,
+}
+
+#[pymethods]
+impl NativeTraceStorage {
+    #[new]
+    #[pyo3(signature = (database, url, reader_url = None))]
+    fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult<Self> {
+        litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?;
+        Ok(Self {
+            writer: Connection::writer(url).map_err(map_error)?,
+            reader: reader_url
+                .map(|value| Connection::reader(value, &database))
+                .transpose()
+                .map_err(map_error)?,
+            database,
+        })
+    }
+
+    fn ensure_schema<'py>(
+        &self,
+        py: Python<'py>,
+        trace_retention_days: u32,
+        spend_log_retention_days: u32,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.writer.clone();
+        let database = self.database.clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::ensure_schema(
+                    &client,
+                    &connection,
+                    &database,
+                    trace_retention_days,
+                    spend_log_retention_days,
+                )
+                .await
+            },
+            map_error,
+        )
+    }
+
+    fn insert_rows<'py>(
+        &self,
+        py: Python<'py>,
+        table: &str,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec<
+            BTreeMap<String, serde_json::Value>,
+        >,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let table = InsertTable::parse(table).map_err(map_error)?;
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.writer.clone();
+        let database = self.database.clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::insert_rows(&client, &connection, &database, table, rows).await
+            },
+            map_error,
+        )
+    }
+
+    fn lens_query<'py>(
+        &self,
+        py: Python<'py>,
+        name: &str,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap<
+            String,
+            Parameter,
+        >,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let query = litellm_traces::LensQuery::parse(name).map_err(map_error)?;
+        let connection = self.reader.clone().ok_or_else(|| {
+            PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL")
+        })?;
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::execute_read(&client, &connection, query.sql(), &parameters).await
+            },
+            map_error,
+        )
+    }
+
+    fn query<'py>(
+        &self,
+        py: Python<'py>,
+        query: &str,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap<
+            String,
+            Parameter,
+        >,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let query = ReadQuery::parse(query).map_err(map_error)?;
+        let connection = self.reader.clone().ok_or_else(|| {
+            PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL")
+        })?;
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::execute_named_read(&client, &connection, query, &parameters).await
+            },
+            map_error,
+        )
+    }
+}
+
+#[pyfunction]
+pub fn trace_decode_otlp<'py>(
+    py: Python<'py>,
+    body: &[u8],
+    content_type: Option<&str>,
+    content_encoding: Option<&str>,
+    max_decompressed_bytes: usize,
+) -> PyResult<Bound<'py, PyAny>> {
+    let spans = py
+        .detach(|| {
+            litellm_traces::decode_otlp(
+                body,
+                content_type,
+                content_encoding,
+                max_decompressed_bytes,
+            )
+        })
+        .map_err(|error| match error {
+            litellm_traces::DecodeError::TooLarge => PyOverflowError::new_err(error.to_string()),
+            _ => PyValueError::new_err(error.to_string()),
+        })?;
+    litellm_host_python::Pythonized(spans).into_pyobject(py)
+}

litellm-rust/crates/traces/AGENTS.md

@@ -0,0 +1,7 @@
+- Rust owns OTLP wire decoding, ClickHouse schema, row encoding, named reads, connection validation and transport
+- Keep this crate independent of Python; PyO3 conversion and public Python exceptions belong in `python-bridge`
+- Keep the SQL migrations here as the only ClickHouse schema definition
+- Use typed query parameters and a dedicated SELECT-only reader with server-side limits
+- Keep `config/reader.xml` grants on the database the schema is created in (CLICKHOUSE_DATABASE, default `litellm`)
+- Bound insert time and encoded bytes; make retry deduplication behavior explicit for supported ClickHouse versions
+- Test storage behavior through the crate's public API against ClickHouse

litellm-rust/crates/traces/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "litellm-traces"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+flate2.workspace = true
+opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["gen-tonic-messages", "trace", "with-serde"] }
+prost = "0.14.4"
+time = { workspace = true, features = ["formatting"] }
+litellm-http.workspace = true
+sha2.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+url.workspace = true
+
+[dev-dependencies]
+litellm-http = { workspace = true, features = ["test-support"] }
+rstest.workspace = true
+testcontainers-modules = { version = "0.15.0", features = ["clickhouse"] }
+tokio.workspace = true

litellm-rust/crates/traces/config/reader.xml

@@ -0,0 +1,32 @@
+<clickhouse>
+    <profiles>
+        <litellm_traces_reader>
+            <readonly>1</readonly>
+            <max_execution_time>10</max_execution_time>
+            <max_result_rows>1000</max_result_rows>
+            <max_result_bytes>4194304</max_result_bytes>
+            <result_overflow_mode>throw</result_overflow_mode>
+            <max_memory_usage>268435456</max_memory_usage>
+            <constraints>
+                <readonly><readonly/></readonly>
+                <max_execution_time><readonly/></max_execution_time>
+                <max_result_rows><readonly/></max_result_rows>
+                <max_result_bytes><readonly/></max_result_bytes>
+                <result_overflow_mode><readonly/></result_overflow_mode>
+                <max_memory_usage><readonly/></max_memory_usage>
+            </constraints>
+        </litellm_traces_reader>
+    </profiles>
+    <users>
+        <litellm_traces_reader>
+            <password from_env="LITELLM_TRACES_READER_PASSWORD"/>
+            <networks><ip>::/0</ip></networks>
+            <profile>litellm_traces_reader</profile>
+            <grants>
+                <query>GRANT SELECT ON litellm.otel_traces</query>
+                <query>GRANT SELECT ON litellm.agent_traces_by_key</query>
+                <query>GRANT SELECT ON litellm.spend_logs</query>
+            </grants>
+        </litellm_traces_reader>
+    </users>
+</clickhouse>

litellm-rust/crates/traces/migrations/0001_otel_traces.sql

@@ -0,0 +1,47 @@
+CREATE TABLE IF NOT EXISTS {database}.otel_traces
+(
+    Timestamp          DateTime64(9) CODEC(Delta, ZSTD(1)),
+    TraceId            String CODEC(ZSTD(1)),
+    SpanId             String CODEC(ZSTD(1)),
+    ParentSpanId       String CODEC(ZSTD(1)),
+    TraceState         String CODEC(ZSTD(1)),
+    SpanName           LowCardinality(String) CODEC(ZSTD(1)),
+    SpanKind           LowCardinality(String) CODEC(ZSTD(1)),
+    ServiceName        LowCardinality(String) CODEC(ZSTD(1)),
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeName          String CODEC(ZSTD(1)),
+    ScopeVersion       String CODEC(ZSTD(1)),
+    SpanAttributes     Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    Duration           UInt64 CODEC(ZSTD(1)),
+    StatusCode         LowCardinality(String) CODEC(ZSTD(1)),
+    StatusMessage      String CODEC(ZSTD(1)),
+    `Events.Timestamp`  Array(DateTime64(9)) CODEC(ZSTD(1)),
+    `Events.Name`       Array(LowCardinality(String)) CODEC(ZSTD(1)),
+    `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
+    `Links.TraceId`     Array(String) CODEC(ZSTD(1)),
+    `Links.SpanId`      Array(String) CODEC(ZSTD(1)),
+    `Links.TraceState`  Array(String) CODEC(ZSTD(1)),
+    `Links.Attributes`  Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)),
+    TeamId             LowCardinality(String) DEFAULT ResourceAttributes['litellm.team_id'],
+    ApiKeyHash         String DEFAULT ResourceAttributes['litellm.api_key_hash'],
+    ObservationType    LowCardinality(String) DEFAULT multiIf(
+                           ParentSpanId = '', 'agent',
+                           SpanAttributes['gen_ai.operation.name'] = 'invoke_agent', 'agent',
+                           SpanAttributes['gen_ai.operation.name'] IN ('chat', 'text_completion', 'generate_content'), 'llm',
+                           SpanAttributes['gen_ai.operation.name'] = 'execute_tool', 'tool',
+                           'chain'),
+    AgentName          LowCardinality(String) DEFAULT SpanAttributes['gen_ai.agent.name'],
+    LiteLLMRequestId   String DEFAULT SpanAttributes['gen_ai.response.id'],
+    Model              LowCardinality(String) DEFAULT SpanAttributes['gen_ai.request.model'],
+    InputTokens        UInt32 DEFAULT toUInt32OrZero(SpanAttributes['gen_ai.usage.input_tokens']),
+    OutputTokens       UInt32 DEFAULT toUInt32OrZero(SpanAttributes['gen_ai.usage.output_tokens']),
+    Input              String CODEC(ZSTD(3)),
+    Output             String CODEC(ZSTD(3)),
+    InputPreview       String DEFAULT substring(Input, 1, 240),
+    INDEX idx_trace_id TraceId          TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_req_id   LiteLLMRequestId TYPE bloom_filter(0.01)  GRANULARITY 1
+)
+ENGINE = MergeTree
+PARTITION BY toDate(Timestamp)
+ORDER BY (TeamId, ServiceName, toDateTime(Timestamp), TraceId)
+SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000

litellm-rust/crates/traces/migrations/0002_agent_traces.sql

@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS {database}.agent_traces_by_key
+(
+    TeamId        LowCardinality(String),
+    ApiKeyHash    String,
+    TraceId       String,
+    StartTs       SimpleAggregateFunction(min, DateTime64(9)),
+    EndTs         SimpleAggregateFunction(max, DateTime64(9)),
+    ServiceName   SimpleAggregateFunction(any, LowCardinality(String)),
+    RootName      SimpleAggregateFunction(anyLast, Nullable(String)),
+    RootInput     SimpleAggregateFunction(anyLast, Nullable(String)),
+    RootStatus    SimpleAggregateFunction(anyLast, Nullable(String)),
+    SpanCount     SimpleAggregateFunction(sum, UInt64),
+    AgentCount    SimpleAggregateFunction(sum, UInt64),
+    LlmCount      SimpleAggregateFunction(sum, UInt64),
+    ToolCount     SimpleAggregateFunction(sum, UInt64),
+    ErrorCount    SimpleAggregateFunction(sum, UInt64),
+    InputTokens   SimpleAggregateFunction(sum, UInt64),
+    OutputTokens  SimpleAggregateFunction(sum, UInt64),
+    Models        SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    AgentNames    SimpleAggregateFunction(groupUniqArrayArray, Array(String)),
+    RequestIds    SimpleAggregateFunction(groupArrayArray, Array(String))
+)
+ENGINE = AggregatingMergeTree
+ORDER BY (TeamId, ApiKeyHash, TraceId)
+SETTINGS non_replicated_deduplication_window = 1000

litellm-rust/crates/traces/migrations/0003_agent_traces_mv.sql

@@ -0,0 +1,22 @@
+CREATE MATERIALIZED VIEW IF NOT EXISTS {database}.agent_traces_by_key_mv
+TO {database}.agent_traces_by_key AS
+SELECT
+    TeamId, ApiKeyHash, TraceId,
+    min(Timestamp)                                         AS StartTs,
+    max(Timestamp + toIntervalNanosecond(Duration))        AS EndTs,
+    any(ServiceName)                                       AS ServiceName,
+    anyLastIf(toNullable(SpanName), ParentSpanId = '')     AS RootName,
+    anyLastIf(toNullable(InputPreview), ParentSpanId = '') AS RootInput,
+    anyLastIf(toNullable(StatusCode), ParentSpanId = '')   AS RootStatus,
+    count()                                                AS SpanCount,
+    countIf(ObservationType = 'agent')                     AS AgentCount,
+    countIf(ObservationType = 'llm')                       AS LlmCount,
+    countIf(ObservationType = 'tool')                      AS ToolCount,
+    countIf(StatusCode = 'STATUS_CODE_ERROR')              AS ErrorCount,
+    sum(InputTokens)                                       AS InputTokens,
+    sum(OutputTokens)                                      AS OutputTokens,
+    groupUniqArrayIf(toString(Model), Model != '')         AS Models,
+    groupUniqArrayIf(SpanName, ObservationType = 'agent')  AS AgentNames,
+    groupArrayIf(LiteLLMRequestId, LiteLLMRequestId != '') AS RequestIds
+FROM {database}.otel_traces
+GROUP BY TeamId, ApiKeyHash, TraceId

litellm-rust/crates/traces/migrations/0004_spend_logs.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS {database}.spend_logs
+(
+    request_id            String,
+    response_id           String,
+    call_type             LowCardinality(String),
+    api_key               String,
+    key_alias             String,
+    team_id               LowCardinality(String),
+    team_alias            String,
+    organization_id       String,
+    user                  String,
+    end_user              String,
+    model                 LowCardinality(String),
+    model_group           LowCardinality(String),
+    model_id              String,
+    custom_llm_provider   LowCardinality(String),
+    api_base              String,
+    spend                 Float64,
+    prompt_tokens         UInt32,
+    completion_tokens     UInt32,
+    total_tokens          UInt32,
+    cache_read_tokens     UInt32,
+    cache_write_tokens    UInt32,
+    start_time            DateTime64(3),
+    end_time              DateTime64(3),
+    completion_start_time Nullable(DateTime64(3)),
+    status                LowCardinality(String),
+    error_str             String,
+    cache_hit             Bool,
+    session_id            String,
+    trace_id              String,
+    span_id               String,
+    request_tags          Array(String),
+    metadata              String CODEC(ZSTD(3)),
+    messages              String CODEC(ZSTD(3)),
+    response              String CODEC(ZSTD(3)),
+    INDEX idx_response_id response_id TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_trace_id    trace_id    TYPE bloom_filter(0.001) GRANULARITY 1
+)
+ENGINE = ReplacingMergeTree(end_time)
+PARTITION BY toYYYYMM(start_time)
+ORDER BY (team_id, start_time, request_id)

litellm-rust/crates/traces/migrations/0005_otel_traces_ttl.sql

@@ -0,0 +1 @@
+ALTER TABLE {database}.otel_traces MODIFY TTL toDateTime(Timestamp) + INTERVAL {trace_retention_days} DAY

litellm-rust/crates/traces/migrations/0006_agent_traces_ttl.sql

@@ -0,0 +1 @@
+ALTER TABLE {database}.agent_traces_by_key MODIFY TTL toDateTime(StartTs) + INTERVAL {trace_retention_days} DAY

litellm-rust/crates/traces/migrations/0007_spend_logs_ttl.sql

@@ -0,0 +1 @@
+ALTER TABLE {database}.spend_logs MODIFY TTL toDateTime(start_time) + INTERVAL {spend_log_retention_days} DAY

litellm-rust/crates/traces/migrations/0008_trace_received.sql

@@ -0,0 +1 @@
+ALTER TABLE {database}.otel_traces ADD COLUMN IF NOT EXISTS EngineReceivedMs UInt64 DEFAULT 0

litellm-rust/crates/traces/migrations/0009_spend_received.sql

@@ -0,0 +1 @@
+ALTER TABLE {database}.spend_logs ADD COLUMN IF NOT EXISTS EngineReceivedMs UInt64 DEFAULT 0

litellm-rust/crates/traces/query/lens_content.sql

@@ -0,0 +1,35 @@
+WITH greatest(toInt64({offset:UInt32})-1,1) AS content_offset,
+(value, budget) -> if(lengthUTF8(value) <= budget, value,
+    concat(substringUTF8(value, 1, intDiv(budget, 3)), '\n[... content omitted ...]\n',
+        substringUTF8(value, -(budget - intDiv(budget, 3))))) AS excerpt
+SELECT * FROM (
+    SELECT SpanId AS span_id, ParentSpanId AS parent_span_id, SpanName AS name,
+        ObservationType AS kind,
+        if({offset:UInt32}=1 AND lengthUTF8(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage))>8000,
+            concat('Input: ',excerpt(Input,2000),'\nOutput: ',excerpt(Output,5000),
+                '\nStatus: ',StatusCode,' ',excerpt(StatusMessage,500)),
+            substringUTF8(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage),
+                content_offset,8000)) AS content,
+        lengthUTF8(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage))
+            >= content_offset+8000 AS truncated
+    FROM otel_traces WHERE {source:String}='traces'
+      AND ({all_teams:UInt8}=1 OR TeamId={team:String})
+      AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String})
+      AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId > {cursor:String}
+    ORDER BY SpanId LIMIT 1 BY SpanId LIMIT 40
+)
+UNION ALL
+SELECT * FROM (
+    SELECT request_id AS span_id, '' AS parent_span_id, model AS name, 'llm' AS kind,
+        if({offset:UInt32}=1 AND lengthUTF8(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str))>8000,
+            concat('Input: ',excerpt(messages,2000),'\nOutput: ',excerpt(response,5000),'\nError: ',excerpt(error_str,500)),
+            substringUTF8(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str),
+                content_offset,8000)) AS content,
+        lengthUTF8(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str))
+            >= content_offset+8000 AS truncated
+    FROM spend_logs FINAL WHERE {source:String}='requests'
+      AND ({all_teams:UInt8}=1 OR team_id={team:String})
+      AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND request_id={id:String} AND team_id={record_team:String} LIMIT 1
+)

litellm-rust/crates/traces/query/lens_evidence.sql

@@ -0,0 +1,14 @@
+SELECT sum(matches) AS count FROM (
+    SELECT count() AS matches FROM otel_traces WHERE {source:String}='traces'
+      AND ({all_teams:UInt8}=1 OR TeamId={team:String})
+      AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String})
+      AND TraceId={id:String} AND TeamId={record_team:String} AND SpanId={span:String}
+      AND position(concat('Input: ',Input,'\nOutput: ',Output,'\nStatus: ',StatusCode,' ',StatusMessage),{quote:String})>0
+    UNION ALL
+    SELECT count() AS matches FROM spend_logs FINAL WHERE {source:String}='requests'
+      AND ({all_teams:UInt8}=1 OR team_id={team:String})
+      AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND request_id={id:String} AND team_id={record_team:String} AND request_id={span:String}
+      AND position(concat('Input: ',messages,'\nOutput: ',response,'\nError: ',error_str),{quote:String})>0
+)

litellm-rust/crates/traces/query/lens_sample.sql

@@ -0,0 +1,65 @@
+WITH concat(leftPad(toString(cityHash64(concat(source,team_id,trace_ref,trace_id))),20,'0'),
+    hex(concat(source,char(0),team_id,char(0),trace_ref,char(0),trace_id))) AS selection_key
+SELECT *, selection_key FROM (
+    SELECT *, if({sample_cap:UInt64}=0, ceiling(eligible*{sample_percent:Float64}/100),
+        least(toFloat64({sample_cap:UInt64}),ceiling(eligible*{sample_percent:Float64}/100))) AS selected
+    FROM (
+        SELECT *, count() OVER () AS eligible,
+            row_number() OVER (ORDER BY selection_key) AS position
+        FROM (
+    SELECT 'traces' AS source, TraceId AS trace_id, TeamId AS team_id, hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref,
+        coalesce(nullIf(argMin(ResourceAttributes['run.name'], Timestamp), ''),
+            argMin(SpanName, Timestamp)) AS name, toString(min(Timestamp)) AS start_time,
+        uniqExact(SpanId) AS span_count, countIf(ParentSpanId='') > 0 AS root_seen,
+        argMin(ServiceName, Timestamp) AS service,
+        arrayZip(mapKeys(argMin(mapConcat(ResourceAttributes, SpanAttributes), tuple(ParentSpanId!='',Timestamp))),
+            mapValues(argMin(mapConcat(ResourceAttributes, SpanAttributes), tuple(ParentSpanId!='',Timestamp)))) AS attributes
+    FROM otel_traces
+    WHERE {source:String} IN ('traces','both')
+      AND ({all_teams:UInt8}=1 OR TeamId={team:String})
+      AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+      AND (TeamId,ApiKeyHash,TraceId) IN (
+          SELECT TeamId,ApiKeyHash,TraceId FROM otel_traces
+          WHERE ({all_teams:UInt8}=1 OR TeamId={team:String})
+            AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String})
+            AND if(EngineReceivedMs>0,toInt64(EngineReceivedMs),
+                toUnixTimestamp64Milli(Timestamp)+toInt64(intDiv(Duration,1000000))) >= {start:UInt64}
+      )
+    GROUP BY TeamId,ApiKeyHash,TraceId
+    HAVING max(EngineReceivedMs) < {end:UInt64}
+       AND max(toUnixTimestamp64Milli(Timestamp)+toInt64(intDiv(Duration,1000000))) < {end:UInt64}
+       AND countIf(arrayAll((k,v) -> ResourceAttributes[k]=v OR SpanAttributes[k]=v,
+           {filter_keys:Array(String)},{filter_values:Array(String)})
+           AND ({service:String}='' OR ServiceName={service:String})) > 0
+    UNION ALL
+    SELECT 'requests' AS source, request_id AS trace_id, team_id, '' AS trace_ref, model AS name,
+        toString(start_time) AS start_time, toUInt64(1) AS span_count, toUInt8(1) AS root_seen,
+        model_group AS service,
+        arrayConcat(JSONExtractKeysAndValues(metadata, 'requester_metadata', 'String'),
+            arrayMap(t -> tuple('tag', t), request_tags)) AS attributes
+    FROM spend_logs FINAL
+    WHERE {source:String} IN ('requests','both')
+      AND ({all_teams:UInt8}=1 OR team_id={team:String})
+      AND ({key_hash:String}='' OR api_key={key_hash:String})
+      AND if(EngineReceivedMs>0,toInt64(EngineReceivedMs),toUnixTimestamp64Milli(end_time)) >= {start:UInt64}
+      AND EngineReceivedMs < {end:UInt64}
+      AND toUnixTimestamp64Milli(end_time) < {end:UInt64}
+      AND arrayAll((k,v) -> JSONExtractString(metadata,k)=v
+          OR JSONExtractString(metadata,'requester_metadata',k)=v OR (k='tag' AND has(request_tags,v)),
+          {filter_keys:Array(String)},{filter_values:Array(String)})
+      AND ({service:String}='' OR model_group={service:String})
+      AND NOT JSONExtractBool(metadata,'litellm_lens_internal')
+      AND ({source:String}!='both' OR (team_id,api_key,response_id) NOT IN (
+          SELECT TeamId,ApiKeyHash,LiteLLMRequestId FROM otel_traces
+          WHERE ({all_teams:UInt8}=1 OR TeamId={team:String})
+            AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) AND LiteLLMRequestId!=''
+      ))
+)
+WHERE ({selected_team:String}='' OR team_id={selected_team:String})
+  AND (empty({execution_ids:Array(String)}) OR has({execution_ids:Array(String)},
+      concat(source,char(0),team_id,char(0),if(trace_ref='',trace_id,trace_ref))))
+)
+)
+WHERE ({preview:UInt8}=1 OR position <= selected)
+  AND selection_key > {after:String}
+ORDER BY selection_key LIMIT {limit:UInt32} OFFSET {offset:UInt64}

litellm-rust/crates/traces/query/list_traces.sql

@@ -0,0 +1,23 @@
+SELECT TraceId AS trace_id,
+       hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref,
+       TeamId AS team_id, ApiKeyHash AS api_key_hash,
+       ifNull(any(RootName), '') AS name, any(ServiceName) AS service,
+       ifNull(any(RootInput), '') AS input_preview, ifNull(any(RootStatus), '') AS status,
+       toUnixTimestamp64Milli(min(StartTs)) AS start_ms,
+       dateDiff('millisecond', min(StartTs), max(EndTs)) AS duration_ms,
+       sum(SpanCount) AS span_count, length(groupUniqArrayArray(AgentNames)) AS agent_count,
+       sum(AgentCount) AS agent_invocations,
+       sum(LlmCount) AS llm_calls, sum(ToolCount) AS tool_calls,
+       sum(InputTokens) AS input_tokens, sum(OutputTokens) AS output_tokens,
+       groupUniqArrayArray(Models) AS models, sum(ErrorCount) AS error_count,
+       arrayDistinct(groupArrayArray(RequestIds)) AS request_ids
+FROM agent_traces_by_key
+WHERE (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})
+  AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})
+GROUP BY TeamId, ApiKeyHash, TraceId
+HAVING min(StartTs) >= fromUnixTimestamp64Milli({start_ms:Int64})
+   AND min(StartTs) < fromUnixTimestamp64Milli({end_ms:Int64})
+   AND ({cursor_ms:Int64} = 0 OR (toUnixTimestamp64Milli(min(StartTs)), trace_ref)
+        < ({cursor_ms:Int64}, {cursor_trace_id:String}))
+ORDER BY start_ms DESC, trace_ref DESC
+LIMIT {limit:UInt32}

litellm-rust/crates/traces/query/span_detail.sql

@@ -0,0 +1,8 @@
+SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes
+FROM otel_traces
+WHERE TraceId = {trace_id:String} AND SpanId = {span_id:String}
+  AND (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})
+  AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})
+  AND ({trace_ref:String} = '' OR
+       hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) = {trace_ref:String})
+LIMIT 1

litellm-rust/crates/traces/query/spend_by_response_ids.sql

@@ -0,0 +1,9 @@
+SELECT request_id, response_id, team_id, api_key, spend,
+       toUnixTimestamp64Milli(start_time) AS start_ms
+FROM spend_logs FINAL
+WHERE response_id IN {response_ids:Array(String)}
+  AND start_time >= fromUnixTimestamp64Milli({start_ms:Int64})
+  AND start_time < fromUnixTimestamp64Milli({end_ms:Int64})
+  AND (empty({team_ids:Array(String)}) OR team_id IN {team_ids:Array(String)})
+  AND ({api_key_hash:String} = '' OR api_key = {api_key_hash:String})
+ORDER BY start_time DESC

litellm-rust/crates/traces/query/trace_spans.sql

@@ -0,0 +1,16 @@
+SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+       o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status,
+       o.StatusMessage AS status_message,
+       toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns,
+       o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model,
+       o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens,
+       o.LiteLLMRequestId AS litellm_request_id,
+       o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash
+FROM otel_traces AS o
+WHERE o.TraceId = {trace_id:String}
+  AND (empty({team_ids:Array(String)}) OR o.TeamId IN {team_ids:Array(String)})
+  AND ({api_key_hash:String} = '' OR o.ApiKeyHash = {api_key_hash:String})
+  AND ({trace_ref:String} = '' OR
+       hex(SHA256(concat(o.TeamId, char(0), o.ApiKeyHash, char(0), o.TraceId))) = {trace_ref:String})
+ORDER BY o.Timestamp
+LIMIT 1 BY o.SpanId

litellm-rust/crates/traces/src/error.rs

@@ -0,0 +1,37 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("invalid ClickHouse insert row")]
+    InvalidRow,
+    #[error("invalid ClickHouse insert table")]
+    InvalidTable,
+    #[error("invalid ClickHouse HTTP URL")]
+    InvalidUrl,
+    #[error("database must be a nonempty SQL identifier and retention must be positive")]
+    InvalidSchema,
+    #[error("SQL query must not be empty")]
+    EmptySql,
+    #[error("unknown ClickHouse read query")]
+    InvalidQuery,
+    #[error("ClickHouse query failed with HTTP status {0}")]
+    QueryFailed(u16),
+    #[error("ClickHouse insert failed with HTTP status {0}")]
+    InsertFailed(u16),
+    #[error("ClickHouse insert exceeds the encoded size limit")]
+    InsertTooLarge,
+    #[error("ClickHouse schema setup failed with HTTP status {0}")]
+    SchemaFailed(u16),
+    #[error("ClickHouse query exceeded the response size limit")]
+    ResponseTooLarge,
+    #[error("ClickHouse returned an invalid or failed JSON query response")]
+    InvalidResponse,
+    #[error("ClickHouse query transport failed")]
+    Transport,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum DecodeError {
+    #[error("invalid OTLP trace payload")]
+    InvalidPayload,
+    #[error("OTLP trace payload exceeds the decompressed size limit")]
+    TooLarge,
+}

litellm-rust/crates/traces/src/insert.rs

@@ -0,0 +1,188 @@
+use std::{collections::BTreeMap, io::Write, time::Duration};
+
+use flate2::{Compression, write::GzEncoder};
+use litellm_http::Client;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use time::{OffsetDateTime, format_description::well_known::Rfc3339};
+
+use crate::{Connection, Error};
+
+const MAX_INSERT_BYTES: usize = 64 * 1024 * 1024;
+const INSERT_TIMEOUT: Duration = Duration::from_secs(30);
+
+pub enum InsertTable {
+    OtelTraces,
+    SpendLogs,
+}
+
+impl InsertTable {
+    pub fn parse(value: &str) -> Result<Self, Error> {
+        match value {
+            "otel_traces" => Ok(Self::OtelTraces),
+            "spend_logs" => Ok(Self::SpendLogs),
+            _ => Err(Error::InvalidTable),
+        }
+    }
+
+    fn name(&self) -> &'static str {
+        match self {
+            Self::OtelTraces => "otel_traces",
+            Self::SpendLogs => "spend_logs",
+        }
+    }
+}
+
+pub async fn insert_rows(
+    client: &Client,
+    connection: &Connection,
+    database: &str,
+    table: InsertTable,
+    rows: Vec<BTreeMap<String, Value>>,
+) -> Result<(), Error> {
+    if rows.is_empty() {
+        return Ok(());
+    }
+    let token = format!(
+        "{:x}",
+        Sha256::digest(encode_rows_with_limit(rows.clone(), MAX_INSERT_BYTES)?.as_bytes())
+    );
+    let received_ms = OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000;
+    let rows = rows
+        .into_iter()
+        .map(|row| {
+            row.into_iter()
+                .filter(|(key, _)| key != "EngineReceivedMs")
+                .chain(std::iter::once((
+                    "EngineReceivedMs".to_owned(),
+                    Value::from(received_ms as u64),
+                )))
+                .collect()
+        })
+        .collect();
+    let encoded = encode_rows_with_limit(rows, MAX_INSERT_BYTES)?;
+    let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+    encoder
+        .write_all(encoded.as_bytes())
+        .map_err(|_| Error::InvalidRow)?;
+    let body = encoder.finish().map_err(|_| Error::InvalidRow)?;
+    let mut url = connection.url().clone();
+    let existing_pairs: Vec<(String, String)> = url
+        .query_pairs()
+        .filter(|(key, _)| {
+            !matches!(
+                key.as_ref(),
+                "query"
+                    | "async_insert"
+                    | "async_insert_deduplicate"
+                    | "wait_for_async_insert"
+                    | "input_format_skip_unknown_fields"
+                    | "date_time_input_format"
+            )
+        })
+        .map(|(key, value)| (key.into_owned(), value.into_owned()))
+        .collect();
+    url.query_pairs_mut()
+        .clear()
+        .extend_pairs(existing_pairs)
+        .append_pair(
+            "query",
+            &format!(
+                "INSERT INTO `{database}`.{} FORMAT JSONEachRow",
+                table.name()
+            ),
+        )
+        .append_pair("insert_deduplication_token", &token)
+        .append_pair("async_insert", "1")
+        .append_pair("async_insert_deduplicate", "1")
+        .append_pair("wait_for_async_insert", "1")
+        .append_pair("input_format_skip_unknown_fields", "0")
+        .append_pair("date_time_input_format", "best_effort");
+    let response = client
+        .post(url)
+        .timeout(INSERT_TIMEOUT)
+        .header("Content-Encoding", "gzip")
+        .body(body)
+        .send()
+        .await
+        .map_err(|_| Error::Transport)?;
+    if !response.status().is_success() {
+        return Err(Error::InsertFailed(response.status().as_u16()));
+    }
+    Ok(())
+}
+
+pub fn encode_rows(rows: Vec<BTreeMap<String, Value>>) -> Result<String, Error> {
+    encode_rows_with_limit(rows, usize::MAX)
+}
+
+fn encode_rows_with_limit(
+    rows: Vec<BTreeMap<String, Value>>,
+    limit: usize,
+) -> Result<String, Error> {
+    let mut body = Vec::new();
+    for row in rows {
+        let encoded = row
+            .into_iter()
+            .map(|(name, value)| insert_value(&name, value).map(|value| (name, value)))
+            .collect::<Result<BTreeMap<_, _>, _>>()?;
+        let record = serde_json::to_vec(&encoded).map_err(|_| Error::InvalidRow)?;
+        let size = body
+            .len()
+            .checked_add(record.len())
+            .and_then(|size| size.checked_add(usize::from(!body.is_empty())))
+            .ok_or(Error::InsertTooLarge)?;
+        if size > limit {
+            return Err(Error::InsertTooLarge);
+        }
+        if !body.is_empty() {
+            body.push(b'\n');
+        }
+        body.extend_from_slice(&record);
+    }
+    String::from_utf8(body).map_err(|_| Error::InvalidRow)
+}
+
+fn insert_value(name: &str, value: Value) -> Result<Value, Error> {
+    let multiplier = match name {
+        "Timestamp" => 1,
+        "start_time" | "end_time" | "completion_start_time" => 1_000_000,
+        _ => return Ok(value),
+    };
+    if name == "completion_start_time" && value.is_null() {
+        return Ok(value);
+    }
+    let timestamp = value.as_i64().ok_or(Error::InvalidRow)?;
+    let datetime = OffsetDateTime::from_unix_timestamp_nanos(i128::from(timestamp) * multiplier)
+        .map_err(|_| Error::InvalidRow)?;
+    datetime
+        .format(&Rfc3339)
+        .map(Value::String)
+        .map_err(|_| Error::InvalidRow)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeMap;
+
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::encode_rows_with_limit;
+    use crate::Error;
+
+    #[rstest]
+    fn encoded_limit_counts_utf8_bytes_across_rows() {
+        let rows = vec![
+            BTreeMap::from([("Input".to_owned(), json!("雪"))]),
+            BTreeMap::from([("Input".to_owned(), json!("雪"))]),
+        ];
+        let encoded = encode_rows_with_limit(rows.clone(), usize::MAX).expect("valid rows");
+
+        assert!(encode_rows_with_limit(rows.clone(), encoded.len()).is_ok());
+        assert!(matches!(
+            encode_rows_with_limit(rows, encoded.len() - 1),
+            Err(Error::InsertTooLarge)
+        ));
+    }
+}

litellm-rust/crates/traces/src/lib.rs

@@ -0,0 +1,90 @@
+mod error;
+mod insert;
+mod otlp;
+mod schema;
+mod sql;
+
+pub use error::{DecodeError, Error};
+pub use insert::{InsertTable, encode_rows, insert_rows};
+pub use otlp::{DecodedSpan, decode_otlp};
+pub use schema::{ensure_schema, schema_statements};
+pub use sql::{LensQuery, Parameter, ReadQuery, execute_named_read, execute_read};
+use url::Url;
+
+#[derive(Clone)]
+pub struct Connection {
+    url: Url,
+}
+
+impl Connection {
+    pub fn parse(value: &str) -> Result<Self, Error> {
+        let url = Url::parse(value).map_err(|_| Error::InvalidUrl)?;
+        if !matches!(url.scheme(), "http" | "https") || url.host().is_none() {
+            return Err(Error::InvalidUrl);
+        }
+        Ok(Self { url })
+    }
+
+    pub fn configured(
+        url: &str,
+        database: &str,
+        user: &str,
+        password: &str,
+    ) -> Result<Self, Error> {
+        let mut connection = Self::parse(url)?;
+        connection
+            .url
+            .set_username(user)
+            .map_err(|_| Error::InvalidUrl)?;
+        connection
+            .url
+            .set_password(Some(password))
+            .map_err(|_| Error::InvalidUrl)?;
+        let pairs: Vec<_> = connection
+            .url
+            .query_pairs()
+            .filter(|(key, _)| !matches!(key.as_ref(), "database" | "user" | "password"))
+            .map(|(key, value)| (key.into_owned(), value.into_owned()))
+            .collect();
+        connection
+            .url
+            .query_pairs_mut()
+            .clear()
+            .extend_pairs(pairs)
+            .append_pair("database", database);
+        Ok(connection)
+    }
+
+    pub fn writer(url: &str) -> Result<Self, Error> {
+        let mut connection = Self::parse(url)?;
+        let pairs: Vec<_> = connection
+            .url
+            .query_pairs()
+            .filter(|(key, _)| !matches!(key.as_ref(), "database" | "readonly" | "query"))
+            .map(|(key, value)| (key.into_owned(), value.into_owned()))
+            .collect();
+        connection.url.query_pairs_mut().clear().extend_pairs(pairs);
+        Ok(connection)
+    }
+
+    pub fn reader(url: &str, database: &str) -> Result<Self, Error> {
+        let mut connection = Self::parse(url)?;
+        let pairs: Vec<_> = connection
+            .url
+            .query_pairs()
+            .filter(|(key, _)| key != "database")
+            .map(|(key, value)| (key.into_owned(), value.into_owned()))
+            .collect();
+        connection
+            .url
+            .query_pairs_mut()
+            .clear()
+            .extend_pairs(pairs)
+            .append_pair("database", database);
+        Ok(connection)
+    }
+
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+}

litellm-rust/crates/traces/src/otlp.rs

@@ -0,0 +1,221 @@
+use std::{collections::BTreeMap, io::Read};
+
+use base64::Engine;
+use flate2::read::GzDecoder;
+use opentelemetry_proto::tonic::{
+    collector::trace::v1::ExportTraceServiceRequest,
+    common::v1::{AnyValue, KeyValue, any_value::Value as AttributeValue},
+    trace::v1::{Span, span::SpanKind, status::StatusCode},
+};
+use prost::Message;
+use serde::Serialize;
+use serde_json::Value;
+
+use crate::DecodeError;
+
+#[derive(Serialize)]
+pub struct DecodedEvent {
+    pub name: String,
+    pub attributes: BTreeMap<String, String>,
+}
+
+#[derive(Serialize)]
+pub struct DecodedSpan {
+    pub trace_id: String,
+    pub span_id: String,
+    pub parent_span_id: String,
+    pub trace_state: String,
+    pub name: String,
+    pub kind: String,
+    pub resource_attributes: BTreeMap<String, String>,
+    pub scope_name: String,
+    pub scope_version: String,
+    pub attributes: BTreeMap<String, String>,
+    pub start_ns: u64,
+    pub end_ns: u64,
+    pub status_code: String,
+    pub status_message: String,
+    pub events: Vec<DecodedEvent>,
+}
+
+pub fn decode_otlp(
+    body: &[u8],
+    content_type: Option<&str>,
+    content_encoding: Option<&str>,
+    max_decompressed_bytes: usize,
+) -> Result<Vec<DecodedSpan>, DecodeError> {
+    let payload = if content_encoding == Some("gzip") || body.starts_with(&[0x1f, 0x8b]) {
+        let limit = u64::try_from(max_decompressed_bytes).map_err(|_| DecodeError::TooLarge)?;
+        let mut decoded = Vec::new();
+        GzDecoder::new(body)
+            .take(limit + 1)
+            .read_to_end(&mut decoded)
+            .map_err(|_| DecodeError::InvalidPayload)?;
+        decoded
+    } else {
+        body.to_vec()
+    };
+    if payload.len() > max_decompressed_bytes {
+        return Err(DecodeError::TooLarge);
+    }
+    let request = if content_type.is_some_and(|value| value.contains("json")) {
+        let value: Value =
+            serde_json::from_slice(&payload).map_err(|_| DecodeError::InvalidPayload)?;
+        serde_json::from_value(normalize_json_ids(value)?)
+            .map_err(|_| DecodeError::InvalidPayload)?
+    } else {
+        ExportTraceServiceRequest::decode(payload.as_slice())
+            .map_err(|_| DecodeError::InvalidPayload)?
+    };
+    Ok(request
+        .resource_spans
+        .into_iter()
+        .flat_map(|resource_spans| {
+            let resource_attributes = attributes(
+                resource_spans
+                    .resource
+                    .map(|resource| resource.attributes)
+                    .unwrap_or_default(),
+            );
+            resource_spans
+                .scope_spans
+                .into_iter()
+                .flat_map(move |scope_spans| {
+                    let scope = scope_spans.scope.unwrap_or_default();
+                    let resource_attributes = resource_attributes.clone();
+                    scope_spans.spans.into_iter().map(move |span| {
+                        decoded_span(span, &resource_attributes, &scope.name, &scope.version)
+                    })
+                })
+        })
+        .collect())
+}
+
+fn normalize_json_ids(value: Value) -> Result<Value, DecodeError> {
+    match value {
+        Value::Object(fields) => fields
+            .into_iter()
+            .map(|(name, value)| {
+                let normalized = if matches!(name.as_str(), "traceId" | "spanId" | "parentSpanId") {
+                    let encoded = value.as_str().ok_or(DecodeError::InvalidPayload)?;
+                    let bytes = base64::engine::general_purpose::STANDARD
+                        .decode(encoded)
+                        .map_err(|_| DecodeError::InvalidPayload)?;
+                    Value::String(hex_bytes(&bytes))
+                } else if name == "kind" && value.is_string() {
+                    let kind = SpanKind::from_str_name(value.as_str().unwrap_or_default())
+                        .ok_or(DecodeError::InvalidPayload)?;
+                    Value::from(kind as i32)
+                } else if name == "code" && value.is_string() {
+                    let code = StatusCode::from_str_name(value.as_str().unwrap_or_default())
+                        .ok_or(DecodeError::InvalidPayload)?;
+                    Value::from(code as i32)
+                } else {
+                    normalize_json_ids(value)?
+                };
+                Ok((name, normalized))
+            })
+            .collect::<Result<serde_json::Map<_, _>, _>>()
+            .map(Value::Object),
+        Value::Array(values) => values
+            .into_iter()
+            .map(normalize_json_ids)
+            .collect::<Result<Vec<_>, _>>()
+            .map(Value::Array),
+        value => Ok(value),
+    }
+}
+
+fn hex_bytes(bytes: &[u8]) -> String {
+    bytes.iter().map(|byte| format!("{byte:02x}")).collect()
+}
+
+fn decoded_span(
+    span: Span,
+    resource_attributes: &BTreeMap<String, String>,
+    scope_name: &str,
+    scope_version: &str,
+) -> DecodedSpan {
+    let status = span.status.unwrap_or_default();
+    DecodedSpan {
+        trace_id: hex_bytes(&span.trace_id),
+        span_id: hex_bytes(&span.span_id),
+        parent_span_id: hex_bytes(&span.parent_span_id),
+        trace_state: span.trace_state,
+        name: span.name,
+        kind: SpanKind::try_from(span.kind)
+            .unwrap_or(SpanKind::Unspecified)
+            .as_str_name()
+            .to_owned(),
+        resource_attributes: resource_attributes.clone(),
+        scope_name: scope_name.to_owned(),
+        scope_version: scope_version.to_owned(),
+        attributes: attributes(span.attributes),
+        start_ns: span.start_time_unix_nano,
+        end_ns: span.end_time_unix_nano,
+        status_code: StatusCode::try_from(status.code)
+            .unwrap_or(StatusCode::Unset)
+            .as_str_name()
+            .to_owned(),
+        status_message: status.message,
+        events: span
+            .events
+            .into_iter()
+            .map(|event| DecodedEvent {
+                name: event.name,
+                attributes: attributes(event.attributes),
+            })
+            .collect(),
+    }
+}
+
+fn attributes(values: Vec<KeyValue>) -> BTreeMap<String, String> {
+    values
+        .into_iter()
+        .map(|entry| {
+            (
+                entry.key,
+                entry.value.as_ref().map(attribute_text).unwrap_or_default(),
+            )
+        })
+        .collect()
+}
+
+fn attribute_text(value: &AnyValue) -> String {
+    match value.value.as_ref() {
+        Some(AttributeValue::StringValue(value)) => value.clone(),
+        Some(AttributeValue::BoolValue(value)) => value.to_string(),
+        Some(AttributeValue::IntValue(value)) => value.to_string(),
+        Some(AttributeValue::DoubleValue(value)) => {
+            serde_json::to_string(value).unwrap_or_default()
+        }
+        Some(AttributeValue::BytesValue(value)) => String::from_utf8_lossy(value).into_owned(),
+        Some(AttributeValue::ArrayValue(value)) => format!(
+            "[{}]",
+            value
+                .values
+                .iter()
+                .map(|value| serde_json::to_string(&attribute_text(value)).unwrap_or_default())
+                .collect::<Vec<_>>()
+                .join(", ")
+        ),
+        Some(AttributeValue::KvlistValue(value)) => format!(
+            "{{{}}}",
+            value
+                .values
+                .iter()
+                .map(|entry| format!(
+                    "{}: {}",
+                    serde_json::to_string(&entry.key).unwrap_or_default(),
+                    serde_json::to_string(
+                        &entry.value.as_ref().map(attribute_text).unwrap_or_default()
+                    )
+                    .unwrap_or_default()
+                ))
+                .collect::<Vec<_>>()
+                .join(", ")
+        ),
+        Some(AttributeValue::StringValueStrindex(value)) => value.to_string(),
+        None => String::new(),
+    }
+}

litellm-rust/crates/traces/src/schema.rs

@@ -0,0 +1,89 @@
+use litellm_http::Client;
+use std::time::Duration;
+
+use crate::Connection;
+use crate::Error;
+
+const SCHEMA_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
+
+const MIGRATIONS: [&str; 9] = [
+    include_str!("../migrations/0001_otel_traces.sql"),
+    include_str!("../migrations/0002_agent_traces.sql"),
+    include_str!("../migrations/0003_agent_traces_mv.sql"),
+    include_str!("../migrations/0004_spend_logs.sql"),
+    include_str!("../migrations/0005_otel_traces_ttl.sql"),
+    include_str!("../migrations/0006_agent_traces_ttl.sql"),
+    include_str!("../migrations/0007_spend_logs_ttl.sql"),
+    include_str!("../migrations/0008_trace_received.sql"),
+    include_str!("../migrations/0009_spend_received.sql"),
+];
+
+pub fn schema_statements(
+    database: &str,
+    trace_retention_days: u32,
+    spend_log_retention_days: u32,
+) -> Result<Vec<String>, Error> {
+    if database.is_empty()
+        || !database
+            .bytes()
+            .all(|c| c.is_ascii_alphanumeric() || c == b'_')
+        || trace_retention_days == 0
+        || spend_log_retention_days == 0
+    {
+        return Err(Error::InvalidSchema);
+    }
+    let database = format!("`{database}`");
+    Ok(
+        std::iter::once(format!("CREATE DATABASE IF NOT EXISTS {database}"))
+            .chain(MIGRATIONS.iter().map(|sql| {
+                sql.replace("{database}", &database)
+                    .replace("{trace_retention_days}", &trace_retention_days.to_string())
+                    .replace(
+                        "{spend_log_retention_days}",
+                        &spend_log_retention_days.to_string(),
+                    )
+            }))
+            .collect(),
+    )
+}
+
+pub async fn ensure_schema(
+    client: &Client,
+    connection: &Connection,
+    database: &str,
+    trace_retention_days: u32,
+    spend_log_retention_days: u32,
+) -> Result<(), Error> {
+    ensure_schema_with_timeout(
+        client,
+        connection,
+        database,
+        trace_retention_days,
+        spend_log_retention_days,
+        SCHEMA_REQUEST_TIMEOUT,
+    )
+    .await
+}
+
+async fn ensure_schema_with_timeout(
+    client: &Client,
+    connection: &Connection,
+    database: &str,
+    trace_retention_days: u32,
+    spend_log_retention_days: u32,
+    request_timeout: Duration,
+) -> Result<(), Error> {
+    for statement in schema_statements(database, trace_retention_days, spend_log_retention_days)? {
+        let response = client
+            .post(connection.url().clone())
+            .timeout(request_timeout)
+            .body(statement)
+            .send()
+            .await
+            .map_err(|_| Error::Transport)?;
+        if !response.status().is_success() {
+            return Err(Error::SchemaFailed(response.status().as_u16()));
+        }
+    }
+    Ok(())
+}

litellm-rust/crates/traces/src/sql.rs

@@ -0,0 +1,176 @@
+use std::{collections::BTreeMap, time::Duration};
+
+use serde::Deserialize;
+
+use litellm_http::Client;
+
+use crate::{Connection, Error};
+
+const MAX_RESPONSE_BYTES: usize = 4 * 1024 * 1024;
+
+pub enum ReadQuery {
+    ListTraces,
+    TraceSpans,
+    SpanDetail,
+    SpendByResponseIds,
+}
+
+impl ReadQuery {
+    pub fn parse(value: &str) -> Result<Self, Error> {
+        match value {
+            "list_traces" => Ok(Self::ListTraces),
+            "trace_spans" => Ok(Self::TraceSpans),
+            "span_detail" => Ok(Self::SpanDetail),
+            "spend_by_response_ids" => Ok(Self::SpendByResponseIds),
+            _ => Err(Error::InvalidQuery),
+        }
+    }
+
+    fn sql(&self) -> &'static str {
+        match self {
+            Self::ListTraces => include_str!("../query/list_traces.sql"),
+            Self::TraceSpans => include_str!("../query/trace_spans.sql"),
+            Self::SpanDetail => include_str!("../query/span_detail.sql"),
+            Self::SpendByResponseIds => include_str!("../query/spend_by_response_ids.sql"),
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(untagged)]
+pub enum Parameter {
+    Text(String),
+    Integer(i64),
+    Strings(Vec<String>),
+}
+
+impl Parameter {
+    fn encoded(&self) -> String {
+        match self {
+            Self::Text(value) => escaped(value),
+            Self::Integer(value) => value.to_string(),
+            Self::Strings(values) => format!(
+                "[{}]",
+                values
+                    .iter()
+                    .map(|value| format!("'{}'", escaped(value).replace('\'', "\\'")))
+                    .collect::<Vec<_>>()
+                    .join(",")
+            ),
+        }
+    }
+}
+
+fn escaped(value: &str) -> String {
+    value
+        .replace('\\', "\\\\")
+        .replace('\t', "\\t")
+        .replace('\n', "\\n")
+        .replace('\r', "\\r")
+        .replace('\0', "\\0")
+}
+
+pub async fn execute_read(
+    client: &Client,
+    connection: &Connection,
+    sql: &str,
+    parameters: &BTreeMap<String, Parameter>,
+) -> Result<String, Error> {
+    if sql.trim().is_empty() {
+        return Err(Error::EmptySql);
+    }
+
+    let mut url = connection.url().clone();
+
+    let existing_pairs: Vec<(String, String)> = url
+        .query_pairs()
+        .filter(|(key, _)| {
+            !key.starts_with("param_")
+                && !matches!(
+                    key.as_ref(),
+                    "query"
+                        | "readonly"
+                        | "default_format"
+                        | "max_result_rows"
+                        | "result_overflow_mode"
+                        | "max_execution_time"
+                        | "wait_end_of_query"
+                )
+        })
+        .map(|(key, value)| (key.into_owned(), value.into_owned()))
+        .collect();
+    url.query_pairs_mut()
+        .clear()
+        .extend_pairs(existing_pairs)
+        .append_pair("readonly", "1")
+        .append_pair("max_result_rows", "1000")
+        .append_pair("result_overflow_mode", "throw")
+        .append_pair("max_execution_time", "10")
+        .append_pair("wait_end_of_query", "1")
+        .append_pair("default_format", "JSON");
+
+    url.query_pairs_mut().extend_pairs(
+        parameters
+            .iter()
+            .map(|(name, value)| (format!("param_{name}"), value.encoded())),
+    );
+
+    let request = client
+        .post(url)
+        .timeout(Duration::from_secs(15))
+        .body(sql.to_owned());
+    let mut response = request.send().await.map_err(|_| Error::Transport)?;
+    if !response.status().is_success() {
+        return Err(Error::QueryFailed(response.status().as_u16()));
+    }
+
+    let mut body = Vec::new();
+    while let Some(chunk) = response.chunk().await.map_err(|_| Error::Transport)? {
+        if body.len() + chunk.len() > MAX_RESPONSE_BYTES {
+            return Err(Error::ResponseTooLarge);
+        }
+        body.extend_from_slice(&chunk);
+    }
+
+    let json: serde_json::Value =
+        serde_json::from_slice(&body).map_err(|_| Error::InvalidResponse)?;
+    if json.get("exception").is_some() || !json.get("data").is_some_and(serde_json::Value::is_array)
+    {
+        return Err(Error::InvalidResponse);
+    }
+    String::from_utf8(body).map_err(|_| Error::InvalidResponse)
+}
+
+#[derive(Clone, Copy)]
+pub enum LensQuery {
+    Sample,
+    Content,
+    Evidence,
+}
+
+impl LensQuery {
+    pub fn parse(name: &str) -> Result<Self, Error> {
+        match name {
+            "sample" => Ok(Self::Sample),
+            "content" => Ok(Self::Content),
+            "evidence" => Ok(Self::Evidence),
+            _ => Err(Error::InvalidQuery),
+        }
+    }
+    pub fn sql(self) -> &'static str {
+        match self {
+            Self::Sample => include_str!("../query/lens_sample.sql"),
+            Self::Content => include_str!("../query/lens_content.sql"),
+            Self::Evidence => include_str!("../query/lens_evidence.sql"),
+        }
+    }
+}
+
+pub async fn execute_named_read(
+    client: &Client,
+    connection: &Connection,
+    query: ReadQuery,
+    parameters: &BTreeMap<String, Parameter>,
+) -> Result<String, Error> {
+    execute_read(client, connection, query.sql(), parameters).await
+}

litellm-rust/crates/traces/tests/admin_sql.rs

@@ -0,0 +1,273 @@
+use litellm_http::Client;
+use litellm_traces::{Connection, Error, Parameter, execute_read};
+use rstest::{fixture, rstest};
+use serde_json::Value;
+use std::collections::BTreeMap;
+use testcontainers_modules::{
+    clickhouse::ClickHouse,
+    testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner},
+};
+
+const CLICKHOUSE_TAG: &str =
+    "26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e";
+
+struct Database {
+    _container: ContainerAsync<ClickHouse>,
+    url: String,
+    admin_url: String,
+    client: Client,
+}
+
+#[fixture]
+async fn database() -> Result<Database, Box<dyn std::error::Error>> {
+    let container = ClickHouse::default()
+        .with_tag(CLICKHOUSE_TAG)
+        .with_env_var("CLICKHOUSE_SKIP_USER_SETUP", "1")
+        .with_env_var("LITELLM_TRACES_READER_PASSWORD", "test_password")
+        .with_copy_to(
+            "/etc/clickhouse-server/users.d/litellm-traces-reader.xml",
+            include_bytes!("../config/reader.xml").to_vec(),
+        )
+        .start()
+        .await?;
+    let admin_url = format!(
+        "http://{}:{}",
+        container.get_host().await?,
+        container.get_host_port_ipv4(8123).await?,
+    );
+    let client = Client::no_redirect_for_test();
+    for sql in [
+        "CREATE DATABASE litellm",
+        "CREATE TABLE litellm.otel_traces (n UInt8) ENGINE = Memory",
+        "INSERT INTO litellm.otel_traces VALUES (1)",
+        "CREATE TABLE litellm.agent_traces_by_key (n UInt8) ENGINE = Memory",
+        "INSERT INTO litellm.agent_traces_by_key VALUES (4)",
+        "CREATE TABLE litellm.spend_logs (n UInt8) ENGINE = Memory",
+        "INSERT INTO litellm.spend_logs VALUES (3)",
+        "CREATE TABLE litellm.private_traces (n UInt8) ENGINE = Memory",
+        "CREATE TABLE private_traces (n UInt8) ENGINE = Memory",
+    ] {
+        client
+            .post(&admin_url)
+            .body(sql)
+            .send()
+            .await?
+            .error_for_status()?;
+    }
+    let url = format!(
+        "{}?database=litellm",
+        admin_url.replacen("http://", "http://litellm_traces_reader:test_password@", 1)
+    );
+    Ok(Database {
+        _container: container,
+        url,
+        admin_url,
+        client,
+    })
+}
+
+#[rstest]
+#[tokio::test]
+async fn admin_sql_reads_rows_with_enforced_settings(
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    let connection = Connection::parse(&format!(
+        "{}&readonly=0&default_format=TabSeparated&query=SELECT+2",
+        database.url,
+    ))?;
+
+    let result = read(
+        &database.client,
+        &connection,
+        "SELECT n AS answer FROM otel_traces",
+    )
+    .await?;
+    let json: Value = serde_json::from_str(&result)?;
+    assert_eq!(json["data"][0]["answer"], 1);
+
+    let result = read(
+        &database.client,
+        &connection,
+        "SELECT n AS answer FROM agent_traces_by_key",
+    )
+    .await?;
+    let json: Value = serde_json::from_str(&result)?;
+    assert_eq!(json["data"][0]["answer"], 4);
+
+    Ok(())
+}
+
+#[rstest]
+#[case::table("CREATE TABLE admin_sql_test (n UInt8) ENGINE = Memory")]
+#[case::insert("INSERT INTO otel_traces VALUES (2)")]
+#[case::drop("DROP TABLE otel_traces")]
+#[case::named_collection("CREATE NAMED COLLECTION admin_sql_test AS host = 'localhost'")]
+#[case::settings("SET readonly = 0")]
+#[case::inline_settings("SELECT n FROM otel_traces SETTINGS readonly = 0")]
+#[case::time_limit("SELECT n FROM otel_traces SETTINGS max_execution_time = 0")]
+#[case::row_limit("SELECT n FROM otel_traces SETTINGS max_result_rows = 0")]
+#[case::byte_limit("SELECT n FROM otel_traces SETTINGS max_result_bytes = 0")]
+#[case::memory_limit("SELECT n FROM otel_traces SETTINGS max_memory_usage = 0")]
+#[case::other_table("SELECT * FROM private_traces")]
+#[tokio::test]
+async fn reader_rejects_writes_and_privilege_escalation(
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+    #[case] sql: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    let connection = Connection::parse(&format!("{}&readonly=0", database.url))?;
+
+    let result = read(&database.client, &connection, sql).await;
+
+    assert!(matches!(result, Err(Error::QueryFailed(_))), "{result:?}");
+    let rows = read(&database.client, &connection, "SELECT n FROM otel_traces").await?;
+    let json: Value = serde_json::from_str(&rows)?;
+    assert_eq!(json["data"], serde_json::json!([{ "n": 1 }]));
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn admin_sql_rejects_errors_after_output_starts(
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    let connection = Connection::parse(&format!(
+        "{}?max_block_size=1&buffer_size=1&http_write_exception_in_output_format=1\
+         &send_progress_in_http_headers=1&http_headers_progress_interval_ms=0",
+        database.admin_url,
+    ))?;
+
+    let result = read(
+        &database.client,
+        &connection,
+        "SELECT sleepEachRow(0.2), throwIf(number = 2) FROM numbers(5)",
+    )
+    .await;
+
+    assert!(
+        matches!(result, Err(Error::InvalidResponse)),
+        "expected an error embedded in a successful HTTP response: {result:?}"
+    );
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn admin_sql_enforces_result_row_limit(
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    let connection = Connection::parse(&format!(
+        "{}&max_result_rows=0&result_overflow_mode=throw&wait_end_of_query=1",
+        database.url,
+    ))?;
+
+    let result = read(
+        &database.client,
+        &connection,
+        "SELECT number FROM numbers(1001)",
+    )
+    .await;
+
+    assert!(matches!(result, Err(Error::QueryFailed(_))), "{result:?}");
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn admin_sql_enforces_response_byte_limit(
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    let connection = Connection::parse(&database.admin_url)?;
+
+    let result = read(
+        &database.client,
+        &connection,
+        "SELECT repeat('x', 512 * 1024) AS payload FROM numbers(9)",
+    )
+    .await;
+
+    assert!(matches!(result, Err(Error::ResponseTooLarge)), "{result:?}");
+    Ok(())
+}
+
+#[rstest]
+#[case::plain("test_password", "test_password")]
+#[case::encoded("p@ss/word%", "p%40ss%2Fword%25")]
+#[tokio::test]
+async fn admin_sql_authenticates_url_credentials(
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+    #[case] password: &str,
+    #[case] encoded_password: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    database
+        .client
+        .post(&database.admin_url)
+        .body(format!(
+            "CREATE USER sql_reader IDENTIFIED WITH plaintext_password BY '{password}'"
+        ))
+        .send()
+        .await?
+        .error_for_status()?;
+    let connection = Connection::parse(&database.admin_url.replacen(
+        "http://",
+        &format!("http://sql_reader:{encoded_password}@"),
+        1,
+    ))?;
+
+    let result = read(
+        &database.client,
+        &connection,
+        "SELECT currentUser() AS username",
+    )
+    .await?;
+    let json: Value = serde_json::from_str(&result)?;
+
+    assert_eq!(json["data"][0]["username"], "sql_reader");
+
+    Ok(())
+}
+
+async fn read(client: &Client, connection: &Connection, sql: &str) -> Result<String, Error> {
+    execute_read(client, connection, sql, &BTreeMap::new()).await
+}
+
+#[rstest]
+#[case::sql("'; DROP TABLE otel_traces; --")]
+#[case::escapes("back\\slash\ttab\nline\0null")]
+#[tokio::test]
+async fn query_parameters_preserve_values_and_replace_url_parameters(
+    #[case] value: &str,
+    #[future(awt)] database: Result<Database, Box<dyn std::error::Error>>,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let database = database?;
+    let connection = Connection::parse(&format!("{}&param_value=wrong", database.url))?;
+    let values = vec![
+        "a'b".to_owned(),
+        "back\\slash".to_owned(),
+        "line\nbreak".to_owned(),
+        "雪".to_owned(),
+    ];
+    let parameters = BTreeMap::from([
+        ("value".to_owned(), Parameter::Text(value.into())),
+        ("teams".to_owned(), Parameter::Strings(values.clone())),
+        ("number".to_owned(), Parameter::Integer(-42)),
+    ]);
+    let body = execute_read(&database.client, &connection,
+        "SELECT {value:String} AS value, {teams:Array(String)} AS teams, toInt32({number:Int64}) AS number",
+        &parameters).await?;
+    let json: Value = serde_json::from_str(&body)?;
+    assert_eq!(json["data"][0]["value"], value);
+    assert_eq!(json["data"][0]["teams"], serde_json::json!(values));
+    assert_eq!(json["data"][0]["number"], -42);
+    assert!(
+        read(&database.client, &connection, "SELECT n FROM otel_traces")
+            .await
+            .is_ok()
+    );
+    Ok(())
+}

litellm-rust/crates/traces/tests/insert.rs

@@ -0,0 +1,40 @@
+use std::collections::BTreeMap;
+
+use litellm_traces::encode_rows;
+use rstest::rstest;
+use serde_json::{Value, json};
+
+#[rstest]
+#[case::span("Timestamp", json!(1_234_567_890), json!("1970-01-01T00:00:01.23456789Z"))]
+#[case::start("start_time", json!(1_234), json!("1970-01-01T00:00:01.234Z"))]
+#[case::end("end_time", json!(2_345), json!("1970-01-01T00:00:02.345Z"))]
+#[case::completion("completion_start_time", json!(1_345), json!("1970-01-01T00:00:01.345Z"))]
+#[case::absent_completion("completion_start_time", Value::Null, Value::Null)]
+#[case::before_epoch("Timestamp", json!(-1), json!("1969-12-31T23:59:59.999999999Z"))]
+fn insert_encoding_preserves_timestamp_precision_and_other_fields(
+    #[case] field: &str,
+    #[case] value: Value,
+    #[case] expected: Value,
+) {
+    let rows = vec![BTreeMap::from([
+        (field.to_owned(), value),
+        ("SpanAttributes".into(), json!({"message": "a\nb\\c\"雪"})),
+        ("InputTokens".into(), json!(42)),
+    ])];
+    let encoded = encode_rows(rows).expect("valid row");
+    let actual: Value = serde_json::from_str(&encoded).expect("JSONEachRow record");
+    assert_eq!(
+        actual,
+        json!({
+            field: expected, "SpanAttributes": {"message": "a\nb\\c\"雪"}, "InputTokens": 42
+        })
+    );
+}
+
+#[rstest]
+#[case::fractional(json!(1.25))]
+#[case::out_of_range(json!(u64::MAX))]
+#[case::null(Value::Null)]
+fn insert_encoding_rejects_invalid_span_timestamps(#[case] timestamp: Value) {
+    assert!(encode_rows(vec![BTreeMap::from([("Timestamp".into(), timestamp)])]).is_err());
+}

litellm-rust/crates/traces/tests/migrations.rs

@@ -0,0 +1,837 @@
+use std::{collections::BTreeMap, time::Duration};
+
+use litellm_http::Client;
+use litellm_traces::{
+    Connection, Error, InsertTable, Parameter, ReadQuery, encode_rows, ensure_schema,
+    execute_named_read, execute_read, schema_statements,
+};
+use rstest::{fixture, rstest};
+use testcontainers_modules::{
+    clickhouse::ClickHouse,
+    testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner},
+};
+
+const CLICKHOUSE_TAG: &str =
+    "26.9.6.6@sha256:eb4870e7ca7ed70c259eebfcfbee6cf797017f6b5436c2926bbbfe3d4d28486e";
+
+type TestResult<T = ()> = Result<T, Box<dyn std::error::Error>>;
+
+struct ClickHouseDatabase {
+    _container: ContainerAsync<ClickHouse>,
+    url: String,
+    client: Client,
+}
+
+#[fixture]
+async fn database() -> TestResult<ClickHouseDatabase> {
+    let container = ClickHouse::default()
+        .with_tag(CLICKHOUSE_TAG)
+        .with_env_var("CLICKHOUSE_SKIP_USER_SETUP", "1")
+        .start()
+        .await?;
+    let url = format!(
+        "http://{}:{}",
+        container.get_host().await?,
+        container.get_host_port_ipv4(8123).await?
+    );
+    Ok(ClickHouseDatabase {
+        _container: container,
+        url,
+        client: Client::no_redirect_for_test(),
+    })
+}
+
+async fn insert_rows(
+    database: &ClickHouseDatabase,
+    table: &str,
+    rows: Vec<BTreeMap<String, serde_json::Value>>,
+) -> TestResult {
+    database
+        .client
+        .post(&database.url)
+        .query(&[
+            (
+                "query",
+                format!("INSERT INTO trace_test.{table} FORMAT JSONEachRow"),
+            ),
+            ("date_time_input_format", "best_effort".into()),
+        ])
+        .body(encode_rows(rows)?)
+        .send()
+        .await?
+        .error_for_status()?;
+    Ok(())
+}
+
+async fn execute_write(database: &ClickHouseDatabase, sql: &str) -> TestResult {
+    database
+        .client
+        .post(&database.url)
+        .body(sql.to_owned())
+        .send()
+        .await?
+        .error_for_status()?;
+    Ok(())
+}
+
+async fn read_json(database: &ClickHouseDatabase, sql: &str) -> TestResult<serde_json::Value> {
+    let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let body = execute_read(&database.client, &connection, sql, &BTreeMap::new()).await?;
+    Ok(serde_json::from_str(&body)?)
+}
+
+async fn table_rows(database: &ClickHouseDatabase, table: &str) -> TestResult<u64> {
+    let response = read_json(
+        database,
+        &format!("SELECT count() AS rows FROM trace_test.{table}"),
+    )
+    .await?;
+    Ok(response["data"][0]["rows"]
+        .as_u64()
+        .expect("ClickHouse returns row counts as unsigned integers"))
+}
+
+async fn mutation_rows(database: &ClickHouseDatabase) -> TestResult<u64> {
+    let response = read_json(
+        database,
+        "SELECT count() AS rows FROM system.mutations WHERE database = 'trace_test'",
+    )
+    .await?;
+    Ok(response["data"][0]["rows"]
+        .as_u64()
+        .expect("ClickHouse returns mutation counts as unsigned integers"))
+}
+
+#[rstest]
+#[tokio::test]
+async fn schema_supports_span_rollups_and_spend_joins(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
+    let span = serde_json::from_value(serde_json::json!({
+        "Timestamp": timestamp, "TraceId": "trace-1", "SpanId": "span-1", "ParentSpanId": "",
+        "ServiceName": "proxy", "SpanName": "request", "Input": "hello world",
+        "ResourceAttributes": {"litellm.team_id": "team-1", "litellm.api_key_hash": "hash-1"},
+        "SpanAttributes": {"gen_ai.response.id": "response-1", "gen_ai.usage.input_tokens": "12"}
+    }))?;
+    let spend = serde_json::from_value(serde_json::json!({
+        "request_id": "request-1", "response_id": "response-1", "team_id": "team-1", "spend": 0.125,
+        "start_time": timestamp / 1_000_000, "end_time": timestamp / 1_000_000 + 100,
+        "completion_start_time": null
+    }))?;
+    insert_rows(&database, "otel_traces", vec![span]).await?;
+    insert_rows(&database, "spend_logs", vec![spend]).await?;
+    let reader = Connection::reader(&database.url, "trace_test")?;
+    let list_parameters = BTreeMap::from([
+        ("team_ids".into(), Parameter::Strings(vec!["team-1".into()])),
+        ("api_key_hash".into(), Parameter::Text(String::new())),
+        (
+            "start_ms".into(),
+            Parameter::Integer(timestamp / 1_000_000 - 1000),
+        ),
+        (
+            "end_ms".into(),
+            Parameter::Integer(timestamp / 1_000_000 + 1000),
+        ),
+        ("cursor_ms".into(), Parameter::Integer(0)),
+        ("cursor_trace_id".into(), Parameter::Text(String::new())),
+        ("limit".into(), Parameter::Integer(10)),
+    ]);
+    let listed: serde_json::Value = serde_json::from_str(
+        &execute_named_read(
+            &database.client,
+            &reader,
+            ReadQuery::ListTraces,
+            &list_parameters,
+        )
+        .await?,
+    )?;
+    assert_eq!(
+        listed["data"][0]["request_ids"],
+        serde_json::json!(["response-1"])
+    );
+    let spend_parameters = BTreeMap::from([
+        (
+            "response_ids".into(),
+            Parameter::Strings(vec!["response-1".into()]),
+        ),
+        ("team_ids".into(), Parameter::Strings(vec!["team-1".into()])),
+        ("api_key_hash".into(), Parameter::Text(String::new())),
+        (
+            "start_ms".into(),
+            Parameter::Integer(timestamp / 1_000_000 - 1000),
+        ),
+        (
+            "end_ms".into(),
+            Parameter::Integer(timestamp / 1_000_000 + 1000),
+        ),
+    ]);
+    let matched: serde_json::Value = serde_json::from_str(
+        &execute_named_read(
+            &database.client,
+            &reader,
+            ReadQuery::SpendByResponseIds,
+            &spend_parameters,
+        )
+        .await?,
+    )?;
+    assert_eq!(matched["data"][0]["spend"], 0.125);
+    let body = read_json(
+        &database,
+        "SELECT o.TeamId, o.ApiKeyHash, o.ObservationType, o.InputPreview, s.spend, \
+         toString(toUnixTimestamp64Nano(o.Timestamp)) AS timestamp_ns, \
+         toString(toUnixTimestamp64Milli(s.start_time)) AS start_ms \
+         FROM trace_test.otel_traces o JOIN trace_test.spend_logs s \
+         ON o.LiteLLMRequestId = s.response_id AND o.TeamId = s.team_id",
+    )
+    .await?;
+    assert_eq!(
+        body["data"],
+        serde_json::json!([{
+            "TeamId": "team-1", "ApiKeyHash": "hash-1", "ObservationType": "agent",
+            "InputPreview": "hello world", "spend": 0.125,
+            "timestamp_ns": timestamp.to_string(), "start_ms": (timestamp / 1_000_000).to_string()
+        }])
+    );
+    let body = read_json(
+        &database,
+        "SELECT toUInt32(sum(SpanCount)) AS spans, toUInt32(sum(InputTokens)) AS tokens \
+         FROM trace_test.agent_traces_by_key WHERE TeamId = 'team-1' AND TraceId = 'trace-1'",
+    )
+    .await?;
+    assert_eq!(
+        body["data"],
+        serde_json::json!([{"spans": 1, "tokens": 12}])
+    );
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn insert_rejects_unknown_columns_even_if_url_requests_skipping_them(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&format!(
+        "{}?input_format_skip_unknown_fields=1",
+        database.url
+    ))?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let row = BTreeMap::from([
+        (
+            "Timestamp".to_owned(),
+            serde_json::json!(1_700_000_000_000_000_000_i64),
+        ),
+        (
+            "unexpected".to_owned(),
+            serde_json::json!("dropped silently"),
+        ),
+    ]);
+
+    assert!(matches!(
+        litellm_traces::insert_rows(
+            &database.client,
+            &writer,
+            "trace_test",
+            InsertTable::OtelTraces,
+            vec![row]
+        )
+        .await,
+        Err(Error::InsertFailed(_))
+    ));
+    assert_eq!(table_rows(&database, "otel_traces").await?, 0);
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn retried_trace_insert_does_not_inflate_rollup(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let row: BTreeMap<String, serde_json::Value> = serde_json::from_value(serde_json::json!({
+        "Timestamp": time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64,
+        "TraceId": "retried-trace", "SpanId": "span-1", "ParentSpanId": "",
+        "TeamId": "team-1", "ApiKeyHash": "key-1", "SpanName": "root", "InputTokens": 7
+    }))?;
+    for _ in 0..2 {
+        litellm_traces::insert_rows(
+            &database.client,
+            &writer,
+            "trace_test",
+            InsertTable::OtelTraces,
+            vec![row.clone()],
+        )
+        .await?;
+    }
+    let counts = read_json(
+        &database,
+        "SELECT toUInt32(sum(SpanCount)) AS spans, toUInt32(sum(InputTokens)) AS tokens \
+         FROM trace_test.agent_traces_by_key WHERE TraceId = 'retried-trace'",
+    )
+    .await?;
+    assert_eq!(table_rows(&database, "otel_traces").await?, 1);
+    assert_eq!(counts["data"][0]["spans"], 1);
+    assert_eq!(counts["data"][0]["tokens"], 7);
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn keyed_rollup_keeps_same_trace_ids_separate_by_api_key(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
+    let rows = vec![
+        serde_json::from_value(serde_json::json!({
+            "Timestamp": timestamp, "TraceId": "shared-id", "SpanId": "root-one",
+            "ParentSpanId": "", "SpanName": "root-one", "Input": "private-one",
+            "ResourceAttributes": {"litellm.api_key_hash": "key-one"}
+        }))?,
+        serde_json::from_value(serde_json::json!({
+            "Timestamp": timestamp, "TraceId": "shared-id", "SpanId": "root-two",
+            "ParentSpanId": "", "SpanName": "root-two", "Input": "private-two",
+            "ResourceAttributes": {"litellm.api_key_hash": "key-two"}
+        }))?,
+    ];
+    insert_rows(&database, "otel_traces", rows).await?;
+    execute_write(
+        &database,
+        "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL",
+    )
+    .await?;
+    let rows = read_json(
+        &database,
+        "SELECT ApiKeyHash, any(RootInput) AS RootInput \
+         FROM trace_test.agent_traces_by_key WHERE TraceId = 'shared-id' \
+         GROUP BY ApiKeyHash ORDER BY ApiKeyHash",
+    )
+    .await?;
+    assert_eq!(
+        rows["data"],
+        serde_json::json!([
+            {"ApiKeyHash": "key-one", "RootInput": "private-one"},
+            {"ApiKeyHash": "key-two", "RootInput": "private-two"}
+        ])
+    );
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn rollup_merges_spans_across_days_without_losing_root_fields(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let day_start = time::OffsetDateTime::now_utc()
+        .replace_time(time::Time::MIDNIGHT)
+        .unix_timestamp_nanos() as i64;
+    let root = serde_json::from_value(serde_json::json!({
+        "Timestamp": day_start - 1_000_000_000, "TraceId": "cross-day", "SpanId": "span-root",
+        "ParentSpanId": "", "ServiceName": "proxy", "SpanName": "root", "Input": "root input",
+        "StatusCode": "STATUS_CODE_ERROR",
+        "ResourceAttributes": {"litellm.team_id": "team-1"}
+    }))?;
+    insert_rows(&database, "otel_traces", vec![root]).await?;
+    let child = serde_json::from_value(serde_json::json!({
+        "Timestamp": day_start + 1_000_000_000, "TraceId": "cross-day", "SpanId": "span-child",
+        "ParentSpanId": "span-root", "ServiceName": "proxy", "SpanName": "child",
+        "StatusCode": "STATUS_CODE_UNSET",
+        "ResourceAttributes": {"litellm.team_id": "team-1"}
+    }))?;
+    insert_rows(&database, "otel_traces", vec![child]).await?;
+    execute_write(
+        &database,
+        "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL",
+    )
+    .await?;
+    let response = read_json(
+        &database,
+        "SELECT count() AS rows, any(RootName) AS RootName, any(RootInput) AS RootInput, \
+         any(RootStatus) AS RootStatus, sum(SpanCount) AS SpanCount \
+         FROM trace_test.agent_traces_by_key",
+    )
+    .await?;
+    assert_eq!(
+        response["data"],
+        serde_json::json!([{
+            "rows": 1, "RootName": "root", "RootInput": "root input",
+            "RootStatus": "STATUS_CODE_ERROR", "SpanCount": 2
+        }])
+    );
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn spend_deduplication_preserves_subsecond_requests_and_retries(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let now_ms = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64 / 1_000_000;
+    let base_start_time = now_ms / 1000 * 1000;
+    let first_start_time = base_start_time + 100;
+    let second_start_time = base_start_time + 200;
+    let first = serde_json::from_value(serde_json::json!({
+        "request_id": "same-request", "team_id": "team-1", "spend": 1.0,
+        "start_time": first_start_time, "end_time": first_start_time + 1000
+    }))?;
+    let second = serde_json::from_value(serde_json::json!({
+        "request_id": "same-request", "team_id": "team-1", "spend": 2.0,
+        "start_time": second_start_time, "end_time": second_start_time + 1200
+    }))?;
+    let retry = serde_json::from_value(serde_json::json!({
+        "request_id": "same-request", "team_id": "team-1", "spend": 1.0,
+        "start_time": first_start_time, "end_time": first_start_time + 2000
+    }))?;
+    insert_rows(&database, "spend_logs", vec![first]).await?;
+    insert_rows(&database, "spend_logs", vec![second]).await?;
+    insert_rows(&database, "spend_logs", vec![retry]).await?;
+    execute_write(&database, "OPTIMIZE TABLE trace_test.spend_logs FINAL").await?;
+    let rows = read_json(
+        &database,
+        "SELECT toString(toUnixTimestamp64Milli(start_time)) AS start_time, \
+         toString(toUnixTimestamp64Milli(end_time)) AS end_time \
+         FROM trace_test.spend_logs ORDER BY start_time",
+    )
+    .await?;
+    assert_eq!(
+        rows["data"],
+        serde_json::json!([
+            {
+                "start_time": first_start_time.to_string(),
+                "end_time": (first_start_time + 2000).to_string()
+            },
+            {
+                "start_time": second_start_time.to_string(),
+                "end_time": (second_start_time + 1200).to_string()
+            }
+        ])
+    );
+    assert_eq!(table_rows(&database, "spend_logs").await?, 2);
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn retention_changes_materialize_existing_rows_and_remain_idempotent(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 30, 30).await?;
+    let old_time = time::OffsetDateTime::now_utc() - time::Duration::days(20);
+    let old_timestamp_ns = old_time.unix_timestamp_nanos() as i64;
+    let old_timestamp_ms = old_timestamp_ns / 1_000_000;
+    let span = serde_json::from_value(serde_json::json!({
+        "Timestamp": old_timestamp_ns, "TraceId": "expired", "SpanId": "span-old",
+        "ParentSpanId": "", "ServiceName": "proxy", "SpanName": "old-root", "Input": "old input",
+        "ResourceAttributes": {"litellm.team_id": "team-1"}
+    }))?;
+    let spend = serde_json::from_value(serde_json::json!({
+        "request_id": "old-request", "team_id": "team-1", "spend": 1.0,
+        "start_time": old_timestamp_ms, "end_time": old_timestamp_ms + 1000
+    }))?;
+    insert_rows(&database, "otel_traces", vec![span]).await?;
+    insert_rows(&database, "spend_logs", vec![spend]).await?;
+    assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 1);
+    ensure_schema(&database.client, &writer, "trace_test", 14, 14).await?;
+    let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
+    loop {
+        let response = read_json(
+            &database,
+            "SELECT countIf(is_done = 0) AS pending \
+             FROM system.mutations WHERE database = 'trace_test'",
+        )
+        .await?;
+        let pending = response["data"][0]["pending"]
+            .as_u64()
+            .expect("ClickHouse returns pending mutation counts as unsigned integers");
+        if pending == 0 {
+            break;
+        }
+        assert!(
+            tokio::time::Instant::now() < deadline,
+            "ClickHouse TTL mutations did not finish before the deadline"
+        );
+        tokio::time::sleep(Duration::from_millis(100)).await;
+    }
+    execute_write(&database, "OPTIMIZE TABLE trace_test.otel_traces FINAL").await?;
+    execute_write(
+        &database,
+        "OPTIMIZE TABLE trace_test.agent_traces_by_key FINAL",
+    )
+    .await?;
+    execute_write(&database, "OPTIMIZE TABLE trace_test.spend_logs FINAL").await?;
+    assert_eq!(table_rows(&database, "otel_traces").await?, 0);
+    assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 0);
+    assert_eq!(table_rows(&database, "spend_logs").await?, 0);
+    let mutation_count = mutation_rows(&database).await?;
+    ensure_schema(&database.client, &writer, "trace_test", 14, 14).await?;
+    assert_eq!(mutation_rows(&database).await?, mutation_count);
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn schema_statement_timeout_maps_to_transport_error() -> TestResult {
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+    let address = listener.local_addr()?;
+    let server = tokio::spawn(async move {
+        let (_connection, _) = listener.accept().await.expect("accept schema request");
+        std::future::pending::<()>().await;
+    });
+    let client = Client::no_redirect_for_test();
+    let url = format!("http://{address}");
+    let writer = Connection::writer(&url)?;
+    let result = tokio::time::timeout(
+        Duration::from_secs(35),
+        ensure_schema(&client, &writer, "trace_test", 7, 14),
+    )
+    .await;
+    server.abort();
+    assert!(matches!(result, Ok(Err(Error::Transport))), "{result:?}");
+    Ok(())
+}
+
+#[rstest]
+#[case::empty("", 7, 14)]
+#[case::sql("db; DROP DATABASE default", 7, 14)]
+#[case::trace_retention("traces", 0, 14)]
+#[case::spend_retention("traces", 7, 0)]
+fn schema_rejects_invalid_configuration(
+    #[case] database: &str,
+    #[case] traces: u32,
+    #[case] spend: u32,
+) {
+    assert!(schema_statements(database, traces, spend).is_err());
+}
+
+#[rstest]
+#[tokio::test]
+async fn lens_filters_reads_and_evidence_keep_reused_trace_ids_separate(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    use litellm_traces::{LensQuery, Parameter};
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64;
+    for (key, text) in [("one", "timeout"), ("two", "success")] {
+        insert_rows(&database, "otel_traces", vec![serde_json::from_value(serde_json::json!({
+            "Timestamp": timestamp, "TraceId": "shared", "SpanId": "root", "ParentSpanId": "",
+            "ServiceName": "review", "SpanName": "release", "Input": text,
+            "ResourceAttributes": {"litellm.team_id": "team", "litellm.api_key_hash": key, "swarm": "release"}
+        }))?]).await?;
+    }
+    let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let parameters = BTreeMap::from([
+        ("source".into(), Parameter::Text("traces".into())),
+        ("all_teams".into(), Parameter::Integer(1)),
+        ("team".into(), Parameter::Text(String::new())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        (
+            "start".into(),
+            Parameter::Integer(timestamp / 1_000_000 - 1000),
+        ),
+        (
+            "end".into(),
+            Parameter::Integer(timestamp / 1_000_000 + 1000),
+        ),
+        ("service".into(), Parameter::Text("review".into())),
+        (
+            "filter_keys".into(),
+            Parameter::Strings(vec!["swarm".into()]),
+        ),
+        (
+            "filter_values".into(),
+            Parameter::Strings(vec!["release".into()]),
+        ),
+        ("limit".into(), Parameter::Integer(10)),
+        ("offset".into(), Parameter::Integer(0)),
+        ("after".into(), Parameter::Text(String::new())),
+        ("sample_percent".into(), Parameter::Text("100".into())),
+        ("sample_cap".into(), Parameter::Integer(0)),
+        ("preview".into(), Parameter::Integer(0)),
+        ("selected_team".into(), Parameter::Text(String::new())),
+        ("execution_ids".into(), Parameter::Strings(vec![])),
+    ]);
+    let sample: serde_json::Value = serde_json::from_str(
+        &execute_read(
+            &database.client,
+            &connection,
+            LensQuery::Sample.sql(),
+            &parameters,
+        )
+        .await?,
+    )?;
+    let rows = sample["data"].as_array().expect("sample rows");
+    assert_eq!(rows.len(), 2);
+    assert_ne!(rows[0]["trace_ref"], rows[1]["trace_ref"]);
+    let first_ref = rows[0]["trace_ref"].as_str().expect("reference");
+    let read_parameters: BTreeMap<_, _> = parameters
+        .into_iter()
+        .chain([
+            ("id".into(), Parameter::Text("shared".into())),
+            ("record_team".into(), Parameter::Text("team".into())),
+            ("trace_ref".into(), Parameter::Text(first_ref.into())),
+            ("cursor".into(), Parameter::Text(String::new())),
+            ("offset".into(), Parameter::Integer(1)),
+            ("span".into(), Parameter::Text("root".into())),
+        ])
+        .collect();
+    let content: serde_json::Value = serde_json::from_str(
+        &execute_read(
+            &database.client,
+            &connection,
+            LensQuery::Content.sql(),
+            &read_parameters,
+        )
+        .await?,
+    )?;
+    assert_eq!(content["data"].as_array().map(Vec::len), Some(1));
+    let text = content["data"][0]["content"].as_str().expect("content");
+    let opposite = if text.contains("timeout") {
+        "success"
+    } else {
+        "timeout"
+    };
+    let evidence_parameters = read_parameters
+        .into_iter()
+        .chain([("quote".into(), Parameter::Text(opposite.into()))])
+        .collect();
+    let evidence: serde_json::Value = serde_json::from_str(
+        &execute_read(
+            &database.client,
+            &connection,
+            LensQuery::Evidence.sql(),
+            &evidence_parameters,
+        )
+        .await?,
+    )?;
+    assert_eq!(evidence["data"][0]["count"], 0);
+    Ok(())
+}
+
+#[rstest]
+#[tokio::test]
+async fn lens_request_sample_does_not_trust_caller_tags(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    use litellm_traces::{LensQuery, Parameter};
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    ensure_schema(&database.client, &writer, "trace_test", 7, 14).await?;
+    let timestamp = time::OffsetDateTime::now_utc().unix_timestamp_nanos() as i64 / 1_000_000;
+    for (id, internal) in [("external", false), ("internal", true)] {
+        let row = serde_json::from_value(serde_json::json!({
+            "request_id": id, "team_id": "team", "start_time": timestamp, "end_time": timestamp,
+            "request_tags": ["litellm-engine"],
+            "metadata": serde_json::json!({"litellm_lens_internal": internal}).to_string()
+        }))?;
+        insert_rows(&database, "spend_logs", vec![row]).await?;
+    }
+    let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let parameters = BTreeMap::from([
+        ("source".into(), Parameter::Text("requests".into())),
+        ("all_teams".into(), Parameter::Integer(1)),
+        ("team".into(), Parameter::Text(String::new())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        ("start".into(), Parameter::Integer(timestamp - 1000)),
+        ("end".into(), Parameter::Integer(timestamp + 60000)),
+        ("service".into(), Parameter::Text(String::new())),
+        ("filter_keys".into(), Parameter::Strings(vec![])),
+        ("filter_values".into(), Parameter::Strings(vec![])),
+        ("limit".into(), Parameter::Integer(10)),
+        ("offset".into(), Parameter::Integer(0)),
+        ("after".into(), Parameter::Text(String::new())),
+        ("sample_percent".into(), Parameter::Text("100".into())),
+        ("sample_cap".into(), Parameter::Integer(0)),
+        ("preview".into(), Parameter::Integer(0)),
+        ("selected_team".into(), Parameter::Text(String::new())),
+        ("execution_ids".into(), Parameter::Strings(vec![])),
+    ]);
+    let sample: serde_json::Value = serde_json::from_str(
+        &execute_read(
+            &database.client,
+            &connection,
+            LensQuery::Sample.sql(),
+            &parameters,
+        )
+        .await?,
+    )?;
+    let rows = sample["data"].as_array().expect("sample rows");
+    assert_eq!(rows.len(), 1);
+    assert_eq!(rows[0]["trace_id"], "external");
+    Ok(())
+}
+
+#[rstest]
+#[case::changing("100", 0, 0, 1001, 100, true)]
+#[case::all("100", 0, 0, 1001, 100, false)]
+#[case::percentage("10", 0, 0, 101, 100, false)]
+#[case::capped("100", 25, 0, 25, 100, false)]
+#[case::preview("10", 25, 1, 1001, 100, false)]
+#[tokio::test]
+async fn lens_selection_pages_without_losing_or_repeating_runs(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+    #[case] percent: &str,
+    #[case] cap: i64,
+    #[case] preview: i64,
+    #[case] expected: usize,
+    #[case] page_size: usize,
+    #[case] changing: bool,
+) -> TestResult {
+    use litellm_traces::LensQuery;
+    let database = database?;
+    ensure_schema(
+        &database.client,
+        &Connection::writer(&database.url)?,
+        "trace_test",
+        7,
+        14,
+    )
+    .await?;
+    execute_write(&database, "INSERT INTO trace_test.spend_logs (request_id,team_id,start_time,end_time) SELECT toString(number),'team',now64(3)-INTERVAL 5 MINUTE,now64(3)-INTERVAL 5 MINUTE FROM numbers(1001)").await?;
+    let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let end = time::OffsetDateTime::now_utc().unix_timestamp() * 1000 + 60000;
+    let mut seen = std::collections::BTreeSet::new();
+    let mut cursor = String::new();
+    let step = if page_size == 0 { expected } else { page_size };
+    for offset in (0..expected).step_by(step) {
+        let parameters = BTreeMap::from([
+            ("source".into(), Parameter::Text("requests".into())),
+            ("all_teams".into(), Parameter::Integer(0)),
+            ("team".into(), Parameter::Text("team".into())),
+            ("key_hash".into(), Parameter::Text(String::new())),
+            ("start".into(), Parameter::Integer(0)),
+            ("end".into(), Parameter::Integer(end)),
+            ("service".into(), Parameter::Text(String::new())),
+            ("filter_keys".into(), Parameter::Strings(vec![])),
+            ("filter_values".into(), Parameter::Strings(vec![])),
+            ("limit".into(), Parameter::Integer(page_size as i64)),
+            (
+                "offset".into(),
+                Parameter::Integer(if changing { 0 } else { offset as i64 }),
+            ),
+            ("after".into(), Parameter::Text(cursor.clone())),
+            ("sample_percent".into(), Parameter::Text(percent.into())),
+            ("sample_cap".into(), Parameter::Integer(cap)),
+            ("preview".into(), Parameter::Integer(preview)),
+            ("selected_team".into(), Parameter::Text(String::new())),
+            ("execution_ids".into(), Parameter::Strings(vec![])),
+        ]);
+        let body = execute_read(
+            &database.client,
+            &connection,
+            LensQuery::Sample.sql(),
+            &parameters,
+        )
+        .await?;
+        let json: serde_json::Value = serde_json::from_str(&body)?;
+        let rows = json["data"].as_array().expect("sample rows");
+        assert_eq!(rows.len(), step.min(expected - offset));
+        for row in rows {
+            assert_eq!(
+                row["eligible"],
+                if changing && offset > 0 { 1000 } else { 1001 }
+            );
+            assert!(seen.insert(row["trace_id"].as_str().expect("run id").to_owned()));
+        }
+        if changing {
+            cursor = rows.last().expect("last run")["selection_key"]
+                .as_str()
+                .expect("selection key")
+                .to_owned();
+            if offset == 0 {
+                let removed = rows[0]["trace_id"].as_str().expect("request id");
+                execute_write(&database, &format!("ALTER TABLE trace_test.spend_logs DELETE WHERE request_id='{removed}' SETTINGS mutations_sync=1")).await?;
+            }
+        }
+    }
+    assert_eq!(seen.len(), expected);
+    Ok(())
+}
+
+#[rstest]
+#[case::short(100)]
+#[case::boundary(7970)]
+#[case::long(16000)]
+#[tokio::test]
+async fn lens_content_keeps_output_visible_after_long_input(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+    #[case] input_length: usize,
+) -> TestResult {
+    use litellm_traces::LensQuery;
+    let database = database?;
+    ensure_schema(
+        &database.client,
+        &Connection::writer(&database.url)?,
+        "trace_test",
+        7,
+        14,
+    )
+    .await?;
+    insert_rows(&database, "spend_logs", vec![serde_json::from_value(serde_json::json!({
+        "request_id": "request", "team_id": "team", "start_time": time::OffsetDateTime::now_utc().unix_timestamp()*1000, "end_time": time::OffsetDateTime::now_utc().unix_timestamp()*1000, "messages": "x".repeat(input_length), "response": "Delivered result"
+    }))?]).await?;
+    let connection = Connection::configured(&database.url, "trace_test", "default", "")?;
+    let mut parameters = BTreeMap::from([
+        ("source".into(), Parameter::Text("requests".into())),
+        ("all_teams".into(), Parameter::Integer(0)),
+        ("team".into(), Parameter::Text("team".into())),
+        ("record_team".into(), Parameter::Text("team".into())),
+        ("key_hash".into(), Parameter::Text(String::new())),
+        ("trace_ref".into(), Parameter::Text(String::new())),
+        ("id".into(), Parameter::Text("request".into())),
+        ("cursor".into(), Parameter::Text(String::new())),
+        ("offset".into(), Parameter::Integer(1)),
+    ]);
+    let body = execute_read(
+        &database.client,
+        &connection,
+        LensQuery::Content.sql(),
+        &parameters,
+    )
+    .await?;
+    let json: serde_json::Value = serde_json::from_str(&body)?;
+    let text = json["data"][0]["content"].as_str().expect("content");
+    assert!(text.contains("Output: Delivered result"));
+    assert!(text.len() <= 8000);
+    assert_eq!(
+        json["data"][0]["truncated"],
+        u8::from(input_length + "Input: \nOutput: Delivered result\nError: ".len() > 8000)
+    );
+    let original = format!(
+        "Input: {}\nOutput: Delivered result\nError: ",
+        "x".repeat(input_length)
+    );
+    let mut recovered = String::new();
+    for offset in (2..original.len() + 2).step_by(8000) {
+        parameters.insert("offset".into(), Parameter::Integer(offset as i64));
+        let body = execute_read(
+            &database.client,
+            &connection,
+            LensQuery::Content.sql(),
+            &parameters,
+        )
+        .await?;
+        let page: serde_json::Value = serde_json::from_str(&body)?;
+        recovered.push_str(page["data"][0]["content"].as_str().expect("content"));
+    }
+    assert_eq!(recovered, original);
+    Ok(())
+}

litellm-rust/crates/traces/tests/otlp.rs

@@ -0,0 +1,47 @@
+use flate2::{Compression, write::GzEncoder};
+use litellm_traces::decode_otlp;
+use rstest::rstest;
+use std::io::Write;
+
+const FIXTURE: &[u8] = include_bytes!(
+    "../../../../tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json"
+);
+
+#[rstest]
+#[case::json(FIXTURE, Some("application/json"), None)]
+#[case::gzip_json(FIXTURE, Some("application/json"), Some("gzip"))]
+fn decodes_neutral_spans(
+    #[case] body: &[u8],
+    #[case] content_type: Option<&str>,
+    #[case] content_encoding: Option<&str>,
+) {
+    let payload = if content_encoding == Some("gzip") {
+        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(body).expect("gzip input");
+        encoder.finish().expect("gzip payload")
+    } else {
+        body.to_vec()
+    };
+    let spans = decode_otlp(&payload, content_type, content_encoding, 8 * 1024 * 1024)
+        .expect("valid OTLP export");
+    assert_eq!(spans.len(), 6);
+    assert_eq!(spans[0].trace_id, "4bad42b84e9de3ba46fc870185f8f023");
+    assert_eq!(spans[0].resource_attributes["service.name"], "agent-demo");
+    assert_eq!(spans[0].scope_name, "langsmith");
+    assert!(
+        spans
+            .iter()
+            .any(|span| span.attributes.contains_key("gen_ai.prompt"))
+    );
+}
+
+#[rstest]
+#[case::invalid(b"not protobuf", None, 8 * 1024 * 1024)]
+#[case::too_large(FIXTURE, Some("application/json"), 1)]
+fn rejects_invalid_or_oversized_payload(
+    #[case] body: &[u8],
+    #[case] content_type: Option<&str>,
+    #[case] limit: usize,
+) {
+    assert!(decode_otlp(body, content_type, None, limit).is_err());
+}

litellm-rust/crates/traces/tests/queries.rs

@@ -0,0 +1,11 @@
+use litellm_traces::Connection;
+use rstest::rstest;
+
+#[rstest]
+#[case::http("http://localhost:8123", true)]
+#[case::https("https://localhost:8443", true)]
+#[case::tcp("tcp://localhost:9000", false)]
+#[case::missing_host("http://", false)]
+fn accepts_only_clickhouse_http_urls(#[case] value: &str, #[case] expected: bool) {
+    assert_eq!(Connection::parse(value).is_ok(), expected);
+}

litellm/__init__.py

@@ -157,6 +157,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "smtp_email",
     "deepeval",
     "s3_v2",
+    "clickhouse",
     "pointfive",
     "zerobus",
     "aws_sqs",

litellm/anthropic_beta_headers_config.json

@@ -34,7 +34,9 @@
     "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": "web-fetch-2025-09-10",
     "web-search-2025-03-05": "web-search-2025-03-05",
-    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01"
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01"
   },
   "azure_ai": {
     "advisor-tool-2026-03-01": null,
@@ -47,7 +49,7 @@
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
-    "dangerous-tool-use-2026-09-03": null,
+    "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": "files-api-2025-04-14",
@@ -136,7 +138,9 @@
     "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": null,
-    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01"
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01"
   },
   "bedrock_mantle": {
     "advanced-tool-use-2025-11-20": "tool-search-tool-2025-10-19",

litellm/caching/caching.py

@@ -8,7 +8,6 @@
 #  Thank you users! We ❤️ you! - Krrish & Ishaan
 
 import ast
-import asyncio
 import hashlib
 import json
 import logging
@@ -31,11 +30,10 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
 from .azure_blob_cache import AzureBlobCache
 from .base_cache import BaseCache
 from .disk_cache import DiskCache
-from .dual_cache import DualCache
+from .dual_cache import DualCache  # noqa: F401  # re-exported, callers import DualCache from litellm.caching.caching
 from .gcs_cache import GCSCache
 from .in_memory_cache import InMemoryCache
 from .qdrant_semantic_cache import QdrantSemanticCache
-from .redis_batch import active_post_call_redis_batch
 from .redis_cache import RedisCache, log_redis_failure
 from .redis_cluster_cache import RedisClusterCache
 from .redis_semantic_cache import RedisSemanticCache
@@ -70,15 +68,6 @@ def print_verbose(print_statement):
         pass
 
 
-def _ttl_seconds(raw: object) -> int | None:
-    if not isinstance(raw, (int, float, str)):
-        return None
-    try:
-        return int(raw)
-    except ValueError:
-        return None
-
-
 class CacheMode(str, Enum):
     default_on = "default_on"
     default_off = "default_off"
@@ -770,8 +759,6 @@ class Cache:
                 await self.batch_cache_write(result, **kwargs)
             else:
                 cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs)
-                if await self._defer_set_to_post_call_batch(cache_key, cached_data, kwargs, dynamic_cache_object):
-                    return
                 if dynamic_cache_object is not None:
                     await dynamic_cache_object.async_set_cache(cache_key, cached_data, **kwargs)
                 else:
@@ -779,39 +766,6 @@ class Cache:
         except Exception as e:
             self._log_add_cache_failure(e)
 
-    async def _defer_set_to_post_call_batch(
-        self,
-        cache_key: str,
-        cached_data: object,
-        kwargs: Mapping[str, object],
-        dynamic_cache_object: BaseCache | None,
-    ) -> bool:
-        """A plain SET on the Redis response cache rides the request's post-call pipeline with the counters,
-        instead of its own round trip. Anything with SET options keeps the direct path."""
-        if kwargs.get("nx"):
-            return False
-        ttl: Final = _ttl_seconds(kwargs.get("ttl"))
-        if isinstance(dynamic_cache_object, DualCache):
-            deferred: Final = await dynamic_cache_object.async_set_cache_post_call(cache_key, cached_data, ttl)
-            if deferred is None:
-                return False
-            deferred.on_settled(self._log_deferred_add_cache_failure)
-            return True
-        if dynamic_cache_object is not None or not isinstance(self.cache, RedisCache):
-            return False
-        batch: Final = active_post_call_redis_batch(self.cache)
-        if batch is None:
-            return False
-        batch.set(cache_key, cached_data, ttl).on_settled(self._log_deferred_add_cache_failure)
-        return True
-
-    def _log_deferred_add_cache_failure(self, future: asyncio.Future[None]) -> None:
-        if future.cancelled():
-            return
-        failure: Final = future.exception()
-        if isinstance(failure, Exception):
-            self._log_add_cache_failure(failure)
-
     def _convert_to_cached_embedding(
         self,
         embedding_response: Any,

litellm/caching/dual_cache.py

@@ -525,12 +525,6 @@ class DualCache(BaseCache):
         batch: Final = None if self.redis_cache is None else active_request_redis_batch(self.redis_cache)
         return None if batch is None else await self._set_on_batch(batch, key, value, ttl)
 
-    async def async_set_cache_post_call(self, key: str, value: object, ttl: float | None) -> BatchResult[None] | None:
-        """Memory now, the Redis SET on the request's post-call pipeline; None when no pipeline is open, so the
-        caller takes its direct path."""
-        batch: Final = None if self.redis_cache is None else active_post_call_redis_batch(self.redis_cache)
-        return None if batch is None else await self._set_on_batch(batch, key, value, ttl)
-
     async def async_delete_cache_pre_call(self, key: str) -> BatchResult[None] | None:
         """Memory now, the Redis DEL on the request's pipeline; None when no pipeline is open, so the caller
         takes its direct path."""

litellm/constants.py

@@ -46,6 +46,18 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset(
 )
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
 DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
+CLICKHOUSE_BATCH_SIZE: Final = get_env_int("CLICKHOUSE_BATCH_SIZE", 10_000)
+CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", "1.0"))
+CLICKHOUSE_MAX_BUFFERED_ROWS: Final = get_env_int("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000)
+CLICKHOUSE_MAX_RETRIES: Final = get_env_int("CLICKHOUSE_MAX_RETRIES", 3)
+AGENT_TRACING_RETENTION_DAYS: Final = get_env_int("AGENT_TRACING_RETENTION_DAYS", 30)
+AGENT_TRACING_SPEND_LOG_RETENTION_DAYS: Final = get_env_int("AGENT_TRACING_SPEND_LOG_RETENTION_DAYS", 90)
+OTLP_MAX_BODY_BYTES: Final = get_env_int("OTLP_MAX_BODY_BYTES", 8 * 1024 * 1024)
+OTLP_MAX_ATTRIBUTE_VALUE_BYTES: Final = get_env_int("OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 64 * 1024)
+OTLP_RETRY_AFTER_SECONDS: Final = get_env_int("OTLP_RETRY_AFTER_SECONDS", 2)
+OTLP_OFFLOAD_DECODE_BYTES: Final = get_env_int("OTLP_OFFLOAD_DECODE_BYTES", 256 * 1024)
+AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = get_env_int("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240)
+AGENT_TRACING_LIST_PAGE_SIZE: Final = get_env_int("AGENT_TRACING_LIST_PAGE_SIZE", 50)
 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))
 DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
 DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16"))
@@ -948,6 +960,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://api.meta.ai/v1",
     "https://api.sailresearch.com/v1",
     "https://api.cognition.ai/v1",
+    "https://api.cortecs.ai/v1",
     "https://api.scx.ai/v1",
     "https://api.prisminference.com/v1",
     "https://gigachat.devices.sberbank.ru/api/v1",
@@ -1022,6 +1035,7 @@ openai_compatible_providers: Final[list] = [
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
+    "cortecs",
     "scx-ai",
     "prism",
     "sail",

litellm/integrations/azure_storage/azure_storage.py

@@ -30,6 +30,14 @@ from litellm.types.secret_managers.get_azure_ad_token_provider import (
 from litellm.types.utils import StandardLoggingPayload
 
 AZURE_STORAGE_TOKEN_SCOPE: Final = "https://storage.azure.com/.default"
+_ADLS_SAFE_NAME: Final = str.maketrans("/", "_", "=")
+
+
+def adls_safe_file_name(payload_id: str | None) -> str:
+    """`=` padding and `/` in a base64 payload id are what the Data Lake service rejects, so the name drops the
+    padding and maps `/` to `_`. Standard base64 has no `_` and its padding is fixed by the length, so ids from
+    that alphabet stay distinct; anything else is left as is."""
+    return f"{(payload_id or str(uuid.uuid4())).translate(_ADLS_SAFE_NAME)}.json"
 
 
 @cache
@@ -46,6 +54,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
         build_credential_chain_token_provider: Callable[
             [], Callable[[], str]
         ] = _cached_credential_chain_token_provider,
+        clock: Callable[[], float] = time.time,
         **kwargs,
     ):
         try:
@@ -69,6 +78,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
             self.azure_storage_endpoint_suffix: str = (
                 os.getenv("AZURE_STORAGE_ENDPOINT_SUFFIX") or AZURE_STORAGE_DEFAULT_ENDPOINT_SUFFIX
             )
+            self._clock: Callable[[], float] = clock
             self._service_client = None
             # Time that the azure service client expires, in order to reset the connection pool and keep it fresh
             self._service_client_timeout: float | None = None
@@ -182,7 +192,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
                 async_client: Final = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
                 json_payload: Final = safe_dumps(payload) + "\n"  # Add newline for each log entry
                 payload_bytes: Final = json_payload.encode("utf-8")
-                filename: Final = f"{payload.get('id') or str(uuid.uuid4())}.json"
+                filename: Final = adls_safe_file_name(payload.get("id"))
                 base_url = f"{self.azure_storage_dfs_endpoint}/{self.azure_storage_file_system}/{filename}"
 
                 # Execute the 3-step upload process
@@ -331,7 +341,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
         from azure.storage.filedatalake.aio import DataLakeServiceClient
 
         # expire old clients to recover from connection issues
-        if self._service_client_timeout and self._service_client and self._service_client_timeout > time.time():
+        if self._service_client_timeout and self._service_client and self._service_client_timeout <= self._clock():
             await self._service_client.close()
             self._service_client = None
         if not self._service_client:
@@ -339,7 +349,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
                 account_url=self.azure_storage_dfs_endpoint,
                 credential=self.azure_storage_account_key,
             )
-            self._service_client_timeout = time.time() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
+            self._service_client_timeout = self._clock() + _DEFAULT_TTL_FOR_HTTPX_CLIENTS
         return self._service_client
 
     async def upload_to_azure_data_lake_with_azure_account_key(self, payload: StandardLoggingPayload):
@@ -368,7 +378,7 @@ class AzureBlobStorageLogger(CustomBatchLogger):
                 verbose_logger.debug("Created directory: %s", today)
 
             # Create a file client
-            file_name: Final = f"{payload.get('id') or str(uuid.uuid4())}.json"
+            file_name: Final = adls_safe_file_name(payload.get("id"))
             file_client: Final = directory_client.get_file_client(file_name)
 
             # Create the file

litellm/integrations/clickhouse/clickhouse_batch_logger.py

@@ -0,0 +1,100 @@
+"""
+Shared base for everything LiteLLM writes to ClickHouse.
+
+Built on `CustomBatchLogger`: rows accumulate in `log_queue` and are flushed as one
+gzip JSONEachRow insert, either every `CLICKHOUSE_FLUSH_INTERVAL_SECONDS` or as soon as
+`batch_size` rows are queued. Subclasses only pick the table and build rows:
+
+- `ClickHouseSpendLogger`  -> spend_logs   (LiteLLM requests when tracing is enabled)
+"""
+
+import asyncio
+import os
+from collections.abc import Mapping, Sequence
+from typing import Any, ClassVar
+
+from litellm._logging import verbose_logger
+from litellm.constants import (
+    CLICKHOUSE_BATCH_SIZE,
+    CLICKHOUSE_FLUSH_INTERVAL_SECONDS,
+    CLICKHOUSE_MAX_BUFFERED_ROWS,
+    CLICKHOUSE_MAX_RETRIES,
+)
+from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.rust_bridge.traces import TraceStorage
+
+
+def clickhouse_storage_from_env() -> TraceStorage:
+    return TraceStorage(
+        database=os.getenv("CLICKHOUSE_DATABASE", "litellm"),
+        url=os.getenv("CLICKHOUSE_URL", ""),
+    )
+
+
+class ClickHouseBatchLogger(CustomBatchLogger):
+    table: ClassVar[str]
+
+    def __init__(self, storage: TraceStorage | None = None) -> None:
+        self.storage = storage or clickhouse_storage_from_env()
+        self.rows_written = 0
+        self.rows_dropped = 0
+        self._failed_attempts = 0
+        super().__init__(
+            flush_lock=asyncio.Lock(),
+            batch_size=CLICKHOUSE_BATCH_SIZE,
+            flush_interval=CLICKHOUSE_FLUSH_INTERVAL_SECONDS,
+        )
+        self._flush_task: asyncio.Task[None] | None = None
+
+    def start(self) -> None:
+        if self._flush_task is None or self._flush_task.done():
+            self._flush_task = asyncio.get_running_loop().create_task(self.periodic_flush())
+
+    def is_full(self) -> bool:
+        """Backpressure signal: producers should reject (429) instead of enqueueing."""
+        return len(self.log_queue) >= CLICKHOUSE_MAX_BUFFERED_ROWS
+
+    def enqueue(self, rows: Sequence[Mapping[str, object]]) -> None:
+        """Never awaits ClickHouse. Kicks off an early flush once a full batch is queued."""
+        self.start()
+        self.log_queue.extend(rows)
+        if len(self.log_queue) >= self.batch_size:
+            asyncio.get_running_loop().create_task(self.flush_queue())
+
+    async def flush_queue(self) -> None:
+        # Swap the queue under the lock so rows enqueued during the insert are kept.
+        if self.flush_lock is None:
+            return
+        async with self.flush_lock:
+            while self.log_queue:
+                batch = self.log_queue[: self.batch_size]
+                self.log_queue = self.log_queue[len(batch) :]
+                if not await self._insert(batch):
+                    break
+
+    async def async_send_batch(self) -> None:
+        await self.flush_queue()
+
+    async def _insert(self, batch: list[dict[str, Any]]) -> bool:
+        try:
+            await self.storage.insert_rows(self.table, batch)
+            self.rows_written += len(batch)
+            self._failed_attempts = 0
+            return True
+        except Exception as e:
+            self._failed_attempts += 1
+            if self._failed_attempts >= CLICKHOUSE_MAX_RETRIES:
+                self.rows_dropped += len(batch)
+                self._failed_attempts = 0
+                verbose_logger.error(
+                    "ClickHouse: dropped %s rows for %s after %s attempts: %s",
+                    len(batch),
+                    self.table,
+                    CLICKHOUSE_MAX_RETRIES,
+                    e,
+                )
+            else:
+                # put it back; the next periodic flush retries it
+                self.log_queue = batch + self.log_queue
+                verbose_logger.warning("ClickHouse: insert into %s failed, will retry: %s", self.table, e)
+            return False

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -0,0 +1,170 @@
+"""
+`clickhouse` logging callback: one `spend_logs` row per LiteLLM request.
+
+Agent LLM spans join to these rows on `otel_traces.LiteLLMRequestId = spend_logs.response_id`,
+so `response_id` is always the raw provider response id (cache-hit suffix stripped).
+"""
+
+import json
+import re
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Any, Final
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
+from litellm.integrations.clickhouse.context import is_lens_analysis
+from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
+from litellm.tracing.types import SpendLogRecord
+from litellm.types.utils import StandardLoggingPayload
+
+# litellm_logging.py rewrites cache-hit ids as f"{id}_cache_hit{time.time()}"
+MILLISECONDS_PER_SECOND: Final = 1000
+_CACHE_HIT_SUFFIX: Final = re.compile(r"_cache_hit[0-9.]*$")
+# W3C trace context: version-traceid-parentid-flags
+_TRACEPARENT: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$")
+_INVALID_TRACE_ID: Final = "0" * 32
+_INVALID_SPAN_ID: Final = "0" * 16
+TRACE_INGEST_ROUTE: Final = "/v1/traces"
+
+
+def strip_cache_hit_suffix(request_id: str) -> str:
+    return _CACHE_HIT_SUFFIX.sub("", request_id)
+
+
+def parse_traceparent(value: object) -> tuple[str, str]:
+    """(trace_id, span_id) from a W3C `traceparent` header, or ("", "") if absent/invalid."""
+    if not isinstance(value, str):
+        return "", ""
+    match = _TRACEPARENT.match(value.strip().lower())
+    if match is None or match.group(1) == _INVALID_TRACE_ID or match.group(2) == _INVALID_SPAN_ID:
+        return "", ""
+    return match.group(1), match.group(2)
+
+
+def _to_ms(seconds: object) -> int | None:
+    return int(float(seconds) * MILLISECONDS_PER_SECOND) if isinstance(seconds, (int, float)) else None
+
+
+def _int(value: object) -> int:
+    return value if isinstance(value, int) and not isinstance(value, bool) else 0
+
+
+def _json(value: object) -> str:
+    if value is None or value == "":
+        return ""
+    return value if isinstance(value, str) else json.dumps(value, default=str)
+
+
+def _json_mapping(value: Mapping[str, Any]) -> str:
+    return _json(dict(value))  # mutable-ok: [LIT002] JSON serialization requires a dict
+
+
+def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> tuple[str, str]:
+    custom_headers = metadata.get("requester_custom_headers") or MappingProxyType({})
+    proxy_request = (kwargs.get("litellm_params") or MappingProxyType({})).get(
+        "proxy_server_request"
+    ) or MappingProxyType({})
+    request_headers = proxy_request.get("headers") or MappingProxyType({})
+    for headers in (custom_headers, request_headers):
+        for name, value in headers.items():
+            if str(name).lower() == "traceparent":
+                return parse_traceparent(value)
+    return "", ""
+
+
+def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]:
+    """(cache_read, cache_write) from a Usage dict: OpenAI prompt_tokens_details first, Anthropic fields as fallback."""
+    details = usage.get("prompt_tokens_details") or MappingProxyType({})
+    cache_read = _int(details.get("cached_tokens")) or _int(usage.get("cache_read_input_tokens"))
+    cache_write = (
+        _int(details.get("cache_write_tokens"))
+        or _int(details.get("cache_creation_tokens"))
+        or _int(usage.get("cache_creation_input_tokens"))
+    )
+    return cache_read, cache_write
+
+
+def _request_tags(value: object) -> list[str]:
+    if not isinstance(value, list):
+        return []  # mutable-ok: [LIT002] empty spend-log tag payload
+    return [str(tag) for tag in value]  # mutable-ok: [LIT002] SpendLogRecord schema
+
+
+def _session_id(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> str:
+    """Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id."""
+    request_metadata = (kwargs.get("litellm_params") or MappingProxyType({})).get("metadata") or MappingProxyType({})
+    return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "")
+
+
+def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
+    """OTLP exports to POST /v1/traces are not LLM requests; don't write them as spend rows."""
+    return str(payload.get("call_type") or "").startswith(TRACE_INGEST_ROUTE)
+
+
+def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> SpendLogRecord:
+    metadata: Mapping[str, Any] = payload.get("metadata") or MappingProxyType({})
+    hidden_params: Mapping[str, Any] = payload.get("hidden_params") or MappingProxyType({})
+    usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({})
+    cache_read_tokens, cache_write_tokens = _cache_tokens(usage)
+    trace_id, span_id = _find_traceparent(metadata, kwargs)
+    request_id = str(payload.get("id") or "")
+    redact = litellm.turn_off_message_logging is True
+    completion_start_ms = _to_ms(payload.get("completionStartTime"))
+    return SpendLogRecord(
+        request_id=request_id,
+        response_id=strip_cache_hit_suffix(request_id),
+        call_type=payload.get("call_type") or "",
+        api_key=metadata.get("user_api_key_hash") or "",
+        key_alias=metadata.get("user_api_key_alias") or "",
+        team_id=metadata.get("user_api_key_team_id") or metadata.get("team_id") or "",
+        team_alias=metadata.get("user_api_key_team_alias") or metadata.get("team_alias") or "",
+        organization_id=metadata.get("user_api_key_org_id") or "",
+        user=metadata.get("user_api_key_user_id") or "",
+        end_user=payload.get("end_user") or metadata.get("user_api_key_end_user_id") or "",
+        model=payload.get("model") or "",
+        model_group=payload.get("model_group") or "",
+        model_id=payload.get("model_id") or "",
+        custom_llm_provider=payload.get("custom_llm_provider") or "",
+        api_base=payload.get("api_base") or "",
+        spend=float(payload.get("response_cost") or 0.0),
+        prompt_tokens=_int(payload.get("prompt_tokens")),
+        completion_tokens=_int(payload.get("completion_tokens")),
+        total_tokens=_int(payload.get("total_tokens")),
+        cache_read_tokens=cache_read_tokens,
+        cache_write_tokens=cache_write_tokens,
+        start_time=_to_ms(payload.get("startTime")) or 0,
+        end_time=_to_ms(payload.get("endTime")) or 0,
+        completion_start_time=completion_start_ms or None,
+        status=payload.get("status") or "",
+        error_str=payload.get("error_str") or "",
+        cache_hit=payload.get("cache_hit") is True,
+        session_id=_session_id(payload, kwargs),
+        trace_id=trace_id,
+        span_id=span_id,
+        request_tags=_request_tags(payload.get("request_tags")),
+        metadata=_json_mapping(MappingProxyType({**metadata, "litellm_lens_internal": is_lens_analysis()})),
+        messages="" if redact else _json(payload.get("messages")),
+        response="" if redact else _json(payload.get("response")),
+    )
+
+
+class ClickHouseSpendLogger(ClickHouseBatchLogger):
+    table = SPEND_LOGS_TABLE
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
+        self._log(kwargs)
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None:
+        self._log(kwargs)
+
+    def _log(self, kwargs: Mapping[str, Any]) -> None:
+        try:
+            payload = kwargs.get("standard_logging_object")
+            if payload is None or _is_trace_ingest(payload):
+                return
+            row: Final = spend_log_row_from_payload(payload, kwargs)
+            self.enqueue([dict(row)])  # mutable-ok: [LIT002] batch logger API
+        except Exception as e:
+            verbose_logger.exception("ClickHouseSpendLogger: failed to log request: %s", e)

litellm/integrations/clickhouse/context.py

@@ -0,0 +1,19 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Final
+
+_lens_analysis: Final = ContextVar("litellm_lens_analysis", default=False)
+
+
+def is_lens_analysis() -> bool:
+    return _lens_analysis.get()
+
+
+@contextmanager
+def lens_analysis() -> Iterator[None]:
+    token: Final = _lens_analysis.set(True)
+    try:
+        yield
+    finally:
+        _lens_analysis.reset(token)

litellm/integrations/clickhouse/schema.py

@@ -0,0 +1,11 @@
+from typing import Final
+
+from litellm.rust_bridge.traces import TraceStorage
+
+OTEL_TRACES_TABLE: Final = "otel_traces"
+AGENT_TRACES_BY_KEY_TABLE: Final = "agent_traces_by_key"
+SPEND_LOGS_TABLE: Final = "spend_logs"
+
+
+async def ensure_schema(storage: TraceStorage, trace_retention_days: int, spend_log_retention_days: int) -> None:
+    await storage.ensure_schema(trace_retention_days, spend_log_retention_days)