chore: merge main into litellm_s3_partition_granularity

.circleci/scripts/unit_selection.sh

@@ -89,6 +89,7 @@ legacy_paths() {
     proxy-db-auth-checks)
       echo tests/unit/proxy/auth/test_auth_checks.py
       echo tests/unit/proxy/auth/test_user_api_key_auth.py
+      echo tests/unit/proxy/test_credential_slot_registry.py
       echo tests/unit/proxy/test_deprecated_key_grace_period.py ;;
     proxy-db-budgets)
       echo tests/unit/proxy/auth/test_default_end_user_budget_simple.py
@@ -106,6 +107,7 @@ legacy_paths() {
       echo tests/unit/proxy/test_update_spend.py
       echo tests/unit/skills/test_skills_db.py ;;
     proxy-db-endpoints-and-responses)
+      echo tests/unit/proxy/engine
       echo tests/unit/proxy/auth/test_models_fallback_endpoint.py
       echo tests/unit/proxy/common_utils/test_check_batch_cost.py
       echo tests/unit/proxy/common_utils/test_check_responses_cost.py
@@ -147,7 +149,10 @@ legacy_paths() {
       echo tests/unit/proxy/test_proxy_server.py ;;
     proxy-db-proxy-utils) echo tests/unit/proxy/test_proxy_utils.py ;;
     proxy-extras) echo tests/unit/litellm_proxy_extras ;;
-    proxy-infra) echo tests/unit/gateway ;;
+    proxy-infra)
+      echo tests/unit/gateway
+      echo tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
+      echo tests/unit/proxy/roi_calculator ;;
     responses-caching-types)
       find tests/unit/responses -name 'test_*.py' -not -path 'tests/unit/responses/mcp/*'
       echo tests/unit/types ;;

.github/workflows/lens-worker.yml

@@ -0,0 +1,54 @@
+name: Lens Worker Image
+
+on:
+  pull_request:
+    branches: [main, litellm_oss_branch, "litellm_**"]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/engine/**
+      - .github/workflows/lens-worker.yml
+  push:
+    branches: [main, litellm_agent_engine]
+    paths:
+      - deploy/lens/**
+      - litellm/proxy/engine/**
+      - .github/workflows/lens-worker.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lens-worker-image:
+    permissions:
+      contents: read
+      packages: write
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Build Lens worker
+        run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+      - name: Verify standalone imports with a read-only filesystem
+        run: >-
+          docker run --rm --network none --read-only --cap-drop ALL
+          --security-opt no-new-privileges --entrypoint python
+          lens-worker:${{ github.sha }}
+          -c 'import os; import engine.worker; assert os.getuid() == 65532'
+      - name: Publish versioned Lens worker
+        if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
+        env:
+          REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REGISTRY_USER: ${{ github.actor }}
+          IMAGE: ghcr.io/berriai/litellm-lens-worker:sha-${{ github.sha }}
+        run: |
+          printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin
+          docker tag lens-worker:${{ github.sha }} "$IMAGE"
+          docker push "$IMAGE"
+          printf 'Lens worker image: `%s`\n' "$IMAGE" >> "$GITHUB_STEP_SUMMARY"

.github/workflows/test-postgres.yml

@@ -24,6 +24,7 @@ jobs:
     timeout-minutes: ${{ matrix.job-timeout-minutes }}
     permissions:
       contents: read
+      id-token: write
 
     services:
       postgres:
@@ -134,9 +135,19 @@ jobs:
         env:
           TEST_PATH: ${{ matrix.test-path }}
           WORKERS: ${{ matrix.workers }}
+          PYTEST_ADDOPTS: ${{ matrix.shard == 'proxy-behavior' && '--cov=litellm/proxy/engine --cov-report=xml:coverage-lens-postgres.xml' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
           else
             uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
           fi
+
+      - name: Upload Lens database coverage
+        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'proxy-behavior' && !cancelled()
+        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4
+        with:
+          use_oidc: true
+          files: coverage-lens-postgres.xml
+          flags: lens-postgres
+          fail_ci_if_error: true

.github/workflows/test-unit.yml

@@ -79,7 +79,9 @@ jobs:
 
           - shard: integrations
             artifact-name: integrations
-            test-path: ""
+            test-path: >-
+              tests/test_litellm/integrations
+              tests/test_litellm/tracing
             unit-flag: integrations
             workers: 2
             reruns: 3

backend/routes/allowlist.py

@@ -81,6 +81,8 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Spend / analytics
     "/spend/",
     "/analytics/",
+    "/engine/",
+    "/v1/traces",
     "/global/",
     "/user_agent",
     "/usage/",
@@ -144,6 +146,7 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
     {
         "/",
         "/routes",
+        "/engine",
         "/openapi.json",
         "/docs",
         "/docs/oauth2-redirect",

basedpyright-code-budget.json

@@ -99,7 +99,7 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44358
+    "limit": 44802
   },
   "reportUnknownLambdaType": {
     "limit": 109

cookbook/misc/config.yaml

@@ -24,7 +24,7 @@ model_list:
   - model_name: sagemaker-completion-model
     litellm_params:
       model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4
-      input_cost_per_second: 0.000420  
+      cost_per_second: 0.000420
   - model_name: text-embedding-ada-002
     litellm_params: 
       model: azure/azure-embedding-model

cookbook/misc/test_responses_api.py

@@ -12,7 +12,7 @@ def encode_image(image_path):
 
 
 # Path to your image
-image_path = "litellm/proxy/logo.jpg"
+image_path = "litellm/proxy/logo.png"
 
 # Getting the Base64 string
 base64_image = encode_image(image_path)
@@ -27,7 +27,7 @@ response = client.responses.create(
                 {"type": "input_text", "text": "what color is the image"},
                 {
                     "type": "input_image",
-                    "image_url": f"data:image/jpeg;base64,{base64_image}",
+                    "image_url": f"data:image/png;base64,{base64_image}",
                 },
             ],
         }

deploy/lens/Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.12-slim
+WORKDIR /app
+RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
+COPY litellm/proxy/engine/__init__.py litellm/proxy/engine/models.py litellm/proxy/engine/analysis.py litellm/proxy/engine/worker.py /app/engine/
+USER 65532:65532
+CMD ["python", "-m", "engine.worker"]

deploy/lens/Dockerfile.dockerignore

@@ -0,0 +1,8 @@
+**
+!litellm/
+!litellm/proxy/
+!litellm/proxy/engine/
+!litellm/proxy/engine/__init__.py
+!litellm/proxy/engine/models.py
+!litellm/proxy/engine/analysis.py
+!litellm/proxy/engine/worker.py

deploy/lens/README.md

@@ -0,0 +1,59 @@
+# Lens worker
+
+Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
+
+## Start a worker
+
+Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL, agent tracing (`general_settings.tracing: {store: clickhouse}`), and ClickHouse configured through `CLICKHOUSE_URL` and a separate SELECT-only `CLICKHOUSE_READER_URL`. Enable the ClickHouse callback and request/response logging to analyze LLM requests. Lens can only inspect content you actually retain
+
+In Lens, click **Connect worker**, then **Generate setup command**. The LiteLLM address is filled in for you; change it only if the server running Docker needs a different network address. Copy the command and run it on your server. The dialog changes to **Worker connected** when the container checks in
+
+The command already contains the compatible worker image and one worker token. No separate API key, source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
+
+The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. Worker image releases are independent of proxy releases: update the pinned image when changing their API contract. CI also publishes immutable commit tags for reproducible builds
+
+For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
+
+```bash
+docker compose --env-file /path/to/lens.env -f compose.yaml up -d
+```
+
+Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
+
+The worker needs outbound HTTPS access to LiteLLM. It needs no inbound ports, provider keys, direct database access, or GPU. The proxy calls your selected model through its configured router; trace content reaches that model provider. Use a model with JSON output support and known token prices. One worker handles one scan at a time and can serve multiple lenses. For more throughput, start another worker with a separate credential
+
+V1 setup, manual runs, feedback, and worker credentials are restricted to proxy administrators. Admin viewers can inspect results. Worker credentials can serve the administrator’s lenses. Revoke it in the connection dialog when retiring a worker. Redeploy the worker alongside proxy upgrades so their API versions match
+
+## Configure a lens
+
+Choose agent runs, individual LLM requests, or both. The matching-activity preview updates as you choose an application (the recorded OpenTelemetry service.name) or, for request activity, a LiteLLM model group and add metadata conditions. It shows run names, timestamps, and trace IDs; open a run to inspect its original steps before starting analysis. Suggestions come from up to 100 recent executions and may not include every recorded attribute. You can enter other exact keys and values. Leave service and filters blank for all activity your account can access. Filters are exact key/value matches, combined with AND. Trace filters match span or resource attributes on the same span. Request filters match logged metadata, including caller metadata stored under `requester_metadata`; `tag=value` matches request tags. `swarm=research` works only if your instrumentation records that attribute
+
+Write a few questions, give context about a successful run, choose a model, and set the monthly limit and sample size. Choose an initial history window from 1 hour to 30 days, in hours or days. Creation queues the first scan over that window. New lenses run once by default; opt into background monitoring for a custom interval from 1 minute to 7 days, entered in minutes, hours, or days. **Analyze now** checks activity since the last successful scan; **Recheck the last 24 hours** revisits recent history. The runs API accepts `lookback_hours` from 1 to 720 for other historical windows
+
+Pausing stops future scheduled scans; cancel the active scan separately if needed. The worker polls every 10 seconds; creating a lens or clicking Analyze now queues a scan, and due schedules are queued when the worker polls. Scans for the same lens never overlap, and its next interval starts after completion. Closing the browser does not stop the worker. Configuration edits apply to the next scan. A running scan retains its settings and selected execution IDs across retries
+
+## Read the results
+
+Needs attention shows issues, highest priority first. Patterns contains useful trends and successful behavior that may not need a fix. Each finding starts with a short explanation and a next step when useful. Expand the limitations for uncertainty and counterexamples. Evidence is grouped by run and collapsed until you need it; each quote opens the original step
+
+The Runs tab lists the actual sample frozen for the latest scan. Linked-run counts on findings include cited counterexamples, so they are not failure counts. The Scans tab shows history and coverage. Existing findings retain their original wording; the shorter summaries apply to new analysis
+
+## What a scan does
+
+The proxy selects newly received or updated executions with a two-minute settling period and a five-minute overlap. Older rows without receipt timestamps use execution end time. Overlapping scans do not increment a finding's occurrence count for the same execution ID
+
+A trace is spans sharing a trace ID within one team, not an automatically reconstructed conversation session. Requests are individual LLM calls. When both sources are enabled, requests correlated to a recorded span by response ID are excluded to reduce double counting
+
+The worker screens a deterministic sample, at most the configured 1–500 executions. For each execution it reads up to 160 spans, with 8,000 characters per span section, and splits these into model calls. It consolidates observations across batches, then investigates at most 10 candidate patterns using up to five model turns each. The dashboard shows these three stages, completed work counts, and elapsed time; progress is based on the selected sample, not every eligible execution. The investigator can read more original content from the selected executions. It has no shell, browsing, code-editing, or production-action tools
+
+Each model response must match a bounded JSON schema. A malformed response gets one repair attempt through the same budget controls; repeated invalid output fails the scan. Both the worker and proxy validate quoted evidence. Findings retain exact quotes and open the source trace or request. Resolve a finding after a fix, or dismiss it with a reason. A resolved finding reopens when new execution IDs support the same pattern; dismissed findings remain dismissed
+
+Coverage distinguishes eligible, sampled, reviewed, partial, and unassessable executions. Findings describe observations in the sample, not population-wide success rates or proven causes. A root span does not prove that a trace contains every expected span. Long, missing, redacted, or expired content limits the conclusions
+
+## Operations and limits
+
+PostgreSQL stores configurations, findings and the latest 50 jobs. Workers claim jobs with optimistic concurrency and a five-minute lease, renewed every 30 seconds. A disconnected job can be reclaimed up to three times. Cancellation stops subsequent work; a model call already in flight may finish and incur cost
+
+Before every model call, Lens reserves a conservative amount against the monthly lens budget. Successful calls reconcile to reported cost where pricing is available. Interrupted calls retain their reservation because the provider may have charged. A scan stops when the next reservation would exceed the limit, so it can stop with some budget remaining. Lens budgets are separate from virtual-key budgets; analysis calls use the proxy router directly
+
+V1 requires ClickHouse for both sources. It does not reconstruct sessions from unrelated trace IDs, guarantee exhaustive reviews, cache all per-execution observations across scans, or automatically fix agent code. Trace contents can change as late spans arrive, even though a job's selected IDs are fixed. Findings should be reviewed by a person before acting on them

deploy/lens/compose.build.yaml

@@ -0,0 +1,6 @@
+services:
+  lens-worker:
+    build:
+      context: ../..
+      dockerfile: deploy/lens/Dockerfile
+    image: litellm-lens-worker:local

deploy/lens/compose.yaml

@@ -0,0 +1,10 @@
+services:
+  lens-worker:
+    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:47445afedfb6de2ae37a3a246ea1c939196bfd365436a880ab96ecf5f42b2342}
+    environment:
+      LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
+      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}
+    restart: unless-stopped
+    read_only: true
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]

docker/Dockerfile.non_root

@@ -103,7 +103,7 @@ ENV LITELLM_NON_ROOT=true
 
 RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
     cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
-    cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
+    cp /app/litellm/proxy/logo.png /var/lib/litellm/assets/logo.png && \
     touch /var/lib/litellm/ui/.litellm_ui_ready
 
 RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \

docker/docker-compose.tracing.yml

@@ -0,0 +1,62 @@
+name: litellm-tracing
+
+services:
+  litellm:
+    build:
+      context: ..
+      target: runtime
+    command: ["--config", "/app/tracing-config.yaml", "--port", "4000"]
+    environment:
+      LITELLM_MASTER_KEY: local-tracing-master-key
+      LITELLM_SALT_KEY: sk-local-tracing-salt-key
+      DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm
+      STORE_MODEL_IN_DB: "True"
+      CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123
+      CLICKHOUSE_READER_URL: http://default:local-tracing@clickhouse:8123
+      CLICKHOUSE_DATABASE: litellm
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-}
+    volumes:
+      - ./tracing-config.yaml:/app/tracing-config.yaml:ro
+    ports:
+      - "127.0.0.1:4002:4000"
+    depends_on:
+      db:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: litellm
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "127.0.0.1:15432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:26.9.6.6
+    environment:
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_PASSWORD: local-tracing
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    ports:
+      - "127.0.0.1:18123:8123"
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-tracing", "--query", "SELECT 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+
+volumes:
+  postgres_data:
+  clickhouse_data:

docker/tracing-config.yaml

@@ -0,0 +1,10 @@
+model_list:
+  - model_name: gpt-6.1-sol
+    litellm_params:
+      model: openai/gpt-6.1-sol
+      api_key: os.environ/OPENAI_API_KEY
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  tracing:
+    store: clickhouse

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -22,10 +22,8 @@ from litellm._uuid import uuid
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.management_endpoints.common_utils import (
-    _is_user_team_admin,  # pyright: ignore[reportPrivateUsage]  # shared owner of team-admin membership
-    _set_object_metadata_field,
-)
+from litellm.proxy.management.teams.access import is_team_admin
+from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
 from litellm.proxy.management_helpers.utils import (
     management_endpoint_wrapper,
@@ -117,7 +115,7 @@ async def _check_user_permission_for_project(
         return False
 
     team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
-    return _is_user_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
+    return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
 
 
 async def _validate_team_exists(

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.71"
+version = "0.1.72"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.71"
+version = "0.1.72"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/routes/allowlist.py

@@ -73,6 +73,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/v1/containers",
     "/containers",
     "/v1/evals",
+    "/v1/traces",
     "/v1/memory",
     "/queue/chat/",
     # Google data plane (v1beta is the Google AI Studio version)

litellm-proxy-extras/litellm_proxy_extras/migrations/20260921190000_agent_identity/migration.sql

@@ -0,0 +1,97 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AgentsTable" ADD COLUMN IF NOT EXISTS "enabled" BOOLEAN NOT NULL DEFAULT true,
+ADD COLUMN IF NOT EXISTS "execution_mode" TEXT NOT NULL DEFAULT 'autonomous',
+ADD COLUMN IF NOT EXISTS "identity_managed" BOOLEAN NOT NULL DEFAULT false;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "billing_agent_id" TEXT;
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_AgentIdentity" (
+    "agent_id" TEXT NOT NULL,
+    "active" BOOLEAN NOT NULL DEFAULT true,
+    "provider" TEXT NOT NULL,
+    "issuer" TEXT NOT NULL,
+    "tenant_id" TEXT NOT NULL,
+    "client_id" TEXT NOT NULL,
+    "service_principal_id" TEXT,
+    "required_roles" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "required_scopes" TEXT[] DEFAULT ARRAY['user_impersonation']::TEXT[],
+    "revision" TEXT NOT NULL,
+    "last_authenticated_at" TIMESTAMP(3),
+
+    CONSTRAINT "LiteLLM_AgentIdentity_pkey" PRIMARY KEY ("agent_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_RetiredAgentIdentity" (
+    "binding_id" TEXT NOT NULL,
+    "agent_id" TEXT,
+    "provider" TEXT NOT NULL,
+    "issuer" TEXT NOT NULL,
+    "tenant_id" TEXT NOT NULL,
+    "client_id" TEXT NOT NULL,
+
+    CONSTRAINT "LiteLLM_RetiredAgentIdentity_pkey" PRIMARY KEY ("binding_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_RetiredAgent" (
+    "original_agent_id" TEXT NOT NULL,
+    "retired_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_RetiredAgent_pkey" PRIMARY KEY ("original_agent_id")
+);
+
+-- CreateTable
+CREATE TABLE IF NOT EXISTS "LiteLLM_VerifiedSubject" (
+    "subject_id" TEXT NOT NULL,
+    "issuer" TEXT NOT NULL,
+    "tenant_id" TEXT NOT NULL,
+    "oid" TEXT NOT NULL,
+    "kind" TEXT NOT NULL DEFAULT 'human',
+    "user_id" TEXT,
+    "verified_via" TEXT NOT NULL DEFAULT 'sso_interactive',
+    "verified_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_VerifiedSubject_pkey" PRIMARY KEY ("subject_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_AgentIdentity_provider_tenant_id_client_id_key" ON "LiteLLM_AgentIdentity"("provider", "tenant_id", "client_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_AgentIdentity_issuer_service_principal_id_key" ON "LiteLLM_AgentIdentity"("issuer", "service_principal_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_RetiredAgentIdentity_provider_tenant_id_client_id_key" ON "LiteLLM_RetiredAgentIdentity"("provider", "tenant_id", "client_id");
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_VerifiedSubject_user_id_idx" ON "LiteLLM_VerifiedSubject"("user_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_VerifiedSubject_issuer_tenant_id_oid_key" ON "LiteLLM_VerifiedSubject"("issuer", "tenant_id", "oid");
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_AgentIdentity_agent_id_fkey') THEN
+        ALTER TABLE "LiteLLM_AgentIdentity" ADD CONSTRAINT "LiteLLM_AgentIdentity_agent_id_fkey" FOREIGN KEY ("agent_id") REFERENCES "LiteLLM_AgentsTable"("agent_id") ON DELETE CASCADE ON UPDATE CASCADE;
+    END IF;
+END $$;
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_RetiredAgentIdentity_agent_id_fkey') THEN
+        ALTER TABLE "LiteLLM_RetiredAgentIdentity" ADD CONSTRAINT "LiteLLM_RetiredAgentIdentity_agent_id_fkey" FOREIGN KEY ("agent_id") REFERENCES "LiteLLM_AgentsTable"("agent_id") ON DELETE SET NULL ON UPDATE CASCADE;
+    END IF;
+END $$;
+
+-- AddForeignKey
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'LiteLLM_VerifiedSubject_user_id_fkey') THEN
+        ALTER TABLE "LiteLLM_VerifiedSubject" ADD CONSTRAINT "LiteLLM_VerifiedSubject_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "LiteLLM_UserTable"("user_id") ON DELETE CASCADE ON UPDATE CASCADE;
+    END IF;
+END $$;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260930000000_agent_engine/migration.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_Engine" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "version" INTEGER NOT NULL DEFAULT 0,
+    "data" JSONB NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "LiteLLM_EngineWorker" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "token_hash" TEXT NOT NULL UNIQUE,
+    "data" JSONB NOT NULL
+);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -78,6 +78,11 @@ model LiteLLM_AgentsTable {
   object_permission_id String?
   object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
   spend      Float    @default(0.0)
+  identity_managed Boolean @default(false)
+  enabled Boolean @default(true)
+  execution_mode String @default("autonomous")
+  identity LiteLLM_AgentIdentity?
+  retired_identities LiteLLM_RetiredAgentIdentity[]
   tpm_limit  Int?
   rpm_limit  Int?
   session_tpm_limit Int?
@@ -88,6 +93,56 @@ model LiteLLM_AgentsTable {
   updated_by String
 }
 
+model LiteLLM_AgentIdentity {
+  agent_id String @id
+  active Boolean @default(true)
+  agent LiteLLM_AgentsTable @relation(fields: [agent_id], references: [agent_id], onDelete: Cascade)
+  provider String
+  issuer String
+  tenant_id String
+  client_id String
+  service_principal_id String?
+  required_roles String[] @default([])
+  required_scopes String[] @default(["user_impersonation"])
+  revision String @default(uuid())
+  last_authenticated_at DateTime?
+  @@unique([provider, tenant_id, client_id])
+  @@unique([issuer, service_principal_id])
+}
+
+model LiteLLM_RetiredAgentIdentity {
+  binding_id String @id @default(uuid())
+  agent_id String?
+  agent LiteLLM_AgentsTable? @relation(fields: [agent_id], references: [agent_id], onDelete: SetNull)
+  provider String
+  issuer String
+  tenant_id String
+  client_id String
+  @@unique([provider, tenant_id, client_id])
+}
+
+model LiteLLM_RetiredAgent {
+  original_agent_id String @id
+  retired_at DateTime @default(now())
+}
+
+model LiteLLM_VerifiedSubject {
+  subject_id String @id @default(uuid())
+  issuer String
+  tenant_id String
+  oid String
+  kind String @default("human")
+  user_id String?
+  user LiteLLM_UserTable? @relation(fields: [user_id], references: [user_id], onDelete: Cascade)
+  verified_via String @default("sso_interactive")
+  verified_at DateTime @default(now())
+  @@unique([issuer, tenant_id, oid])
+  @@index([user_id])
+}
+
+
+
+
 model LiteLLM_OrganizationTable {
 		organization_id String @id @default(uuid())
     organization_alias  String
@@ -241,6 +296,7 @@ model LiteLLM_DeletedTeamTable {
 
 // Track spend, rate limit, budget Users
 model LiteLLM_UserTable {
+    verified_subjects LiteLLM_VerifiedSubject[]
 		user_id    String @id
     user_alias String? 
     team_id    String?
@@ -675,6 +731,7 @@ model LiteLLM_SpendLogs {
   session_id          String?
   status              String?
   mcp_namespaced_tool_name String? 
+  billing_agent_id    String?
   agent_id            String?
   proxy_server_request Json?     @default("{}")
   litellm_call_id     String?
@@ -1837,3 +1894,15 @@ model LiteLLM_WorkflowMessage {
   @@unique([run_id, sequence_number])
   @@index([run_id])
 }
+
+model LiteLLM_Engine {
+  id      String @id
+  version Int @default(0)
+  data    Json
+}
+
+model LiteLLM_EngineWorker {
+  id         String @id
+  token_hash String @unique
+  data       Json
+}

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.102"
+version = "0.4.103"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.102"
+version = "0.4.103"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/Cargo.lock

@@ -1274,6 +1274,18 @@ version = "0.4.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
 
+[[package]]
+name = "const-hex"
+version = "1.19.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.2.17",
+ "proptest",
+ "serde_core",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -2372,9 +2384,9 @@ dependencies = [
  "http-body-util",
  "hyper 1.10.1",
  "lazy_static",
- "opentelemetry",
+ "opentelemetry 0.32.0",
  "opentelemetry-semantic-conventions",
- "opentelemetry_sdk",
+ "opentelemetry_sdk 0.32.1",
  "percent-encoding",
  "pin-project",
  "prost",
@@ -4075,6 +4087,7 @@ dependencies = [
  "litellm-secrets-aws",
  "litellm-secrets-types",
  "litellm-token-counter",
+ "litellm-traces",
  "litellm-tracing",
  "pyo3",
  "pyo3-async-runtimes",
@@ -4351,6 +4364,26 @@ dependencies = [
  "tiktoken-rs",
 ]
 
+[[package]]
+name = "litellm-traces"
+version = "0.1.0"
+dependencies = [
+ "base64 0.22.1",
+ "flate2",
+ "litellm-http",
+ "opentelemetry-proto",
+ "prost",
+ "rstest",
+ "serde",
+ "serde_json",
+ "sha2 0.10.9",
+ "testcontainers-modules",
+ "thiserror 2.0.19",
+ "time",
+ "tokio",
+ "url",
+]
+
 [[package]]
 name = "litellm-tracing"
 version = "0.1.0"
@@ -4760,6 +4793,33 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "opentelemetry"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "js-sys",
+ "pin-project-lite",
+ "thiserror 2.0.19",
+]
+
+[[package]]
+name = "opentelemetry-proto"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d"
+dependencies = [
+ "base64 0.22.1",
+ "const-hex",
+ "opentelemetry 0.33.0",
+ "opentelemetry_sdk 0.33.0",
+ "prost",
+ "serde",
+]
+
 [[package]]
 name = "opentelemetry-semantic-conventions"
 version = "0.32.1"
@@ -4775,7 +4835,23 @@ dependencies = [
  "futures-channel",
  "futures-executor",
  "futures-util",
- "opentelemetry",
+ "opentelemetry 0.32.0",
+ "percent-encoding",
+ "portable-atomic",
+ "rand 0.9.5",
+ "thiserror 2.0.19",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.33.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520"
+dependencies = [
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "opentelemetry 0.33.0",
  "percent-encoding",
  "portable-atomic",
  "rand 0.9.5",
@@ -5704,6 +5780,7 @@ checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
 dependencies = [
  "base64 0.23.1",
  "bytes",
+ "encoding_rs",
  "futures-core",
  "futures-util",
  "h2 0.4.15",
@@ -5715,6 +5792,7 @@ dependencies = [
  "hyper-util",
  "js-sys",
  "log",
+ "mime",
  "percent-encoding",
  "pin-project-lite",
  "quinn",
@@ -6945,6 +7023,7 @@ dependencies = [
  "memchr",
  "parse-display",
  "pin-project-lite",
+ "reqwest 0.13.5",
  "serde",
  "serde_json",
  "serde_with",
@@ -7505,7 +7584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
 dependencies = [
  "js-sys",
- "opentelemetry",
+ "opentelemetry 0.32.0",
  "tracing",
  "tracing-core",
  "tracing-subscriber",

litellm-rust/Cargo.toml

@@ -12,6 +12,7 @@ repository = "https://github.com/BerriAI/litellm"
 litellm-config = { path = "crates/config" }
 litellm-router = { path = "crates/router" }
 litellm-tracing = { path = "crates/tracing" }
+litellm-traces = { path = "crates/traces" }
 litellm-core = { path = "crates/core" }
 litellm-gateway-mcp = { path = "crates/gateway-mcp" }
 litellm-gateway = { path = "crates/gateway" }

litellm-rust/crates/cache-response/AGENTS.md

@@ -24,6 +24,6 @@ Keep unary caching independent of stream-only methods. Store streams only after
 
 Test each contract in its owner: storage capabilities in backend tests, envelopes and freshness here, reuse and replay in core, Python callback and fallback behavior at the bridge, and HTTP behavior at the gateway. Run backend contract checks and Python response-codec fixtures before exposing a new backend
 
-`ScopedCache` requires an explicit shared or isolated scope at construction. `CacheOptions` has no default sharing policy. Callers may override policy per invocation without replacing the attached service. Versioned native envelopes reject incompatible API surfaces and versions as misses; this envelope is distinct from the legacy Python response codec
+`ScopedCache` requires an explicit shared or isolated scope at construction. Per-call `CachePolicy` controls reads, writes, expiry, and freshness without replacing the attached scope or service. `CacheOptions` binds that policy to an explicit scope for storage requests and has no default sharing policy. Versioned native envelopes reject incompatible API surfaces and versions as misses; this envelope is distinct from the legacy Python response codec
 
 Response storage is not the source of budget or rate-limit coordination dependencies. Keep counters, reservations, and atomic admission operations out of `ResponseCacheService`, including when both services happen to use Redis

litellm-rust/crates/cache-response/src/lib.rs

@@ -17,6 +17,6 @@ pub use exact::{ConnectionProbe, ExactResponseCache};
 pub use response::{ResponseCache, ResponseCacheRequest};
 
 pub use service::{
-    CacheOptions, CacheScope, ResponseCacheConfig, ResponseCacheService, ResponseEnvelope,
-    ScopedCache,
+    CacheOptions, CachePolicy, CacheScope, ResponseCacheConfig, ResponseCacheService,
+    ResponseEnvelope, ScopedCache,
 };

litellm-rust/crates/cache-response/src/service.rs

@@ -73,32 +73,35 @@ pub enum CacheScope {
     Isolated(String),
 }
 
-#[derive(Clone)]
-pub struct CacheOptions {
+#[derive(Clone, Copy, Default)]
+pub struct CachePolicy {
     pub caching: Option<bool>,
     pub no_cache: bool,
     pub no_store: bool,
     pub ttl: Option<Duration>,
     pub max_age: Option<Duration>,
+}
+
+impl CachePolicy {
+    pub fn enabled(&self) -> bool {
+        self.caching != Some(false) && !(self.no_cache && self.no_store)
+    }
+}
+
+#[derive(Clone)]
+pub struct CacheOptions {
+    pub policy: CachePolicy,
     pub scope: CacheScope,
 }
 
 impl CacheOptions {
     pub fn new(scope: CacheScope) -> Self {
         Self {
-            caching: None,
-            no_cache: false,
-            no_store: false,
-            ttl: None,
-            max_age: None,
+            policy: CachePolicy::default(),
             scope,
         }
     }
 
-    pub fn enabled(&self) -> bool {
-        self.caching != Some(false) && !(self.no_cache && self.no_store)
-    }
-
     pub fn request(self, namespace: &str, surface: &str, mut input: Value) -> ResponseCacheRequest {
         input.sort_all_objects();
         let scope = match self.scope {
@@ -128,13 +131,15 @@ impl CacheOptions {
                 supported_call_type: true,
                 native_backend: true,
                 default_on: true,
-                caching: self.caching,
-                no_cache: self.no_cache,
-                no_store: self.no_store,
+                caching: self.policy.caching,
+                no_cache: self.policy.no_cache,
+                no_store: self.policy.no_store,
                 ..Default::default()
             },
-            context: ExactCacheContext { ttl: self.ttl },
-            max_age: self.max_age,
+            context: ExactCacheContext {
+                ttl: self.policy.ttl,
+            },
+            max_age: self.policy.max_age,
         }
     }
 }
@@ -171,7 +176,10 @@ impl ScopedCache {
         Self { service, scope }
     }
 
-    pub fn options(&self, overrides: Option<CacheOptions>) -> CacheOptions {
-        overrides.unwrap_or_else(|| CacheOptions::new(self.scope.clone()))
+    pub fn options(&self, policy: Option<CachePolicy>) -> CacheOptions {
+        CacheOptions {
+            policy: policy.unwrap_or_default(),
+            scope: self.scope.clone(),
+        }
     }
 }

litellm-rust/crates/cache-response/tests/service.rs

@@ -129,11 +129,20 @@ async fn isolated_policy_controls_actual_entry_reuse(
     #[case] first: &str,
     #[case] second: &str,
     #[case] hit: bool,
+    #[values(false, true)] override_policy: bool,
 ) {
-    use litellm_cache_response::{CacheOptions, CacheScope};
-    let service = ResponseCache::new(Arc::new(InMemoryCache::<CacheEntry>::default()));
-    let request =
-        |scope| CacheOptions::new(scope).request("test", "messages", json!({"prompt":"hello"}));
+    use litellm_cache_response::{CachePolicy, CacheScope, ScopedCache};
+    let service = Arc::new(ResponseCache::new(Arc::new(
+        InMemoryCache::<CacheEntry>::default(),
+    )));
+    let request = |scope| {
+        ScopedCache::new(service.clone(), scope)
+            .options(override_policy.then_some(CachePolicy {
+                ttl: Some(Duration::from_secs(30)),
+                ..CachePolicy::default()
+            }))
+            .request("test", "messages", json!({"prompt":"hello"}))
+    };
     service
         .async_store(
             &request(CacheScope::Isolated(first.into())),

litellm-rust/crates/core/AGENTS.md

@@ -36,7 +36,9 @@ Not here: serving HTTP (axum routes, extractors), config file reading, rollout s
 
 ## Response caching and accounting boundary
 
-Attach a `litellm_cache_response::ScopedCache` with `route.with_cache(cache)`. Cached and uncached routes use the same `execute` and `machine` methods. `CallOptions` carries per-call cache overrides and observation; attaching a service does not change the execution contract
+Attach a `litellm_cache_response::ScopedCache` with `route.with_cache(cache)`. Cached and uncached routes use the same `execute` and `machine` methods. `CallOptions` carries a scope-free `CachePolicy` and observation; per-call policy never replaces the attached scope or service
+
+Messages groups per-call dependencies in `CallContext` and explicitly sequences cache lookup, provider execution, result acceptance, and cache storage. Provider transport does not own cache orchestration. Stream capture remains in the shared cache implementation
 
 Core owns request identity, typed response reconstruction and stream capture/replay. `cache-response` owns cache policy, namespacing, scope encoding, versioned envelopes and freshness. The SDK explicitly chooses shared scope. The gateway derives isolated scope from authenticated identity before attaching its service
 

litellm-rust/crates/core/src/caching.rs

@@ -1,5 +1,6 @@
 use std::{
     future::Future,
+    marker::PhantomData,
     sync::Arc,
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
@@ -7,7 +8,8 @@ use std::{
 use bytes::{Bytes, BytesMut};
 use futures_util::{StreamExt, TryStreamExt, stream};
 use litellm_cache_response::{
-    CacheOptions, ResponseCacheRequest, ResponseCacheService, ResponseEnvelope, cache_key,
+    CacheOptions, CachePolicy, ResponseCacheRequest, ResponseCacheService, ResponseEnvelope,
+    ScopedCache, cache_key,
 };
 use litellm_host::{
     call::{CallOutput, OutputOf},
@@ -77,7 +79,7 @@ impl CacheSession {
         options: Option<CacheOptions>,
         request: &CacheRequest,
     ) -> Option<Self> {
-        let options = options.filter(CacheOptions::enabled)?;
+        let options = options.filter(|options| options.policy.enabled())?;
         let service = service?;
         let input = request.input.clone();
         let request = options.request(&service.config().namespace, P::SURFACE, input);
@@ -198,81 +200,137 @@ where
     let identity = request.identity.clone();
     crate::diagnostic::provider(&identity.model, &identity.provider);
     let session = CacheSession::prepare::<P>(cache, options, &request);
-    let hit = match &session {
-        Some(session) => session.lookup::<P>().await.and_then(|entry| {
-            let output = match entry {
-                CachedOutput::Response(response) => Some(CallOutput::Complete(response)),
-                CachedOutput::Stream(data) => P::replay(Bytes::from(data)),
-            };
-            output.map(|output| (output, cache_key(&session.request.key)))
-        }),
-        None => None,
+    let cache = CallCache::<P> {
+        session,
+        protocol: PhantomData,
     };
+    let hit = cache.lookup().await;
     let (output, source) = match hit {
-        Some((output, key)) => (output, ResultSource::Cache { key }),
+        Some(hit) => hit,
         None => (provider().await?, ResultSource::Provider),
     };
-    let from_provider = source == ResultSource::Provider;
     publish(
         ExecutionFacts {
             provider: identity,
-            source,
+            source: source.clone(),
         },
         interceptors,
         observers,
     )
     .await?;
-    let Some(session) =
-        session.filter(|session| from_provider && session.request.controls.writes())
-    else {
-        return Ok(output);
-    };
-    match output {
-        CallOutput::Complete(response) => {
-            session.store_response::<P>(&response).await;
-            Ok(CallOutput::Complete(response))
-        }
-        CallOutput::Stream { head, chunks } => {
-            let captured = stream::try_unfold(
-                (chunks, Some(Vec::<u8>::new()), session),
-                |(mut chunks, captured, session)| async move {
-                    match chunks.try_next().await? {
-                        Some(chunk) => {
-                            let captured = captured.and_then(|mut data| {
-                                let bytes = P::bytes(&chunk);
-                                if data.len().saturating_add(bytes.len())
-                                    > session.service.config().max_entry_bytes
-                                {
-                                    return None;
-                                }
-                                data.extend_from_slice(bytes);
-                                Some(data)
-                            });
-                            Ok(Some((chunk, (chunks, captured, session))))
-                        }
-                        None => {
-                            if let Some(data) = captured
-                                && let Ok(text) = String::from_utf8(data)
-                                && successful_stream(&text, P::TERMINAL_EVENT)
-                                && let Ok(entry) = serde_json::to_value(ResponseEnvelope::new(
-                                    P::SURFACE,
-                                    CachedOutput::<Value>::Stream(text),
-                                ))
-                            {
-                                session.store(entry).await;
-                            }
-                            Ok::<_, RouteError>(None)
-                        }
-                    }
-                },
-            )
-            .boxed();
-            Ok(CallOutput::Stream {
-                head,
-                chunks: captured,
+    Ok(cache.finish(output, &source).await)
+}
+
+pub(crate) struct CallCache<P> {
+    session: Option<CacheSession>,
+    protocol: PhantomData<P>,
+}
+
+impl<P: StreamCachable> CallCache<P> {
+    pub(crate) fn from_wire(
+        cache: Option<&ScopedCache>,
+        policy: CachePolicy,
+        identity: &ProviderIdentity,
+        wire: &WireRequest,
+    ) -> Self {
+        let session = cache.and_then(|cache| {
+            if !policy.enabled() {
+                return None;
+            }
+            let options = cache.options(Some(policy));
+            let request = CacheRequest::from_wire(identity.clone(), Some(wire));
+            Some(CacheSession {
+                request: options.request(
+                    &cache.service.config().namespace,
+                    P::SURFACE,
+                    request.input,
+                ),
+                service: cache.service.clone(),
             })
+        });
+        Self {
+            session,
+            protocol: PhantomData,
         }
     }
+
+    pub(crate) async fn lookup(&self) -> Option<(OutputOf<P>, ResultSource)>
+    where
+        P::Response: DeserializeOwned,
+    {
+        let session = self.session.as_ref()?;
+        let output = match session.lookup::<P>().await? {
+            CachedOutput::Response(response) => CallOutput::Complete(response),
+            CachedOutput::Stream(data) => P::replay(Bytes::from(data))?,
+        };
+        Some((
+            output,
+            ResultSource::Cache {
+                key: cache_key(&session.request.key),
+            },
+        ))
+    }
+
+    pub(crate) async fn finish(self, output: OutputOf<P>, source: &ResultSource) -> OutputOf<P>
+    where
+        P::Response: Serialize,
+    {
+        let Some(session) = self.session.filter(|session| {
+            *source == ResultSource::Provider && session.request.controls.writes()
+        }) else {
+            return output;
+        };
+        match output {
+            CallOutput::Complete(response) => {
+                session.store_response::<P>(&response).await;
+                CallOutput::Complete(response)
+            }
+            CallOutput::Stream { head, chunks } => CallOutput::Stream {
+                head,
+                chunks: capture_stream::<P>(chunks, session),
+            },
+        }
+    }
+}
+
+fn capture_stream<P: StreamCachable>(
+    chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>>,
+    session: CacheSession,
+) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>> {
+    stream::try_unfold(
+        (chunks, Some(Vec::<u8>::new()), session),
+        |(mut chunks, captured, session)| async move {
+            match chunks.try_next().await? {
+                Some(chunk) => {
+                    let captured = captured.and_then(|mut data| {
+                        let bytes = P::bytes(&chunk);
+                        if data.len().saturating_add(bytes.len())
+                            > session.service.config().max_entry_bytes
+                        {
+                            return None;
+                        }
+                        data.extend_from_slice(bytes);
+                        Some(data)
+                    });
+                    Ok(Some((chunk, (chunks, captured, session))))
+                }
+                None => {
+                    if let Some(data) = captured
+                        && let Ok(text) = String::from_utf8(data)
+                        && successful_stream(&text, P::TERMINAL_EVENT)
+                        && let Ok(entry) = serde_json::to_value(ResponseEnvelope::new(
+                            P::SURFACE,
+                            CachedOutput::<Value>::Stream(text),
+                        ))
+                    {
+                        session.store(entry).await;
+                    }
+                    Ok::<_, RouteError>(None)
+                }
+            }
+        },
+    )
+    .boxed()
 }
 
 fn now() -> Duration {

litellm-rust/crates/core/src/chat_completions/handler.rs

@@ -22,7 +22,7 @@ pub(super) async fn execute(
     auth: &AuthServices,
     request: ProviderChatCompletionsRequest,
     cache: Option<litellm_cache_response::ScopedCache>,
-    cache_options: Option<litellm_cache_response::CacheOptions>,
+    cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ChatCompletionsResponse, Error> {

litellm-rust/crates/core/src/chat_completions/mod.rs

@@ -67,7 +67,7 @@ impl ChatCompletionsRoute {
     async fn run(
         &self,
         request: ChatCompletionsRequest<'_>,
-        cache_options: Option<litellm_cache_response::CacheOptions>,
+        cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ChatCompletionsResponse, Error> {

litellm-rust/crates/core/src/chat_completions/route.rs

@@ -55,7 +55,7 @@ impl ChatCompletionsRoute {
     pub(super) async fn run_call(
         &self,
         call: ChatCompletionsCall,
-        cache_options: Option<litellm_cache_response::CacheOptions>,
+        cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ChatCompletionsResponse, Error> {

litellm-rust/crates/core/src/context.rs

@@ -0,0 +1,48 @@
+use litellm_cache_response::CachePolicy;
+use litellm_host::{
+    interceptors::{ExecutionFacts, Interceptors, RawResponse},
+    lifecycle::{CallEvent, ExecutionEvent},
+    observation::ObservationSender,
+};
+
+use crate::{CallOptions, RouteError};
+
+pub(crate) struct CallContext<'a, I> {
+    pub interceptors: &'a I,
+    pub observers: Option<ObservationSender>,
+    pub cache: CachePolicy,
+}
+
+impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
+    pub fn new(interceptors: &'a I, options: CallOptions) -> Self {
+        Self {
+            interceptors,
+            observers: options.observers,
+            cache: options.cache.unwrap_or_default(),
+        }
+    }
+
+    pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
+        if let Some(observers) = &self.observers {
+            observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
+                facts: facts.clone(),
+            }));
+        }
+        self.interceptors.result_ready(facts).await
+    }
+
+    pub async fn response_received(&self, body: &str) -> Result<(), RouteError> {
+        let raw = RawResponse {
+            body: body.to_owned(),
+        };
+        if let Some(observers) = &self.observers {
+            observers.emit(CallEvent::Execution(
+                ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
+            ));
+        }
+        self.interceptors
+            .after_provider_response(raw)
+            .await
+            .map_err(RouteError::post_call)
+    }
+}

litellm-rust/crates/core/src/lib.rs

@@ -1,3 +1,4 @@
+mod context;
 mod diagnostic;
 
 pub mod audio_transcription;
@@ -16,7 +17,7 @@ pub use error::RouteError;
 
 #[derive(Clone, Default)]
 pub struct CallOptions {
-    pub cache: Option<litellm_cache_response::CacheOptions>,
+    pub cache: Option<litellm_cache_response::CachePolicy>,
     pub observers: Option<litellm_host::observation::ObservationSender>,
 }
 
@@ -29,8 +30,8 @@ impl From<Option<litellm_host::observation::ObservationSender>> for CallOptions
     }
 }
 
-impl From<litellm_cache_response::CacheOptions> for CallOptions {
-    fn from(cache: litellm_cache_response::CacheOptions) -> Self {
+impl From<litellm_cache_response::CachePolicy> for CallOptions {
+    fn from(cache: litellm_cache_response::CachePolicy) -> Self {
         Self {
             cache: Some(cache),
             observers: None,

litellm-rust/crates/core/src/messages/handler.rs

@@ -1,10 +1,8 @@
-use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
 use std::time::Duration;
 
 use bytes::Bytes;
 use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
-use litellm_auth::AuthServices;
-use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
+use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
 use litellm_http::transport::Error as TransportError;
 use litellm_llms::base_llm::{
     auth::{Authenticated, resolve_auth},
@@ -18,102 +16,130 @@ use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
-    Error, MessagesCallResponse, common_utils::truncate_error_body,
+    Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
     prepare::ProviderMessagesRequest,
 };
-use crate::{constants::MESSAGES_TIMEOUT_SECS, outbound::outbound_request};
+use crate::{constants::MESSAGES_TIMEOUT_SECS, context::CallContext, outbound::outbound_request};
 
-pub(super) async fn execute(
-    http: &litellm_http::Client,
-    auth: &AuthServices,
-    request: ProviderMessagesRequest,
-    cache: Option<litellm_cache_response::ScopedCache>,
-    cache_options: Option<litellm_cache_response::CacheOptions>,
-    interceptors: &impl Interceptors<Error>,
-    observers: Option<&ObservationSender>,
-) -> Result<MessagesCallResponse, Error> {
-    let ProviderMessagesRequest {
-        provider,
-        url,
-        body,
-        environment,
-        timeout,
-        api_key,
-    } = request;
-    let stream = body.params.stream == Some(true);
-    let context = RequestContext {
-        model: body.model.clone(),
-        custom_llm_provider: provider.as_str().to_string(),
-        optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
-        secret_fields: Vec::new(),
-        api_key,
-    };
-    let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
-    let identity = litellm_host::interceptors::ProviderIdentity {
-        model: context.model.clone(),
-        provider: context.custom_llm_provider.clone(),
-    };
-    let wire = interceptors
-        .before_provider_request(
-            WireRequest {
-                url,
-                headers: authenticated.headers,
-                body: serde_json::to_value(&body).map_err(serialize_failure)?,
-            },
-            context,
-        )
-        .await?;
-    let cache = cache.filter(|_| authenticated.signer.is_none());
-    let cache_request =
-        crate::caching::CacheRequest::from_wire(identity, cache.as_ref().map(|_| &wire));
-    crate::caching::execute_streaming::<super::route::Messages, _, _>(
-        cache_request,
-        cache.as_ref().map(|cache| cache.service.clone()),
-        cache.as_ref().map(|cache| cache.options(cache_options)),
-        interceptors,
-        observers,
-        || async move {
-            let provider_name = provider.as_str();
-            log_request_body(provider_name, stream, &wire.body);
-            let response = send(
-                http,
-                Authenticated {
-                    headers: wire.headers,
-                    signer: authenticated.signer,
+pub(super) struct ProviderCall {
+    pub identity: ProviderIdentity,
+    pub wire: WireRequest,
+    provider: super::common_utils::MessagesProvider,
+    signer: Option<litellm_auth_aws::SigV4Signer>,
+    timeout: Option<Duration>,
+    stream: bool,
+}
+
+impl ProviderCall {
+    pub fn cacheable(&self) -> bool {
+        self.signer.is_none()
+    }
+}
+
+impl MessagesRoute {
+    pub(super) async fn prepare_outbound(
+        &self,
+        request: ProviderMessagesRequest,
+        context: &CallContext<'_, impl Interceptors<Error>>,
+    ) -> Result<ProviderCall, Error> {
+        let ProviderMessagesRequest {
+            provider,
+            url,
+            body,
+            environment,
+            timeout,
+            api_key,
+        } = request;
+        let request_context = RequestContext {
+            model: body.model.clone(),
+            custom_llm_provider: provider.as_str().to_string(),
+            optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
+            secret_fields: Vec::new(),
+            api_key,
+        };
+        let authenticated =
+            resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
+        let identity = ProviderIdentity {
+            model: request_context.model.clone(),
+            provider: request_context.custom_llm_provider.clone(),
+        };
+        let wire = context
+            .interceptors
+            .before_provider_request(
+                WireRequest {
+                    url,
+                    headers: authenticated.headers,
+                    body: serde_json::to_value(&body).map_err(serialize_failure)?,
                 },
-                &wire.url,
-                &wire.body,
-                timeout,
+                request_context,
             )
             .await?;
-            if !response.status().is_success() {
-                return Err(provider_error(response).await);
-            }
-            let config = provider.config();
-            if stream {
-                return Ok(streaming_response(
-                    response,
-                    config.stream_decoder(),
-                    provider_name,
+        let stream = match wire.body.get("stream") {
+            None | Some(Value::Null) => false,
+            Some(Value::Bool(stream)) => *stream,
+            Some(value) => {
+                return Err(Error::InvalidRequest(
+                    litellm_llms::ErrorDetail::InvalidValue {
+                        field: "stream",
+                        expected: "a boolean",
+                        actual: value.clone(),
+                    },
                 ));
             }
-            let text = response.text().await.map_err(network)?;
-            log_response_body(&text);
-            let raw = RawResponse { body: text.clone() };
-            if let Some(observers) = observers {
-                observers.emit(litellm_host::lifecycle::CallEvent::Execution(
-                    ExecutionEvent::ProviderResponseReceived { raw: raw.clone() },
-                ));
-            }
-            interceptors
-                .after_provider_response(raw)
-                .await
-                .map_err(Error::post_call)?;
-            decode_response(config, &body.model, &text)
-                .map(|message| MessagesCallResponse::Complete(Box::new(message)))
-        },
-    )
-    .await
+        };
+        Ok(ProviderCall {
+            identity,
+            wire,
+            provider,
+            signer: authenticated.signer,
+            timeout,
+            stream,
+        })
+    }
+
+    pub(super) async fn call_provider(
+        &self,
+        request: ProviderCall,
+        context: &CallContext<'_, impl Interceptors<Error>>,
+    ) -> Result<MessagesCallResponse, Error> {
+        let ProviderCall {
+            identity,
+            wire,
+            provider,
+            signer,
+            timeout,
+            stream,
+        } = request;
+        let provider_name = provider.as_str();
+        log_request_body(provider_name, stream, &wire.body);
+        let response = send(
+            &self.http,
+            Authenticated {
+                headers: wire.headers,
+                signer,
+            },
+            &wire.url,
+            &wire.body,
+            timeout,
+        )
+        .await?;
+        if !response.status().is_success() {
+            return Err(provider_error(response).await);
+        }
+        let config = provider.config();
+        if stream {
+            return Ok(streaming_response(
+                response,
+                config.stream_decoder(),
+                provider_name,
+            ));
+        }
+        let text = response.text().await.map_err(network)?;
+        log_response_body(&text);
+        context.response_received(&text).await?;
+        decode_response(config, &identity.model, &text)
+            .map(|message| MessagesCallResponse::Complete(Box::new(message)))
+    }
 }
 
 fn serialize_failure(err: serde_json::Error) -> Error {

litellm-rust/crates/core/src/messages/mod.rs

@@ -1,11 +1,14 @@
-use litellm_host::observation::ObservationSender;
 mod common_utils;
 mod handler;
 mod prepare;
 pub mod route;
 mod types;
 
+use futures_util::FutureExt;
 use litellm_auth::AuthServices;
+use litellm_host::interceptors::{ExecutionFacts, Interceptors, ResultSource};
+
+use crate::{caching::CallCache, context::CallContext};
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
 
@@ -20,74 +23,18 @@ pub struct MessagesRoute {
     cache: Option<litellm_cache_response::ScopedCache>,
 }
 
-#[must_use]
-#[derive(Clone, Default)]
-pub struct MessagesRouteBuilder<Http = (), Auth = (), Secrets = ()> {
-    http: Http,
-    auth: Auth,
-    secrets: Secrets,
-    cache: Option<litellm_cache_response::ScopedCache>,
-}
-
-impl<Http, Auth, Secrets> MessagesRouteBuilder<Http, Auth, Secrets> {
-    pub fn with_http(
-        self,
-        http: litellm_http::Client,
-    ) -> MessagesRouteBuilder<litellm_http::Client, Auth, Secrets> {
-        MessagesRouteBuilder {
-            http,
-            auth: self.auth,
-            secrets: self.secrets,
-            cache: self.cache,
-        }
-    }
-
-    pub fn with_auth(
-        self,
-        auth: Arc<AuthServices>,
-    ) -> MessagesRouteBuilder<Http, Arc<AuthServices>, Secrets> {
-        MessagesRouteBuilder {
-            http: self.http,
-            auth,
-            secrets: self.secrets,
-            cache: self.cache,
-        }
-    }
-
-    pub fn with_secrets(
-        self,
-        secrets: Arc<dyn SecretSource>,
-    ) -> MessagesRouteBuilder<Http, Auth, Arc<dyn SecretSource>> {
-        MessagesRouteBuilder {
-            http: self.http,
-            auth: self.auth,
-            secrets,
-            cache: self.cache,
-        }
-    }
-
-    pub fn with_cache(self, cache: litellm_cache_response::ScopedCache) -> Self {
-        Self {
-            cache: Some(cache),
-            ..self
-        }
-    }
-}
-
-impl MessagesRouteBuilder<litellm_http::Client, Arc<AuthServices>, Arc<dyn SecretSource>> {
-    pub fn build(self) -> MessagesRoute {
-        MessagesRoute {
-            http: self.http,
-            auth: self.auth,
-            secrets: self.secrets,
-            cache: self.cache,
-        }
-    }
-}
-
 impl MessagesRoute {
-    pub fn builder() -> MessagesRouteBuilder {
-        MessagesRouteBuilder::default()
+    pub fn new(
+        http: litellm_http::Client,
+        auth: Arc<AuthServices>,
+        secrets: Arc<dyn SecretSource>,
+    ) -> Self {
+        Self {
+            http,
+            auth,
+            secrets,
+            cache: None,
+        }
     }
 
     #[must_use]
@@ -104,15 +51,9 @@ impl MessagesRoute {
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         options: impl Into<crate::CallOptions>,
     ) -> Result<MessagesCallResponse, Error> {
-        let crate::CallOptions {
-            cache: cache_options,
-            observers,
-        } = options.into();
-        litellm_host::lifecycle::observe_call(
-            observers.clone(),
-            self.run(call, cache_options, interceptors, observers.as_ref()),
-        )
-        .await
+        let context = CallContext::new(interceptors, options.into());
+        litellm_host::lifecycle::observe_call(context.observers.clone(), self.run(call, context))
+            .await
     }
 
     #[tracing::instrument(name = "litellm.route", skip_all, fields(
@@ -126,36 +67,34 @@ impl MessagesRoute {
     async fn run(
         &self,
         call: MessagesCall,
-        cache_options: Option<litellm_cache_response::CacheOptions>,
-        interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
-        observers: Option<&ObservationSender>,
+        context: CallContext<'_, impl Interceptors<Error>>,
     ) -> Result<MessagesCallResponse, Error> {
         crate::diagnostic::call(async {
-            self.run_provider(call, cache_options, interceptors, observers)
-                .await
+            let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
+            crate::diagnostic::provider(&prepared.body.model, prepared.provider.as_str());
+            let request = self.prepare_outbound(prepared, &context).boxed().await?;
+            let cache = CallCache::<route::Messages>::from_wire(
+                self.cache.as_ref().filter(|_| request.cacheable()),
+                context.cache,
+                &request.identity,
+                &request.wire,
+            );
+            let identity = request.identity.clone();
+            let (output, source) = match cache.lookup().await {
+                Some(hit) => hit,
+                None => (
+                    self.call_provider(request, &context).await?,
+                    ResultSource::Provider,
+                ),
+            };
+            context
+                .result_ready(ExecutionFacts {
+                    provider: identity,
+                    source: source.clone(),
+                })
+                .await?;
+            Ok(cache.finish(output, &source).await)
         })
         .await
     }
-
-    async fn run_provider(
-        &self,
-        call: MessagesCall,
-        cache_options: Option<litellm_cache_response::CacheOptions>,
-        interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<MessagesCallResponse, Error> {
-        let request = prepare::prepare(call, self.secrets.as_ref()).await?;
-        crate::diagnostic::provider(&request.body.model, request.provider.as_str());
-        let execute: futures_util::future::BoxFuture<'_, Result<MessagesCallResponse, Error>> =
-            Box::pin(handler::execute(
-                &self.http,
-                &self.auth,
-                request,
-                self.cache.clone(),
-                cache_options,
-                interceptors,
-                observers,
-            ));
-        execute.await
-    }
 }

litellm-rust/crates/core/src/messages/route.rs

@@ -43,8 +43,14 @@ impl super::MessagesRoute {
             request,
             observers,
             move |call, _, interceptors, observers| async move {
-                self.run(call, cache_options, &interceptors, observers.as_ref())
-                    .await
+                let context = crate::context::CallContext::new(
+                    &interceptors,
+                    crate::CallOptions {
+                        cache: cache_options,
+                        observers,
+                    },
+                );
+                self.run(call, context).await
             },
         )
     }

litellm-rust/crates/core/src/responses/handler.rs

@@ -16,7 +16,7 @@ pub(super) async fn execute(
     auth: &litellm_auth::AuthServices,
     request: ProviderResponsesRequest,
     cache: Option<litellm_cache_response::ScopedCache>,
-    cache_options: Option<litellm_cache_response::CacheOptions>,
+    cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ResponsesOutput, Error> {

litellm-rust/crates/core/src/responses/mod.rs

@@ -71,7 +71,7 @@ impl ResponsesRoute {
     async fn run(
         &self,
         call: ResponsesCall,
-        cache_options: Option<litellm_cache_response::CacheOptions>,
+        cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ResponsesOutput, Error> {
@@ -85,7 +85,7 @@ impl ResponsesRoute {
     async fn run_provider(
         &self,
         call: ResponsesCall,
-        cache_options: Option<litellm_cache_response::CacheOptions>,
+        cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ResponsesOutput, Error> {

litellm-rust/crates/core/tests/caching.rs

@@ -12,8 +12,8 @@ use bytes::Bytes;
 use futures_util::{StreamExt, TryStreamExt, stream};
 use litellm_cache_memory::InMemoryCache;
 use litellm_cache_response::{
-    CacheOptions, CacheScope, ResponseCache, ResponseCacheConfig, ResponseCacheService,
-    ResponseEnvelope,
+    CacheOptions, CachePolicy, CacheScope, ResponseCache, ResponseCacheConfig,
+    ResponseCacheService, ResponseEnvelope,
 };
 use litellm_core::{
     RouteError,
@@ -117,9 +117,9 @@ async fn call(
 
 #[rstest]
 #[case::normal(CacheOptions::new(CacheScope::Shared), true, true)]
-#[case::no_cache(CacheOptions { no_cache: true, ..CacheOptions::new(CacheScope::Shared) }, false, true)]
-#[case::no_store(CacheOptions { no_store: true, ..CacheOptions::new(CacheScope::Shared) }, true, false)]
-#[case::disabled(CacheOptions { caching: Some(false), ..CacheOptions::new(CacheScope::Shared) }, false, false)]
+#[case::no_cache(CacheOptions { policy: CachePolicy { no_cache: true, ..CachePolicy::default() }, ..CacheOptions::new(CacheScope::Shared) }, false, true)]
+#[case::no_store(CacheOptions { policy: CachePolicy { no_store: true, ..CachePolicy::default() }, ..CacheOptions::new(CacheScope::Shared) }, true, false)]
+#[case::disabled(CacheOptions { policy: CachePolicy { caching: Some(false), ..CachePolicy::default() }, ..CacheOptions::new(CacheScope::Shared) }, false, false)]
 #[tokio::test]
 async fn cache_controls_apply_to_both_reads_and_writes(
     cache: Arc<dyn ResponseCacheService>,
@@ -723,9 +723,9 @@ async fn unary_call(
 
 #[rstest]
 #[case::normal(CacheOptions::new(CacheScope::Shared), true, true)]
-#[case::no_cache(CacheOptions { no_cache: true, ..CacheOptions::new(CacheScope::Shared) }, false, true)]
-#[case::no_store(CacheOptions { no_store: true, ..CacheOptions::new(CacheScope::Shared) }, true, false)]
-#[case::disabled(CacheOptions { caching: Some(false), ..CacheOptions::new(CacheScope::Shared) }, false, false)]
+#[case::no_cache(CacheOptions { policy: CachePolicy { no_cache: true, ..CachePolicy::default() }, ..CacheOptions::new(CacheScope::Shared) }, false, true)]
+#[case::no_store(CacheOptions { policy: CachePolicy { no_store: true, ..CachePolicy::default() }, ..CacheOptions::new(CacheScope::Shared) }, true, false)]
+#[case::disabled(CacheOptions { policy: CachePolicy { caching: Some(false), ..CachePolicy::default() }, ..CacheOptions::new(CacheScope::Shared) }, false, false)]
 #[tokio::test]
 async fn unary_cache_controls_do_not_change_the_shared_service(
     cache: Arc<dyn ResponseCacheService>,

litellm-rust/crates/core/tests/messages/host.rs

@@ -1,9 +1,9 @@
 use litellm_host::lifecycle::ExecutionEvent;
 use std::sync::Mutex;
 
-use litellm_core::messages::route::Messages;
+use litellm_core::messages::{MessagesCallResponse, route::Messages};
 use litellm_host::{
-    interceptors::{RequestContext, WireRequest},
+    interceptors::{ExecutionFacts, RequestContext, ResultSource, WireRequest},
     lifecycle::CallEvent,
 };
 use litellm_llms::base_llm::messages::context::MessagesModelCapabilities as AnthropicModelCapabilities;
@@ -20,6 +20,8 @@ struct RecordingHost {
     rewrite: Rewrite,
     events: super::support::Observations,
     optional_params: Mutex<Vec<Value>>,
+    facts: Mutex<Vec<ExecutionFacts>>,
+    reject_result: bool,
 }
 
 impl RecordingHost {
@@ -29,6 +31,8 @@ impl RecordingHost {
             rewrite,
             events: super::support::Observations::default(),
             optional_params: Mutex::new(Vec::new()),
+            facts: Mutex::new(Vec::new()),
+            reject_result: false,
         }
     }
 
@@ -73,6 +77,14 @@ impl litellm_host::lifecycle::CallObserver for RecordingHost {
 impl litellm_host::interceptors::Interceptors<<Messages as litellm_host::protocol::Protocol>::Error>
     for RecordingHost
 {
+    async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
+        self.facts.lock().unwrap().push(facts);
+        if self.reject_result {
+            return Err(Error::Unsupported("result rejected"));
+        }
+        Ok(())
+    }
+
     async fn before_provider_request(
         &self,
         wire: WireRequest,
@@ -98,6 +110,91 @@ impl litellm_host::interceptors::Interceptors<<Messages as litellm_host::protoco
     }
 }
 
+#[rstest]
+#[case::native_unary(false, false)]
+#[case::native_stream(true, false)]
+#[case::hosted_unary(false, true)]
+#[case::hosted_stream(true, true)]
+#[tokio::test]
+async fn rejected_results_are_not_delivered_or_cached(
+    call: MessagesCall,
+    #[case] streaming: bool,
+    #[case] hosted: bool,
+) {
+    use futures_util::TryStreamExt;
+    use litellm_cache_memory::InMemoryCache;
+    use litellm_cache_response::{CacheScope, ResponseCache, ScopedCache};
+
+    let response = if streaming {
+        ResponseTemplate::new(200).set_body_raw(
+            "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n",
+            "text/event-stream",
+        )
+    } else {
+        message_response()
+    };
+    let upstream = upstream([response.clone(), response]).await;
+    let route = messages_route(no_secrets()).with_cache(ScopedCache::new(
+        Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new(
+            Some(100),
+            Some(Duration::from_secs(60)),
+        )))),
+        CacheScope::Shared,
+    ));
+    for (reject, expected_requests, cached) in [
+        (true, 1, false),
+        (false, 2, false),
+        (true, 2, true),
+        (false, 2, true),
+    ] {
+        let request = authenticated(
+            with_fields(
+                MessagesCall {
+                    body: call.body.clone(),
+                    ..super::call()
+                },
+                json!({"stream": streaming}),
+            ),
+            upstream.uri(),
+        );
+        let host = RecordingHost {
+            reject_result: reject,
+            ..RecordingHost::passthrough(request)
+        };
+        let result = if hosted {
+            litellm_host_native::in_process::run_hosted(
+                route.clone().machine(host.request().unwrap(), None),
+                host.runtime(),
+            )
+            .await
+            .map(|_| ())
+        } else {
+            match route.execute(host.request().unwrap(), &host, None).await {
+                Ok(MessagesCallResponse::Complete(_)) => Ok(()),
+                Ok(MessagesCallResponse::Stream { chunks, .. }) => {
+                    chunks.try_collect::<Vec<_>>().await.map(|_| ())
+                }
+                Err(error) => Err(error),
+            }
+        };
+        assert_eq!(
+            result,
+            if reject {
+                Err(Error::Unsupported("result rejected"))
+            } else {
+                Ok(())
+            }
+        );
+        assert_eq!(received(&upstream).await.len(), expected_requests);
+        let facts = host.facts.lock().unwrap();
+        assert_eq!(facts.len(), 1);
+        assert_eq!(
+            matches!(facts[0].source, ResultSource::Cache { .. }),
+            cached
+        );
+    }
+}
+
 async fn run_through(host: &RecordingHost) -> Result<MessagesOutput, Error> {
     litellm_host_native::in_process::run_hosted(
         machine(Arc::new(RecordingSecrets::empty()))(host.request()?),
@@ -143,6 +240,81 @@ async fn what_before_send_returns_is_what_the_provider_receives(call: MessagesCa
     assert_eq!(request.header("x-api-key"), Some("sk-ant"));
 }
 
+#[rstest]
+#[case::enable(false, json!(true), Some(true))]
+#[case::disable(true, json!(false), Some(false))]
+#[case::null(true, Value::Null, Some(false))]
+#[case::invalid(false, json!("true"), None)]
+#[tokio::test]
+async fn response_mode_follows_the_intercepted_request(
+    call: MessagesCall,
+    traces: TraceCapture,
+    #[case] original_stream: bool,
+    #[case] rewritten_stream: Value,
+    #[case] expected_stream: Option<bool>,
+) {
+    use futures_util::TryStreamExt;
+
+    let sse = "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\n";
+    let response = if expected_stream == Some(true) {
+        ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream")
+    } else {
+        message_response()
+    };
+    let upstream = upstream([response]).await;
+    let rewrite = rewritten_stream.clone();
+    let host = RecordingHost::new(
+        authenticated(
+            with_fields(call, json!({"stream": original_stream})),
+            upstream.uri(),
+        ),
+        Box::new(move |wire| {
+            let mut body = wire.body;
+            body["stream"] = rewrite.clone();
+            Ok(WireRequest { body, ..wire })
+        }),
+    );
+    let result = traces
+        .logger()
+        .instrument(async {
+            let output = messages_route(no_secrets())
+                .execute(host.request()?, &host, None)
+                .await?;
+            match output {
+                MessagesCallResponse::Stream { chunks, .. } => {
+                    assert_eq!(expected_stream, Some(true));
+                    assert_eq!(
+                        chunks.try_collect::<Vec<_>>().await?.concat(),
+                        sse.as_bytes()
+                    );
+                }
+                MessagesCallResponse::Complete(message) => {
+                    assert_eq!(expected_stream, Some(false));
+                    assert_eq!(*message, serde_json::from_value(message_body()).unwrap());
+                }
+            }
+            Ok::<_, Error>(())
+        })
+        .await;
+
+    let summaries = traces.summaries("litellm.route");
+    assert_eq!(summaries.len(), 1);
+    let Some(expected_stream) = expected_stream else {
+        assert!(matches!(result, Err(Error::InvalidRequest(_))));
+        assert!(received(&upstream).await.is_empty());
+        assert_eq!(summaries[0]["outcome"], "failure");
+        return;
+    };
+    result.unwrap();
+    assert_eq!(
+        only_request(&upstream).await.json()["stream"],
+        rewritten_stream
+    );
+    assert_eq!(host.raw_responses().len(), usize::from(!expected_stream));
+    assert_eq!(summaries[0]["stream"], expected_stream);
+    assert_eq!(summaries[0]["outcome"], "success");
+}
+
 #[rstest]
 #[tokio::test]
 async fn a_before_send_failure_never_sends(call: MessagesCall) {

litellm-rust/crates/core/tests/messages/response.rs

@@ -269,25 +269,22 @@ async fn the_facade_sends_through_the_injected_http_pool_configuration(call: Mes
     };
 
     let resources = support::resources();
-    let response = litellm_core::messages::MessagesRoute::builder()
-        .with_http(provider_http(
-            &resources,
-            &Resolution::from(&settings).config,
-        ))
-        .with_auth(resources.auth)
-        .with_secrets(no_secrets())
-        .build()
-        .execute(
-            MessagesCall {
-                api_key: Some("sk-ant".into()),
-                api_base: Some(base),
-                ..call
-            },
-            &(),
-            None,
-        )
-        .await
-        .expect("messages request succeeds");
+    let response = litellm_core::messages::MessagesRoute::new(
+        provider_http(&resources, &Resolution::from(&settings).config),
+        resources.auth,
+        no_secrets(),
+    )
+    .execute(
+        MessagesCall {
+            api_key: Some("sk-ant".into()),
+            api_base: Some(base),
+            ..call
+        },
+        &(),
+        None,
+    )
+    .await
+    .expect("messages request succeeds");
 
     let MessagesCallResponse::Complete(message) = response else {
         panic!("a non-streaming request returns a message");
@@ -345,7 +342,7 @@ async fn message_route_summary_excludes_payload_diagnostics(
 #[case::uncached(false, 2)]
 #[case::cached(true, 1)]
 #[tokio::test]
-async fn builder_preserves_dependencies_and_optional_cache(
+async fn route_uses_injected_dependencies_and_optional_cache(
     #[case] caching: bool,
     #[case] expected_requests: usize,
 ) {
@@ -355,9 +352,13 @@ async fn builder_preserves_dependencies_and_optional_cache(
 
     let upstream = upstream([message_response(), message_response()]).await;
     let resources = resources();
-    let builder = MessagesRoute::builder();
-    let builder = if caching {
-        builder.with_cache(ScopedCache::new(
+    let route = MessagesRoute::new(
+        provider_http(&resources, &http_config()),
+        resources.auth.clone(),
+        Arc::new(RecordingSecrets::new([("ANTHROPIC_API_KEY", "route-key")])),
+    );
+    let route = if caching {
+        route.with_cache(ScopedCache::new(
             Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new(
                 Some(100),
                 Some(Duration::from_secs(60)),
@@ -365,16 +366,8 @@ async fn builder_preserves_dependencies_and_optional_cache(
             CacheScope::Shared,
         ))
     } else {
-        builder
+        route
     };
-    let route = builder
-        .with_secrets(Arc::new(RecordingSecrets::new([(
-            "ANTHROPIC_API_KEY",
-            "builder-key",
-        )])))
-        .with_auth(resources.auth.clone())
-        .with_http(provider_http(&resources, &http_config()))
-        .build();
     for _ in 0..2 {
         let request = MessagesCall {
             api_base: Some(upstream.uri()),
@@ -392,5 +385,72 @@ async fn builder_preserves_dependencies_and_optional_cache(
     }
     let requests = received(&upstream).await;
     assert_eq!(requests.len(), expected_requests);
-    assert_eq!(requests[0].header("x-api-key"), Some("builder-key"));
+    assert_eq!(requests[0].header("x-api-key"), Some("route-key"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn cache_overrides_preserve_the_routes_isolated_scope(call: MessagesCall) {
+    use litellm_cache_memory::InMemoryCache;
+    use litellm_cache_response::{CachePolicy, CacheScope, ResponseCache, ScopedCache};
+
+    let first_body = message_body();
+    let second_body = Value::Object(
+        first_body
+            .as_object()
+            .unwrap()
+            .iter()
+            .map(|(key, value)| {
+                (
+                    key.clone(),
+                    if key == "id" {
+                        json!("msg_second")
+                    } else {
+                        value.clone()
+                    },
+                )
+            })
+            .collect(),
+    );
+    let upstream = upstream([
+        json_response(first_body.clone()),
+        json_response(second_body.clone()),
+    ])
+    .await;
+    let service = Arc::new(ResponseCache::new(Arc::new(InMemoryCache::new(
+        Some(100),
+        Some(Duration::from_secs(60)),
+    ))));
+    let first = messages_route(no_secrets()).with_cache(ScopedCache::new(
+        service.clone(),
+        CacheScope::Isolated("first".into()),
+    ));
+    let second = messages_route(no_secrets()).with_cache(ScopedCache::new(
+        service,
+        CacheScope::Isolated("second".into()),
+    ));
+    for (route, expected) in [
+        (&first, &first_body),
+        (&second, &second_body),
+        (&first, &first_body),
+        (&second, &second_body),
+    ] {
+        let request = MessagesCall {
+            body: call.body.clone(),
+            api_key: Some("same-key".into()),
+            api_base: Some(upstream.uri()),
+            ..super::call()
+        };
+        let override_options = CachePolicy {
+            ttl: Some(Duration::from_secs(30)),
+            ..CachePolicy::default()
+        };
+        let MessagesCallResponse::Complete(response) =
+            route.execute(request, &(), override_options).await.unwrap()
+        else {
+            panic!("expected a completed message");
+        };
+        assert_eq!(response.id, expected["id"].as_str().unwrap());
+    }
+    assert_eq!(received(&upstream).await.len(), 2);
 }

litellm-rust/crates/core/tests/support/mod.rs

@@ -44,11 +44,11 @@ pub fn provider_http(
 
 pub fn messages_route(secrets: Arc<dyn SecretSource>) -> litellm_core::messages::MessagesRoute {
     let resources = resources();
-    litellm_core::messages::MessagesRoute::builder()
-        .with_http(provider_http(&resources, &http_config()))
-        .with_auth(resources.auth)
-        .with_secrets(secrets)
-        .build()
+    litellm_core::messages::MessagesRoute::new(
+        provider_http(&resources, &http_config()),
+        resources.auth,
+        secrets,
+    )
 }
 
 pub fn chat_completions_route() -> litellm_core::chat_completions::ChatCompletionsRoute {

litellm-rust/crates/cost/tests/calculation.rs

@@ -203,6 +203,85 @@ fn threshold_tiers_and_boundaries() {
     assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
 }
 
+#[rstest]
+#[case::ultrafast_above_threshold(ServiceTier::Ultrafast, 300_000, 9_301_000.0, 37_000.0)]
+#[case::ultrafast_at_threshold(ServiceTier::Ultrafast, 272_000, 544_500.0, 5_000.0)]
+#[case::standard_above_threshold(ServiceTier::Standard, 300_000, 3_300_600.0, 13_000.0)]
+#[case::priority_above_threshold(ServiceTier::Priority, 300_000, 5_701_000.0, 23_000.0)]
+fn tiered_long_context_rates_are_selected_by_service_tier(
+    #[case] service_tier: ServiceTier,
+    #[case] prompt_tokens: u64,
+    #[case] expected_input: f64,
+    #[case] expected_output: f64,
+) {
+    let standard = Rates {
+        cache_read: Rate::Value(3.0),
+        ..rates(Rate::Value(1.0), Rate::Value(2.0))
+    };
+    let tiers = [
+        TierRates {
+            tier: ServiceTier::Priority,
+            rates: Rates {
+                cache_read: Rate::Value(5.0),
+                ..rates(Rate::Value(3.0), Rate::Value(4.0))
+            },
+        },
+        TierRates {
+            tier: ServiceTier::Ultrafast,
+            rates: Rates {
+                cache_read: Rate::Value(7.0),
+                ..rates(Rate::Value(2.0), Rate::Value(5.0))
+            },
+        },
+    ];
+    let threshold_tiers = [
+        TierRates {
+            tier: ServiceTier::Priority,
+            rates: Rates {
+                cache_read: Rate::Value(29.0),
+                ..rates(Rate::Value(19.0), Rate::Value(23.0))
+            },
+        },
+        TierRates {
+            tier: ServiceTier::Ultrafast,
+            rates: Rates {
+                cache_read: Rate::Value(41.0),
+                ..rates(Rate::Value(31.0), Rate::Value(37.0))
+            },
+        },
+    ];
+    let thresholds = [ThresholdRates {
+        above_prompt_tokens: 272_000,
+        standard: Rates {
+            cache_read: Rate::Value(17.0),
+            ..rates(Rate::Value(11.0), Rate::Value(13.0))
+        },
+        tiers: &threshold_tiers,
+    }];
+    let pricing = Pricing {
+        standard,
+        tiers: &tiers,
+        thresholds: &thresholds,
+        off_peak: None,
+    };
+    let base = request();
+    let long_context_request = Request {
+        usage: Usage {
+            prompt_tokens,
+            completion_tokens: 1_000,
+            cache_read_tokens: 100,
+            cache_write_tokens: 0,
+            ..base.usage
+        },
+        service_tier,
+        ..base
+    };
+    let cost = calculate(&pricing, &long_context_request).unwrap();
+
+    assert_eq!(cost.input(), expected_input);
+    assert_eq!(cost.output(), expected_output);
+}
+
 #[test]
 fn compile_rejects_ambiguous_rates() {
     let duplicate = ThresholdRates {

litellm-rust/crates/gateway-inference/src/caching.rs

@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use litellm_cache_response::{CacheOptions, CacheScope};
+use litellm_cache_response::{CacheOptions, CachePolicy, CacheScope};
 use litellm_gateway_auth::AuthenticatedRequest;
 use serde::Deserialize;
 use serde_json::{Map, Value};
@@ -38,11 +38,13 @@ pub(crate) fn prepare(
         .map_err(|error| Error::InvalidBody(error.to_string()))?;
     let caller = identity.caller();
     let options = CacheOptions {
-        caching,
-        no_cache: controls.no_cache,
-        no_store: controls.no_store,
-        ttl: controls.ttl.map(duration).transpose()?,
-        max_age: controls.max_age.map(duration).transpose()?,
+        policy: CachePolicy {
+            caching,
+            no_cache: controls.no_cache,
+            no_store: controls.no_store,
+            ttl: controls.ttl.map(duration).transpose()?,
+            max_age: controls.max_age.map(duration).transpose()?,
+        },
         scope: CacheScope::Isolated(
             serde_json::json!([
                 caller.principal().authority(),

litellm-rust/crates/gateway-inference/src/chat_completions.rs

@@ -69,7 +69,7 @@ async fn handle(
                 extra_headers: None,
                 timeout: deployment.timeout,
             },
-            cache_options,
+            cache_options.policy,
         ),
         (),
         headers.clone(),

litellm-rust/crates/gateway-inference/src/lib.rs

@@ -68,11 +68,7 @@ impl Gateway {
                 auth.clone(),
                 secrets.clone(),
             ),
-            messages: MessagesRoute::builder()
-                .with_http(provider.clone())
-                .with_auth(auth.clone())
-                .with_secrets(secrets.clone())
-                .build(),
+            messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone()),
             responses: ResponsesRoute::new(provider, auth.clone(), secrets.clone()),
             ocr: OcrRoute::new(OcrClient::new(
                 &resources.pool,

litellm-rust/crates/gateway-inference/src/messages.rs

@@ -54,7 +54,7 @@ async fn handle(
     };
 
     let call = project(deployment, body, headers)?;
-    let machine = route.machine(call, cache_options);
+    let machine = route.machine(call, cache_options.policy);
     let stream =
         Sse::<Messages, _, _>::new(Json, |error| Bytes::from(Error::from(error).sse_frame()));
     let headers = crate::caching::CacheHeaders::default();

litellm-rust/crates/gateway-inference/src/responses.rs

@@ -38,7 +38,7 @@ pub(crate) async fn create(
         extra_headers: None,
         timeout: deployment.timeout,
     };
-    let machine = route.machine(call, cache_options);
+    let machine = route.machine(call, cache_options.policy);
     let stream = Sse::<Responses, _, _>::new(Json, |error| {
         let error = Error::from(error);
         Bytes::from(format!(

litellm-rust/crates/gateway-ui/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-axum = { workspace = true, features = ["json", "original-uri"] }
+axum = { workspace = true, features = ["json", "original-uri", "query"] }
 axum-login.workspace = true
 base64.workspace = true
 governor = { version = "0.10.4", default-features = false, features = ["std"] }
@@ -16,6 +16,7 @@ rand.workspace = true
 serde.workspace = true
 thiserror.workspace = true
 time.workspace = true
+tower = { version = "0.5", features = ["util"] }
 tower-cookies = "0.11.0"
 tower-http = { version = "0.6.11", features = ["fs", "set-header"] }
 tower-sessions.workspace = true

litellm-rust/crates/gateway-ui/src/dashboard.rs

@@ -1,7 +1,12 @@
 use std::path::Path;
 
-use axum::{Router, routing::get};
-use serde::Serialize;
+use axum::{
+    Router,
+    extract::{Query, Request},
+    routing::get,
+};
+use serde::{Deserialize, Serialize};
+use tower::ServiceExt;
 use tower_http::services::{ServeDir, ServeFile};
 
 #[derive(Serialize)]
@@ -9,9 +14,39 @@ struct Logo {
     logo_url: &'static str,
 }
 
+#[derive(Clone, Copy, Deserialize)]
+#[serde(rename_all = "lowercase")]
+enum Theme {
+    Light,
+    Dark,
+}
+
+#[derive(Clone, Copy, Deserialize)]
+#[serde(rename_all = "lowercase")]
+enum Variant {
+    Full,
+    Monogram,
+}
+
+#[derive(Deserialize)]
+struct LogoQuery {
+    theme: Option<Theme>,
+    variant: Option<Variant>,
+}
+
+fn logo_file(query: &LogoQuery) -> &'static str {
+    match (query.variant, query.theme) {
+        (Some(Variant::Monogram), Some(Theme::Dark)) => "assets/logos/litellm_monogram_dark.svg",
+        (Some(Variant::Monogram), _) => "assets/logos/litellm_monogram.svg",
+        (_, Some(Theme::Dark)) => "assets/logos/litellm_logo_dark.png",
+        _ => "assets/logos/litellm_logo.png",
+    }
+}
+
 pub fn dashboard_assets(directory: impl AsRef<Path>) -> Router {
     let directory = directory.as_ref();
     let assets = ServeDir::new(directory.join("_next")).append_index_html_on_directories(false);
+    let logos = directory.to_path_buf();
 
     crate::static_assets(directory)
         .route(
@@ -22,9 +57,11 @@ pub fn dashboard_assets(directory: impl AsRef<Path>) -> Router {
                 })
             }),
         )
-        .route_service(
+        .route(
             "/get_image",
-            ServeFile::new(directory.join("assets/logos/litellm_logo.jpg")),
+            get(move |Query(query): Query<LogoQuery>, request: Request| {
+                ServeFile::new(logos.join(logo_file(&query))).oneshot(request)
+            }),
         )
         .route_service(
             "/get_favicon",

litellm-rust/crates/gateway-ui/tests/assets.rs

@@ -40,7 +40,14 @@ fn dashboard(directory: TempDir) -> App {
     let export = directory.path().join("public");
     std::fs::create_dir_all(export.join("_next/static")).unwrap();
     std::fs::create_dir_all(export.join("assets/logos")).unwrap();
-    std::fs::write(export.join("assets/logos/litellm_logo.jpg"), "logo bytes").unwrap();
+    for (file, bytes) in [
+        ("litellm_logo.png", "logo bytes"),
+        ("litellm_logo_dark.png", "dark logo bytes"),
+        ("litellm_monogram.svg", "monogram bytes"),
+        ("litellm_monogram_dark.svg", "dark monogram bytes"),
+    ] {
+        std::fs::write(export.join("assets/logos").join(file), bytes).unwrap();
+    }
     std::fs::write(export.join("favicon.ico"), "icon bytes").unwrap();
     std::fs::write(export.join("_next/static/app.js"), "window.app = true;").unwrap();
     App {
@@ -130,7 +137,15 @@ async fn missing_paths_never_fall_back_to_dashboard(app: App, #[case] path: &str
 )]
 #[case::root_assets("/_next/static/app.js", "window.app = true;", "text/javascript")]
 #[case::nested_assets("/ui/_next/static/app.js", "window.app = true;", "text/javascript")]
-#[case::logo("/get_image", "logo bytes", "image/jpeg")]
+#[case::logo("/get_image", "logo bytes", "image/png")]
+#[case::logo_light("/get_image?theme=light", "logo bytes", "image/png")]
+#[case::logo_dark("/get_image?theme=dark", "dark logo bytes", "image/png")]
+#[case::monogram("/get_image?variant=monogram", "monogram bytes", "image/svg+xml")]
+#[case::monogram_dark(
+    "/get_image?theme=dark&variant=monogram",
+    "dark monogram bytes",
+    "image/svg+xml"
+)]
 #[case::favicon("/get_favicon", "icon bytes", "image/x-icon")]
 #[tokio::test]
 async fn dashboard_adapter_preserves_existing_urls(
@@ -184,3 +199,19 @@ async fn logo_discovery_points_to_served_image(dashboard: App) {
         "logo bytes"
     );
 }
+
+#[rstest]
+#[case::logo("/get_image")]
+#[case::logo_dark("/get_image?theme=dark")]
+#[case::monogram("/get_image?variant=monogram")]
+#[case::monogram_dark("/get_image?theme=dark&variant=monogram")]
+#[tokio::test]
+async fn committed_dashboard_export_serves_every_logo(#[case] path: &str) {
+    let export = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../../litellm/proxy/_experimental/out");
+    let response = litellm_gateway_ui::dashboard_assets(export)
+        .oneshot(Request::get(path).body(Body::empty()).unwrap())
+        .await
+        .unwrap();
+    assert_eq!(response.status(), StatusCode::OK);
+}

litellm-rust/crates/model-catalog/src/model_info.rs

@@ -57,6 +57,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_creation_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_creation_input_token_cost_batches: Option<f64>,
     /// Flex service-tier rate for the same-named base field.
@@ -65,6 +68,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_creation_input_token_cost_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_creation_input_token_cost_ultrafast: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_read_input_audio_token_cost: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -101,6 +107,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_read_input_token_cost_above_272k_tokens_ultrafast: Option<f64>,
     /// Rate applied once the prompt exceeds the token threshold in the field name.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
@@ -115,6 +124,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub cache_read_input_token_cost_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cache_read_input_token_cost_ultrafast: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub citation_cost_per_token: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -125,6 +137,8 @@ pub struct ModelInfo {
     pub computer_use_input_cost_per_1k_tokens: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub computer_use_output_cost_per_1k_tokens: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub cost_per_second: Option<f64>,
     /// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub default_reasoning_effort: Option<ReasoningEffort>,
@@ -211,6 +225,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub input_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
     /// Rate applied once the prompt exceeds the token threshold in the field name.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_above_512k_tokens: Option<f64>,
@@ -228,6 +245,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_token_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub input_cost_per_token_ultrafast: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub input_cost_per_video_per_second: Option<f64>,
     /// Rate applied once the prompt exceeds the token threshold in the field name.
@@ -360,6 +380,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_cost_per_token_above_272k_tokens_ultrafast: Option<f64>,
     /// Rate applied once the prompt exceeds the token threshold in the field name.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_token_above_512k_tokens: Option<f64>,
@@ -375,6 +398,9 @@ pub struct ModelInfo {
     /// Priority service-tier rate for the same-named base field.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_token_priority: Option<f64>,
+    /// Ultrafast service-tier rate for the same-named base field.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub output_cost_per_token_ultrafast: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub output_cost_per_video_per_second: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]

litellm-rust/crates/python-bridge/Cargo.toml

@@ -21,6 +21,7 @@ tiktoken = ["litellm-token-counter/tiktoken"]
 [dependencies]
 fancy-regex.workspace = true
 litellm-tracing.workspace = true
+litellm-traces.workspace = true
 litellm-host.workspace = true
 bytes.workspace = true
 futures-util.workspace = true

litellm-rust/crates/python-bridge/src/cache/native/activation.rs

@@ -1,5 +1,5 @@
 use crate::cache::cache_error;
-use crate::logger::run_sync_value;
+use crate::execution::run_sync_value;
 use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
 use litellm_cache_redis_semantic::RedisSemanticConfig;
 use litellm_host_python::release_gil;

litellm-rust/crates/python-bridge/src/cache/native/backend.rs

@@ -470,7 +470,7 @@ impl NativeResponseCache {
         match self {
             Self::Exact(_) | Self::QdrantSemantic(_) => {
                 let service = self.clone();
-                crate::logger::run_async(
+                crate::execution::run_async(
                     py,
                     async move {
                         service
@@ -495,7 +495,7 @@ impl NativeResponseCache {
         match self {
             Self::Exact(_) | Self::QdrantSemantic(_) => {
                 let service = self.clone();
-                crate::logger::run_async(
+                crate::execution::run_async(
                     py,
                     async move { service.async_lookup(&request, now()).await },
                     cache_error,
@@ -550,7 +550,7 @@ impl NativeResponseCache {
         match self {
             Self::Exact(_) | Self::QdrantSemantic(_) => {
                 let service = self.clone();
-                crate::logger::run_async(
+                crate::execution::run_async(
                     py,
                     async move { service.async_store(&request, response, now()).await },
                     cache_error,
@@ -619,7 +619,7 @@ impl NativeResponseCache {
         match self {
             Self::Exact(_) | Self::QdrantSemantic(_) => {
                 let service = self.clone();
-                crate::logger::run_async(
+                crate::execution::run_async(
                     py,
                     async move { service.async_store_batch(entries, now()).await },
                     cache_error,

litellm-rust/crates/python-bridge/src/cache/native/semantic.rs

@@ -1,5 +1,5 @@
 use crate::cache::cache_error;
-use crate::logger::run_async;
+use crate::execution::run_async;
 use std::{collections::VecDeque, time::Duration};
 
 use litellm_cache::Error;

litellm-rust/crates/python-bridge/src/cache/native/v2.rs

@@ -144,7 +144,7 @@ impl NativeCacheHandle {
         self.check_process()?;
         let request = request(key, None)?;
         let backend = self.backend.clone();
-        crate::logger::run_async(
+        crate::execution::run_async(
             py,
             async move { backend.async_lookup(&request, super::request::now()).await },
             cache_error,
@@ -163,7 +163,7 @@ impl NativeCacheHandle {
         let request = request(key, ttl)?;
         let value: Value = from_py(value)?;
         let backend = self.backend.clone();
-        crate::logger::run_async(
+        crate::execution::run_async(
             py,
             async move {
                 backend
@@ -188,7 +188,7 @@ impl NativeCacheHandle {
             .map(|(key, value)| Ok((request(key, ttl)?, value)))
             .collect::<PyResult<Vec<_>>>()?;
         let backend = self.backend.clone();
-        crate::logger::run_async(
+        crate::execution::run_async(
             py,
             async move {
                 backend
@@ -202,19 +202,19 @@ impl NativeCacheHandle {
     fn flush(&self, py: Python<'_>) -> PyResult<Py<PyAny>> {
         self.check_process()?;
         let backend = self.backend.clone();
-        crate::logger::run_sync(py, async move { backend.async_flush().await }, cache_error)
+        crate::execution::run_sync(py, async move { backend.async_flush().await }, cache_error)
     }
 
     fn async_flush<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         self.check_process()?;
         let backend = self.backend.clone();
-        crate::logger::run_async(py, async move { backend.async_flush().await }, cache_error)
+        crate::execution::run_async(py, async move { backend.async_flush().await }, cache_error)
     }
 
     fn ping<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         self.check_process()?;
         let storage = self.storage.clone();
-        crate::logger::run_async(
+        crate::execution::run_async(
             py,
             async move {
                 match storage {
@@ -229,7 +229,7 @@ impl NativeCacheHandle {
     fn disconnect<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         self.check_process()?;
         let storage = self.storage.clone();
-        crate::logger::run_async(
+        crate::execution::run_async(
             py,
             async move {
                 match storage {
@@ -244,7 +244,7 @@ impl NativeCacheHandle {
     fn delete<'py>(&self, py: Python<'py>, keys: Vec<String>) -> PyResult<Bound<'py, PyAny>> {
         self.check_process()?;
         let storage = self.storage.clone();
-        crate::logger::run_async(
+        crate::execution::run_async(
             py,
             async move {
                 for key in keys {
@@ -330,15 +330,17 @@ pub(in crate::cache) fn configured(
     Ok((
         Some(cache.service.clone()),
         litellm_cache_response::CacheOptions {
-            caching: kwargs
-                .get_item("caching")?
-                .filter(|value| !value.is_none())
-                .map(|value| value.extract())
-                .transpose()?,
-            no_cache: boolean("no-cache")?,
-            no_store: boolean("no-store")?,
-            ttl: seconds("ttl")?,
-            max_age: seconds("s-max-age")?.or(seconds("s-maxage")?),
+            policy: litellm_cache_response::CachePolicy {
+                caching: kwargs
+                    .get_item("caching")?
+                    .filter(|value| !value.is_none())
+                    .map(|value| value.extract())
+                    .transpose()?,
+                no_cache: boolean("no-cache")?,
+                no_store: boolean("no-store")?,
+                ttl: seconds("ttl")?,
+                max_age: seconds("s-max-age")?.or(seconds("s-maxage")?),
+            },
             scope: litellm_cache_response::CacheScope::Shared,
         },
     ))

litellm-rust/crates/python-bridge/src/cache/runtime.rs

@@ -1,4 +1,4 @@
-use crate::logger::run_async;
+use crate::execution::run_async;
 use litellm_cache_response::PartialHits;
 use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py};
 use pyo3::{

litellm-rust/crates/python-bridge/src/cache/selection.rs

@@ -1,5 +1,7 @@
 use super::{native, python};
-use litellm_cache_response::{CacheOptions, CacheScope, ResponseCacheService, ScopedCache};
+use litellm_cache_response::{
+    CacheOptions, CachePolicy, CacheScope, ResponseCacheService, ScopedCache,
+};
 use litellm_host::{
     machine::{HostServices, MachineFault},
     protocol::Protocol,
@@ -154,8 +156,11 @@ pub(crate) fn configure(
             .map(|value| value.unwrap_or(false))
     };
     let options = CacheOptions {
-        no_cache: boolean("no-cache")?,
-        no_store: boolean("no-store")?,
+        policy: CachePolicy {
+            no_cache: boolean("no-cache")?,
+            no_store: boolean("no-store")?,
+            ..CachePolicy::default()
+        },
         ..CacheOptions::new(CacheScope::Shared)
     };
     let namespace = cache

litellm-rust/crates/python-bridge/src/execution.rs

@@ -13,7 +13,7 @@ where
     E: Send + 'static,
     F: Future<Output = Result<T, E>> + Send + 'static,
 {
-    litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error)
+    litellm_host_python::run_sync(py, crate::logger::capture(py).instrument(future), map_error)
 }
 
 pub(crate) fn run_async<T, E, F>(
@@ -26,7 +26,7 @@ where
     E: Send + 'static,
     F: Future<Output = Result<T, E>> + Send + 'static,
 {
-    litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error)
+    litellm_host_python::run_async(py, crate::logger::capture(py).instrument(future), map_error)
 }
 
 pub(crate) fn run_sync_value<T, F>(py: Python<'_>, future: F) -> PyResult<T>
@@ -34,7 +34,7 @@ where
     T: Send + 'static,
     F: Future<Output = PyResult<T>> + Send + 'static,
 {
-    litellm_host_python::run_sync_value(py, super::capture(py).instrument(future))
+    litellm_host_python::run_sync_value(py, crate::logger::capture(py).instrument(future))
 }
 
 pub(crate) fn run_async_value<T, F>(py: Python<'_>, future: F) -> PyResult<Bound<'_, PyAny>>
@@ -42,5 +42,5 @@ where
     T: for<'py> IntoPyObject<'py> + Send + 'static,
     F: Future<Output = PyResult<T>> + Send + 'static,
 {
-    litellm_host_python::run_async_value(py, super::capture(py).instrument(future))
+    litellm_host_python::run_async_value(py, crate::logger::capture(py).instrument(future))
 }

litellm-rust/crates/python-bridge/src/lib.rs

@@ -4,6 +4,7 @@ mod coercion;
 mod credentials;
 mod diagnostics;
 mod errors;
+mod execution;
 mod http;
 mod lifecycle;
 mod logger;
@@ -42,6 +43,8 @@ mod _native {
     use crate::routes::responses::{ResponsesWebSocketConnection, aresponses, responses};
     #[pymodule_export]
     use crate::routes::token_counter::TokenCounter;
+    #[pymodule_export]
+    use crate::routes::traces::{NativeTraceStorage, trace_decode_otlp};
     #[cfg(feature = "huggingface")]
     #[pymodule_export]
     use crate::tokenizer::HuggingFaceEncoding;
@@ -106,6 +109,8 @@ mod tests {
                 "aresponses",
                 "ResponsesWebSocketConnection",
                 "NativeDiagnosticProcessor",
+                "NativeTraceStorage",
+                "trace_decode_otlp",
                 "TokenCounter",
                 "Tokenizer",
                 "gil_stats",

litellm-rust/crates/python-bridge/src/logger/mod.rs

@@ -1,7 +1,5 @@
-mod execution;
 mod machine;
 
-pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value};
 pub(crate) use machine::LoggedMachine;
 
 use litellm_host_python::Pythonized;

litellm-rust/crates/python-bridge/src/logger/tests.rs

@@ -76,7 +76,7 @@ async fn traced_operation(_secret: &str) -> PyResult<()> {
 
 #[pyfunction]
 fn span_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-    super::run_async_value(py, traced_operation("private-key-sentinel"))
+    crate::execution::run_async_value(py, traced_operation("private-key-sentinel"))
 }
 
 #[pyfunction]
@@ -93,7 +93,7 @@ fn levels(py: Python<'_>) {
 
 #[pyfunction]
 fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
-    super::run_async_value(py, async {
+    crate::execution::run_async_value(py, async {
         tokio::task::yield_now().await;
         litellm_tracing::warn!("async warning");
         Ok(())
@@ -102,7 +102,7 @@ fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
 
 #[pyfunction]
 fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
-    super::run_sync_value(py, async {
+    crate::execution::run_sync_value(py, async {
         tokio::task::yield_now().await;
         litellm_tracing::warn!("sync warning");
         Ok(())
@@ -111,7 +111,7 @@ fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
 
 #[pyfunction]
 fn synchronous_failure(py: Python<'_>) -> PyResult<()> {
-    super::run_sync_value(py, async {
+    crate::execution::run_sync_value(py, async {
         litellm_tracing::warn!("failure diagnostic");
         Err(pyo3::exceptions::PyValueError::new_err("request failed"))
     })

litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs

@@ -1,4 +1,4 @@
-use crate::logger::{run_async, run_sync};
+use crate::execution::{run_async, run_sync};
 use litellm_core::audio_transcription::{
     AudioTranscriptionRoute, Error, types::AudioTranscriptionRequest,
 };

litellm-rust/crates/python-bridge/src/routes/chat_completions.rs

@@ -2,7 +2,7 @@ mod host;
 
 use pyo3::types::{PyDict, PyTuple};
 
-use crate::logger::{run_async, run_sync};
+use crate::execution::{run_async, run_sync};
 use litellm_core::chat_completions::{ChatCompletionsRoute, Error, types::ChatCompletionsRequest};
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
 use pyo3::prelude::*;
@@ -175,7 +175,7 @@ fn run_public(
                 )),
                 None => route,
             };
-            Ok(route.machine(request, cache_options))
+            Ok(route.machine(request, cache_options.policy))
         },
         host::ChatCompletionsPythonHost(host),
         hooks,

litellm-rust/crates/python-bridge/src/routes/messages/mod.rs

@@ -26,14 +26,12 @@ fn run_messages(
         py,
         arguments,
         move |py, arguments, request| {
-            let builder = litellm_core::messages::MessagesRoute::builder()
-                .with_http(
-                    crate::http::provider_client(py, arguments, asynchronous)?
-                        .map_err(crate::http::client_error)?,
-                )
-                .with_auth(crate::http::resources().auth.clone())
-                .with_secrets(crate::secrets::source(py)?);
-            let route = builder.build();
+            let route = litellm_core::messages::MessagesRoute::new(
+                crate::http::provider_client(py, arguments, asynchronous)?
+                    .map_err(crate::http::client_error)?,
+                crate::http::resources().auth.clone(),
+                crate::secrets::source(py)?,
+            );
             Ok(litellm_host::call::hosted_call(
                 request,
                 None,
@@ -51,7 +49,7 @@ fn run_messages(
                             call,
                             &interceptors,
                             litellm_core::CallOptions {
-                                cache: Some(options),
+                                cache: Some(options.policy),
                                 observers,
                             },
                         )

litellm-rust/crates/python-bridge/src/routes/mod.rs

@@ -6,6 +6,7 @@ pub(crate) mod messages;
 pub(crate) mod ocr;
 pub(crate) mod responses;
 pub(crate) mod token_counter;
+pub(crate) mod traces;
 
 use litellm_callbacks_legacy_python::LoggingOperation;
 use litellm_callbacks_legacy_python::{LegacyLogging, PublicCall};

litellm-rust/crates/python-bridge/src/routes/responses.rs

@@ -96,7 +96,7 @@ fn run_public(
                 )),
                 None => route,
             };
-            Ok(route.machine(request, cache_options))
+            Ok(route.machine(request, cache_options.policy))
         },
         host::ResponsesPythonHost(host),
         hooks,
@@ -142,7 +142,7 @@ impl ResponsesWebSocketConnection {
     ) -> PyResult<Bound<'py, PyAny>> {
         let headers = marshal_headers(headers)?;
         let timeout = optional_timeout(timeout_seconds);
-        crate::logger::run_async_value(py, async move {
+        crate::execution::run_async_value(py, async move {
             let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
                 .await
                 .map_err(route_error_to_pyerr)?;
@@ -152,21 +152,21 @@ impl ResponsesWebSocketConnection {
 
     fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
         let inner = self.inner.clone();
-        crate::logger::run_async_value(py, async move {
+        crate::execution::run_async_value(py, async move {
             inner.send_text(text).await.map_err(route_error_to_pyerr)
         })
     }
 
     fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         let inner = self.inner.clone();
-        crate::logger::run_async_value(py, async move {
+        crate::execution::run_async_value(py, async move {
             inner.recv_text().await.map_err(route_error_to_pyerr)
         })
     }
 
     fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
         let inner = self.inner.clone();
-        crate::logger::run_async_value(py, async move {
+        crate::execution::run_async_value(py, async move {
             inner.close().await.map_err(route_error_to_pyerr)
         })
     }

litellm-rust/crates/python-bridge/src/routes/token_counter.rs

@@ -1,4 +1,4 @@
-use crate::logger::run_async;
+use crate::execution::run_async;
 use std::sync::Arc;
 use std::{num::NonZero, thread::available_parallelism};
 

litellm-rust/crates/python-bridge/src/routes/traces.rs

@@ -0,0 +1,166 @@
+use std::collections::BTreeMap;
+
+use litellm_http::ClientVariant;
+use litellm_traces::{Connection, Error, InsertTable, Parameter, ReadQuery};
+use pyo3::{
+    exceptions::{PyOverflowError, PyRuntimeError, PyValueError},
+    prelude::*,
+};
+
+fn map_error(error: Error) -> PyErr {
+    match error {
+        Error::InvalidRow
+        | Error::InvalidTable
+        | Error::InvalidSchema
+        | Error::EmptySql
+        | Error::InvalidQuery => PyValueError::new_err(error.to_string()),
+        Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()),
+        Error::InvalidUrl
+        | Error::QueryFailed(_)
+        | Error::InsertFailed(_)
+        | Error::SchemaFailed(_)
+        | Error::ResponseTooLarge
+        | Error::InvalidResponse
+        | Error::Transport => PyRuntimeError::new_err(error.to_string()),
+    }
+}
+
+#[pyclass]
+pub struct NativeTraceStorage {
+    database: String,
+    writer: Connection,
+    reader: Option<Connection>,
+}
+
+#[pymethods]
+impl NativeTraceStorage {
+    #[new]
+    #[pyo3(signature = (database, url, reader_url = None))]
+    fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult<Self> {
+        litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?;
+        Ok(Self {
+            writer: Connection::writer(url).map_err(map_error)?,
+            reader: reader_url
+                .map(|value| Connection::reader(value, &database))
+                .transpose()
+                .map_err(map_error)?,
+            database,
+        })
+    }
+
+    fn ensure_schema<'py>(
+        &self,
+        py: Python<'py>,
+        trace_retention_days: u32,
+        spend_log_retention_days: u32,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.writer.clone();
+        let database = self.database.clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::ensure_schema(
+                    &client,
+                    &connection,
+                    &database,
+                    trace_retention_days,
+                    spend_log_retention_days,
+                )
+                .await
+            },
+            map_error,
+        )
+    }
+
+    fn insert_rows<'py>(
+        &self,
+        py: Python<'py>,
+        table: &str,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec<
+            BTreeMap<String, serde_json::Value>,
+        >,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let table = InsertTable::parse(table).map_err(map_error)?;
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        let connection = self.writer.clone();
+        let database = self.database.clone();
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::insert_rows(&client, &connection, &database, table, rows).await
+            },
+            map_error,
+        )
+    }
+
+    fn lens_query<'py>(
+        &self,
+        py: Python<'py>,
+        name: &str,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap<
+            String,
+            Parameter,
+        >,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let query = litellm_traces::LensQuery::parse(name).map_err(map_error)?;
+        let connection = self.reader.clone().ok_or_else(|| {
+            PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL")
+        })?;
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::execute_read(&client, &connection, query.sql(), &parameters).await
+            },
+            map_error,
+        )
+    }
+
+    fn query<'py>(
+        &self,
+        py: Python<'py>,
+        query: &str,
+        #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap<
+            String,
+            Parameter,
+        >,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let query = ReadQuery::parse(query).map_err(map_error)?;
+        let connection = self.reader.clone().ok_or_else(|| {
+            PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL")
+        })?;
+        let client = crate::http::host_client(py, ClientVariant::NoRedirect)?;
+        crate::execution::run_async(
+            py,
+            async move {
+                litellm_traces::execute_named_read(&client, &connection, query, &parameters).await
+            },
+            map_error,
+        )
+    }
+}
+
+#[pyfunction]
+pub fn trace_decode_otlp<'py>(
+    py: Python<'py>,
+    body: &[u8],
+    content_type: Option<&str>,
+    content_encoding: Option<&str>,
+    max_decompressed_bytes: usize,
+) -> PyResult<Bound<'py, PyAny>> {
+    let spans = py
+        .detach(|| {
+            litellm_traces::decode_otlp(
+                body,
+                content_type,
+                content_encoding,
+                max_decompressed_bytes,
+            )
+        })
+        .map_err(|error| match error {
+            litellm_traces::DecodeError::TooLarge => PyOverflowError::new_err(error.to_string()),
+            _ => PyValueError::new_err(error.to_string()),
+        })?;
+    litellm_host_python::Pythonized(spans).into_pyobject(py)
+}