ci: merge main into litellm_chore_f89937

.github/scripts/assert_ci_coverage.py

@@ -130,37 +130,24 @@ def _unit_selection_arms(repo_root: pathlib.Path = REPO_ROOT) -> Mapping[str, fr
     text: Final = _uncommented(script.read_text())
     return MappingProxyType(
         {
-            label: frozenset(
-                match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body)
-            )
+            label: frozenset(match.group(0).rstrip("/") for match in TEST_TOKEN_RE.finditer(body))
             for label, body in SELECTION_ARM_RE.findall(text)
         }
     )
 
 
 def _unit_selection_tokens(repo_root: pathlib.Path = REPO_ROOT) -> frozenset[str]:
-    return frozenset(
-        token for tokens in _unit_selection_arms(repo_root).values() for token in tokens
-    )
+    return frozenset(token for tokens in _unit_selection_arms(repo_root).values() for token in tokens)
 
 
 def _wired_unit_flags(scalars: Iterable[Scalar]) -> frozenset[str]:
-    return frozenset(
-        scalar.value
-        for scalar in scalars
-        if scalar.key == "unit-flag" and "${{" not in scalar.value
-    )
+    return frozenset(scalar.value for scalar in scalars if scalar.key == "unit-flag" and "${{" not in scalar.value)
 
 
-def _shard_tokens(
-    scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]
-) -> frozenset[str]:
+def _shard_tokens(scalars: Iterable[Scalar], arms: Mapping[str, frozenset[str]]) -> frozenset[str]:
     wired: Final = _wired_unit_flags(scalars)
     return _invoked_test_tokens(scalars) | frozenset(
-        token
-        for label, tokens in arms.items()
-        if label in wired
-        for token in tokens
+        token for label, tokens in arms.items() if label in wired for token in tokens
     )
 
 
@@ -544,17 +531,37 @@ def _integration_groups(runner: pathlib.Path) -> dict[str, tuple[str, ...]]:
     return {group: tuple(folders) for group, folders in ast.literal_eval(mapping).items()}
 
 
+def _integration_github_files(runner: pathlib.Path) -> frozenset[str]:
+    module: Final = ast.parse(runner.read_text())
+    literal: Final = next(
+        (
+            node.value
+            for node in module.body
+            if isinstance(node, ast.AnnAssign)
+            and isinstance(node.target, ast.Name)
+            and node.target.id == "GITHUB_FILES"
+        ),
+        None,
+    )
+    if literal is None:
+        return frozenset()
+    values: Final = literal.args[0] if isinstance(literal, ast.Call) else literal
+    return frozenset(ast.literal_eval(values))
+
+
 def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
     runner: Final = repo_root / "tests/integration/run.py"
     if not runner.exists():
         return frozenset(), ()
     groups: Final = _integration_groups(runner)
+    github_files: Final = _integration_github_files(runner)
     integration_root: Final = repo_root / "tests/integration"
     paths: Final = frozenset(
         str(path.relative_to(repo_root))
         for folders in groups.values()
         for folder in folders
         for path in (integration_root / folder).rglob("test_*.py")
+        if str(path.relative_to(repo_root)) not in github_files
     )
     browser_manifest: Final = repo_root / "tests/e2e/ui/tests/integrationCritical/expected.json"
     browser_nodes: Final = json.loads(browser_manifest.read_text()) if browser_manifest.exists() else ()
@@ -595,10 +602,22 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         for path in (repo_root / ".github/workflows").glob("*.y*ml")
         for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
     )
-    findings: Final = tuple(
-        Finding(path, "integration contract is also selected by GitHub Actions")
-        for path in paths
-        if any(_token_covers(token, path) for token in gha_tokens)
+    findings: Final = (
+        tuple(
+            Finding(path, "integration contract is also selected by GitHub Actions")
+            for path in paths
+            if any(_token_covers(token, path) for token in gha_tokens)
+        )
+        + tuple(
+            Finding(path, "GitHub-owned integration contract has no invoking workflow")
+            for path in sorted(github_files)
+            if not any(_token_covers(token, path) for token in gha_tokens)
+        )
+        + tuple(
+            Finding(path, "GitHub-owned integration file is missing")
+            for path in sorted(github_files)
+            if not (repo_root / path).is_file()
+        )
     )
     browser_commands: Final = tuple(
         scalar.value
@@ -642,7 +661,7 @@ def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozens
         return frozenset(), findings + (
             Finding(str(runner.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
         )
-    return paths | browser_paths, findings + group_findings + browser_findings + exclusion_findings
+    return paths | browser_paths | github_files, findings + group_findings + browser_findings + exclusion_findings
 
 
 def main() -> int:

.github/workflows/image-scan.yml

@@ -15,6 +15,9 @@ on:
       - gateway/main.py
       - backend/Dockerfile
       - backend/main.py
+      - deploy/lens/**
+      - litellm/proxy/lens/release.py
+      - tests/e2e/migrations/lens_compose_smoke.sh
       - docker/component_entrypoint.sh
       - docker/entrypoint.sh
       - litellm/proxy/prisma_migration.py
@@ -113,7 +116,7 @@ jobs:
           persist-credentials: false
 
       - name: Build runtime image
-        run: docker build -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
+        run: docker build --build-arg LITELLM_RELEASE_TAG=v0.0.0-lens-ci -f Dockerfile -t litellm-runtime-scan:${{ github.sha }} .
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -127,6 +130,11 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py tests/proxy_migration_tests/test_image_bedrock_realtime_extra.py -v
 
+      - name: Verify the bundled Lens Compose installation and restart
+        env:
+          LITELLM_IMAGE: litellm-runtime-scan:${{ github.sha }}
+        run: bash tests/e2e/migrations/lens_compose_smoke.sh
+
   migrations-image:
     name: migrations-image
     runs-on: ubuntu-latest

.github/workflows/lens-worker.yml

@@ -34,7 +34,14 @@ jobs:
         with:
           persist-credentials: false
       - name: Build Lens worker
-        run: docker build -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+        run: docker build --build-arg LITELLM_RELEASE_TAG=sha-${{ github.sha }} -f deploy/lens/Dockerfile -t lens-worker:${{ github.sha }} .
+      - name: Reject custom builds without a matching release tag
+        run: |
+          if docker build --progress plain -f deploy/lens/Dockerfile -t lens-worker:unversioned . > missing-tag.log 2>&1; then
+            echo "::error::An unversioned worker build unexpectedly succeeded"
+            exit 1
+          fi
+          grep -F 'LITELLM_RELEASE_TAG: Pass --build-arg LITELLM_RELEASE_TAG matching the gateway' missing-tag.log
       - name: Verify standalone imports with a read-only filesystem
         run: |
           docker run --rm --network none --read-only --cap-drop ALL --tmpfs /tmp:rw,noexec,nosuid,size=1g \

Dockerfile

@@ -116,6 +116,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 
 USER root
 

backend/Dockerfile

@@ -71,6 +71,8 @@ RUN sed -i 's/\r$//' docker/component_entrypoint.sh && chmod +x docker/component
 
 # ---------- Runtime ----------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 
 USER root
 

deploy/lens/Dockerfile

@@ -1,7 +1,10 @@
 FROM python:3.12-slim
+ARG LITELLM_RELEASE_TAG=""
+RUN : "${LITELLM_RELEASE_TAG:?Pass --build-arg LITELLM_RELEASE_TAG matching the gateway}"
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 WORKDIR /app
 RUN pip install --no-cache-dir httpx==0.28.1 pydantic==2.11.7
-COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py /app/lens/
+COPY litellm/proxy/lens/__init__.py litellm/proxy/lens/models.py litellm/proxy/lens/trace_store.py litellm/proxy/lens/analysis.py litellm/proxy/lens/worker.py litellm/proxy/lens/release.py /app/lens/
 COPY litellm/proxy/lens/prompts/ /app/lens/prompts/
 USER 65532:65532
 CMD ["python", "-m", "lens.worker"]

deploy/lens/README.md

@@ -2,7 +2,60 @@
 
 Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`)
 
-## Start a worker
+## Install the release stack
+
+Each stable, RC, and dev release containing Lens publishes the worker at the same version on GHCR and Docker Hub. Use the [LiteLLM releases page](https://github.com/BerriAI/litellm/releases) to select a version that includes the coordinated worker release
+
+For a new local installation, install Docker with Compose, download the two release files, and create a private environment file. Replace `X.Y.Z` with the release version, without `v` (RCs use `X.Y.Z-rc.N`)
+
+```bash
+mkdir litellm-lens
+cd litellm-lens
+LENS_RELEASE=X.Y.Z
+curl -fSLo compose.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/stack.yaml"
+curl -fSLo config.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/config.yaml"
+umask 077
+printf 'LITELLM_VERSION=%s\nLITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' \
+  "$LENS_RELEASE" "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env
+printf 'POSTGRES_PASSWORD=%s\nCLICKHOUSE_PASSWORD=%s\n' \
+  "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> .env
+docker compose up -d
+```
+
+Open `http://localhost:4000/ui/`, log in as `admin` with `LITELLM_MASTER_KEY` from `.env`, and add a model in the dashboard. In Lens, select **Connect worker**, choose that model and a monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?**, copy the worker token, and add `LENS_WORKER_TOKEN=<token>` to `.env`
+
+```bash
+docker compose --profile lens up -d
+```
+
+The stack starts LiteLLM, PostgreSQL, ClickHouse, and the worker from published images. The dashboard shows **Worker connected**. The worker has a limited token, no database credentials, and no provider keys. The stack exposes only the dashboard on localhost; use your normal ingress and managed databases for a public production deployment
+
+Keep `.env` private and preserve its salt key. Keep both named database volumes. To upgrade, wait for active investigations to finish, stop the worker, change only `LITELLM_VERSION`, then pull and recreate the stack:
+
+```bash
+docker compose --profile lens stop lens-worker
+# Update LITELLM_VERSION in .env to the new release
+docker compose --profile lens pull
+docker compose --profile lens up -d
+```
+
+This preserves your investigations, findings, model credentials, and worker token. Never use `down -v` during an upgrade. If moving from an existing installation, keep its databases and add the standalone worker instead of creating an empty replacement stack
+
+## Helm
+
+The componentized `helm/litellm` chart includes an optional Lens worker. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values:
+
+```yaml
+lensWorker:
+  enabled: true
+  tokenSecret:
+    name: litellm-lens-worker
+    key: token
+```
+
+The worker image defaults to the chart's application version, and the chart connects it to the backend service. Keep these values and the Secret when upgrading the chart so the gateway and worker upgrade together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.tag`, and `lensWorker.url`. The dashboard uses the chart's worker image for standalone install commands too
+
+## Standalone worker
 
 Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL and agent tracing. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries:
 
@@ -23,17 +76,17 @@ In **Lens > Investigations**, click **Connect worker**, choose an analysis model
 
 The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis
 
-The dashboard and Compose file pin a verified worker image by digest. The image uses Linux amd64, and the generated command selects that platform. CI also publishes immutable `:sha-<commit>` tags for successful worker builds on `main`. Keep the worker image compatible with your gateway version
+The dashboard selects the worker image matching the running gateway release. Release images support Linux amd64 and arm64. CI also publishes `:sha-<commit>` development images; use those only with a gateway built from the same commit and release tag
 
 After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying
 
-For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL` and `LENS_WORKER_TOKEN` in an environment file. Its default image is already selected:
+For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and `LITELLM_VERSION` (without `v`) in a private environment file. To use another registry, set `LENS_WORKER_IMAGE` to the compatible image instead of setting a version:
 
 ```bash
 docker compose --env-file /path/to/lens.env -f compose.yaml up -d
 ```
 
-Developers can build locally with `LENS_WORKER_IMAGE=litellm-lens-worker:local docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`. To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000
+To work on Lens itself, `make lens-dev` runs the proxy, a worker from source and the hot-reload dashboard together; set `LENS_DEV_PROXY_PORT` / `LENS_DEV_UI_PORT` to move them off 4000/3000. For a local container build, set `LENS_WORKER_IMAGE=litellm-lens-worker:local` and `LITELLM_RELEASE_TAG` to the gateway's release tag, then use `docker compose -f deploy/lens/compose.yaml -f deploy/lens/compose.build.yaml up -d --build`
 
 The generated command gives the worker 1 GiB of temporary memory-backed storage, shared across parallel reviews. Change `size=1g` in the Docker command or set `LENS_WORKER_TMP_SIZE` with Compose to fit your server and workload. A storage failure marks the scan as failed, cleans up temporary traces, and leaves the worker available for other scans; it does not silently truncate the review. Existing workers must be recreated with the new image and mount options
 
@@ -130,3 +183,14 @@ The Lens API now uses `/lens` instead of `/engine`, list responses use `lenses`,
 Stop workers and let active scans finish before upgrading. Deploy proxy instances together: older proxies cannot use the renamed database tables. The schema migration renames the three Lens tables and the run-history identifier column in place, preserving saved investigations, findings, history, worker credentials, and billing assignments. Existing migration files retain their original names and checksums
 
 Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy Lens table exists, preventing Prisma from dropping saved data. Apply `litellm-proxy-extras/litellm_proxy_extras/migrations/20261001100000_rename_lens/migration.sql` to the configured database schema before retrying. Deployments already using migration history can instead start without `--use_prisma_db_push` to apply the shipped migration normally. Fresh databases and databases already using the renamed tables can continue using database push
+
+
+## Release compatibility
+
+Released gateway and worker images carry `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release
+
+The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/litellm/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64
+
+For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version
+
+The hourly development pipeline pins all component images to the same selected commit and publishes its chart only after every build and worker smoke test succeeds. The public commit-tagged worker workflow publishes on Lens-related changes, so an arbitrary `main` commit may require building your own pair; do not substitute the newest available worker

deploy/lens/compose.build.yaml

@@ -3,4 +3,6 @@ services:
     build:
       context: ../..
       dockerfile: deploy/lens/Dockerfile
+      args:
+        LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:?Set the release tag used by the gateway}
     image: litellm-lens-worker:local

deploy/lens/compose.yaml

@@ -1,6 +1,6 @@
 services:
   lens-worker:
-    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker@sha256:44f0597c7583dcfef999ece9a8bc02cfeb9f0f5167a1221cee3bd10b1b79271b}
+    image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION:?Set LITELLM_VERSION to the gateway release, without the v prefix}}
     environment:
       LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container}
       LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI}

deploy/lens/config.yaml

@@ -0,0 +1,7 @@
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  tracing:
+    store:
+      type: clickhouse
+      url: os.environ/CLICKHOUSE_URL
+      retention_days: 14

deploy/lens/stack.yaml

@@ -0,0 +1,91 @@
+name: litellm-lens
+
+services:
+  litellm:
+    image: ghcr.io/berriai/litellm:${LITELLM_VERSION:?Set LITELLM_VERSION to a published release, without the v prefix}
+    entrypoint:
+      - python3
+      - -c
+      - |
+        import os, sys
+        from urllib.parse import quote
+        postgres_password = quote(os.environ["POSTGRES_PASSWORD"], safe="")
+        clickhouse_password = quote(os.environ["CLICKHOUSE_PASSWORD"], safe="")
+        os.environ["DATABASE_URL"] = f"postgresql://litellm:{postgres_password}@db:5432/litellm"
+        os.environ["CLICKHOUSE_URL"] = f"http://default:{clickhouse_password}@clickhouse:8123"
+        os.execv("docker/prod_entrypoint.sh", ["docker/prod_entrypoint.sh", *sys.argv[1:]])
+    command: ["--config", "/app/lens-config.yaml", "--port", "4000"]
+    environment:
+      LITELLM_MASTER_KEY: ${LITELLM_MASTER_KEY:?Set a strong master key}
+      LITELLM_SALT_KEY: ${LITELLM_SALT_KEY:?Set a permanent encryption key and keep it across upgrades}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set a permanent database password}
+      STORE_MODEL_IN_DB: "True"
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set a permanent ClickHouse password}
+      LENS_WORKER_IMAGE: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
+    volumes:
+      - ./config.yaml:/app/lens-config.yaml:ro
+    ports:
+      - "127.0.0.1:${LITELLM_PORT:-4000}:4000"
+    networks: [proxy, storage]
+    depends_on:
+      db:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    restart: unless-stopped
+
+  lens-worker:
+    profiles: [lens]
+    image: ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION}
+    environment:
+      LITELLM_URL: http://litellm:4000
+      LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:-}
+    depends_on: [litellm]
+    networks: [proxy]
+    restart: unless-stopped
+    read_only: true
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,size=${LENS_WORKER_TMP_SIZE:-1g}
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]
+
+  db:
+    image: postgres:16
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: litellm
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    networks: [storage]
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    restart: unless-stopped
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:26.9.6.6
+    environment:
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--user", "default", "--password", "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 20
+    restart: unless-stopped
+    networks: [storage]
+
+networks:
+  proxy:
+  storage:
+    internal: true
+
+volumes:
+  postgres_data:
+  clickhouse_data:

docker/Dockerfile.database

@@ -113,6 +113,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 
 USER root
 

docker/Dockerfile.non_root

@@ -122,6 +122,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG LITELLM_RELEASE_TAG=""
+ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
 WORKDIR /app
 USER root
 

helm/litellm/templates/_helpers.tpl

@@ -471,6 +471,13 @@ Directory of the collector's unix socket, shared by the gateway and
 collector containers through an emptyDir. Empty when the sidecar is off
 or gateway.collector.address is a tcp://127.0.0.1:<port> address.
 */}}
+{{- define "litellm.lensWorker.image" -}}
+{{- $backendTag := .Values.backend.image.tag | default .Chart.AppVersion -}}
+{{- $releaseTag := ternary (printf "v%s" $backendTag) $backendTag (regexMatch "^[0-9]" $backendTag) -}}
+{{- $tag := .Values.lensWorker.image.tag | default $releaseTag -}}
+{{- printf "%s:%s" .Values.lensWorker.image.repository $tag -}}
+{{- end -}}
+
 {{- define "litellm.gateway.collectorSocketDir" -}}
 {{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
 {{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}

helm/litellm/templates/backend/deployment.yaml

@@ -57,6 +57,8 @@ spec:
               containerPort: 4001
               protocol: TCP
           env:
+            - name: LENS_WORKER_IMAGE
+              value: {{ include "litellm.lensWorker.image" . | quote }}
             {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }}
             {{- if .Values.gateway.config.create }}
             - name: CONFIG_FILE_PATH

helm/litellm/templates/lens/deployment.yaml

@@ -0,0 +1,72 @@
+{{- if .Values.lensWorker.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "litellm.fullname" . }}-lens-worker
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: lens-worker
+spec:
+  replicas: {{ .Values.lensWorker.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: lens-worker
+  template:
+    metadata:
+      labels:
+        {{- include "litellm.commonLabels" . | nindent 8 }}
+        app.kubernetes.io/component: lens-worker
+    spec:
+      automountServiceAccountToken: false
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532
+        runAsGroup: 65532
+        fsGroup: 65532
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: lens-worker
+          image: {{ include "litellm.lensWorker.image" . | quote }}
+          imagePullPolicy: {{ .Values.lensWorker.image.pullPolicy }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: [ALL]
+          env:
+            - name: LITELLM_URL
+              value: {{ .Values.lensWorker.url | default (printf "http://%s:%v" (include "litellm.backend.fullname" .) .Values.backend.service.port) | quote }}
+            - name: LENS_WORKER_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ required "lensWorker.tokenSecret.name must reference a Lens worker token" .Values.lensWorker.tokenSecret.name | quote }}
+                  key: {{ .Values.lensWorker.tokenSecret.key | quote }}
+          resources:
+            {{- toYaml .Values.lensWorker.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir:
+            medium: Memory
+            sizeLimit: {{ .Values.lensWorker.tmpSizeLimit }}
+      {{- with .Values.lensWorker.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.lensWorker.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.lensWorker.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

helm/litellm/tests/lens_worker_tests.yaml

@@ -0,0 +1,114 @@
+suite: Lens worker release and credentials
+templates:
+  - lens/deployment.yaml
+  - backend/deployment.yaml
+  - gateway/configmap.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: keeps the worker opt in
+    template: lens/deployment.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+  - it: requires a limited worker credential when enabled
+    template: lens/deployment.yaml
+    set:
+      lensWorker.enabled: true
+    asserts:
+      - failedTemplate:
+          errorMessage: lensWorker.tokenSecret.name must reference a Lens worker token
+  - it: uses the chart release and a secret without granting Kubernetes access
+    template: lens/deployment.yaml
+    chart:
+      appVersion: v1.2.3
+    set:
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3
+      - equal:
+          path: spec.template.spec.containers[0].env[1].valueFrom.secretKeyRef
+          value:
+            name: lens-credential
+            key: token
+      - equal:
+          path: spec.template.spec.automountServiceAccountToken
+          value: false
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
+          value: true
+      - equal:
+          path: spec.template.spec.volumes[0].emptyDir
+          value:
+            medium: Memory
+            sizeLimit: 1Gi
+  - it: advertises the same private dev image to standalone installers
+    template: backend/deployment.yaml
+    set:
+      lensWorker.image.repository: registry.example/lens-worker
+      lensWorker.image.tag: branch-main-1234567
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_WORKER_IMAGE
+            value: registry.example/lens-worker:branch-main-1234567
+  - it: supports an external gateway and a registry override
+    template: lens/deployment.yaml
+    set:
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+      lensWorker.url: https://gateway.example/proxy
+      lensWorker.image.repository: registry.example/lens-worker
+      lensWorker.image.tag: branch-main-1234567
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: registry.example/lens-worker:branch-main-1234567
+      - equal:
+          path: spec.template.spec.containers[0].env[0].value
+          value: https://gateway.example/proxy
+  - it: prefixes a numeric chart release with v
+    template: lens/deployment.yaml
+    chart:
+      appVersion: 1.2.3-rc.4
+    set:
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-rc.4
+  - it: follows a backend image override when no worker tag is set
+    template: lens/deployment.yaml
+    set:
+      backend.image.tag: branch-main-1234567
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:branch-main-1234567
+  - it: recommends the overridden backend release for standalone installers
+    template: backend/deployment.yaml
+    set:
+      backend.image.tag: v1.2.3-dev.4
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENS_WORKER_IMAGE
+            value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4
+  - it: normalizes a numeric backend tag to the published worker tag
+    template: lens/deployment.yaml
+    set:
+      backend.image.tag: 1.2.3-dev.4
+      lensWorker.enabled: true
+      lensWorker.tokenSecret.name: lens-credential
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/berriai/litellm-lens-worker:v1.2.3-dev.4

helm/litellm/values.yaml

@@ -629,3 +629,25 @@ ui:
   affinity: {}
   # Same shape as gateway.topologySpreadConstraints.
   topologySpreadConstraints: []
+
+lensWorker:
+  enabled: false
+  replicaCount: 1
+  image:
+    repository: ghcr.io/berriai/litellm-lens-worker
+    tag: ""
+    pullPolicy: IfNotPresent
+  tokenSecret:
+    name: ""
+    key: token
+  url: ""
+  tmpSizeLimit: 1Gi
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      memory: 2Gi
+  nodeSelector: {}
+  tolerations: []
+  affinity: {}

litellm-rust/Cargo.lock

@@ -4454,6 +4454,7 @@ name = "litellm-traces"
 version = "0.1.0"
 dependencies = [
  "askama",
+ "base64 0.22.1",
  "criterion",
  "indexmap 2.14.0",
  "litellm-llms-types",

litellm-rust/crates/traces-clickhouse/migrations/0015_spend_gateway_call_id.sql

@@ -0,0 +1,6 @@
+ALTER TABLE {database}.spend_logs
+    ADD COLUMN IF NOT EXISTS litellm_call_id String DEFAULT '' AFTER response_id,
+    ADD INDEX IF NOT EXISTS idx_litellm_call_id litellm_call_id
+        TYPE bloom_filter(0.001) GRANULARITY 1,
+    ADD INDEX IF NOT EXISTS idx_request_id request_id
+        TYPE bloom_filter(0.001) GRANULARITY 1

litellm-rust/crates/traces-clickhouse/query/spend_batch.sql

@@ -1,5 +1,5 @@
 SELECT * FROM (
-SELECT request_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
+SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
        toUnixTimestamp64Milli(start_time) AS start_ms
 FROM (
     SELECT *,
@@ -17,7 +17,8 @@ FROM (
 )
 WHERE response_id IN {response_ids:Array(String)}
    OR upstream_response_id IN {response_ids:Array(String)}
-   OR request_id IN {request_ids:Array(String)}
+   OR litellm_call_id IN {request_ids:Array(String)}
+   OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)})
    OR (trace_id != '' AND trace_id IN {trace_ids:Array(String)})
 ORDER BY start_time DESC
 )

litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql

@@ -1,4 +1,4 @@
-SELECT request_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
+SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
        toUnixTimestamp64Milli(start_time) AS start_ms
 FROM (
     SELECT *,
@@ -16,6 +16,7 @@ FROM (
 )
 WHERE response_id IN {response_ids:Array(String)}
    OR upstream_response_id IN {response_ids:Array(String)}
-   OR request_id IN {request_ids:Array(String)}
+   OR litellm_call_id IN {request_ids:Array(String)}
+   OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)})
    OR (trace_id != '' AND trace_id IN {trace_ids:Array(String)})
 ORDER BY start_time DESC

litellm-rust/crates/traces-clickhouse/src/query/named.rs

@@ -204,6 +204,7 @@ impl From<contracts::SpendByResponseIdsParams> for SpendByResponseIdsParams {
 #[serde(remote = "contracts::SpendByResponseIdsRow")]
 struct SpendByResponseIdsRowEncoding {
     pub request_id: String,
+    pub litellm_call_id: String,
     pub response_id: String,
     pub upstream_response_id: String,
     pub trace_id: String,
@@ -355,7 +356,7 @@ mod tests {
             quoted,
         );
         round_trip::<SpendByResponseIdsRow>(
-            json!({"request_id": "request", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
+            json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
             quoted,
         );
     }
@@ -386,7 +387,7 @@ mod tests {
         #[case] expected: Option<f64>,
     ) {
         let row: SpendByResponseIdsRow = serde_json::from_value(json!({
-            "request_id": "request", "response_id": "response", "upstream_response_id": "",
+            "request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "",
             "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key",
             "user": "user", "spend": cost, "start_ms": 0
         }))
@@ -399,7 +400,7 @@ mod tests {
     #[case::boolean(json!(true))]
     fn spend_rows_reject_invalid_cost(#[case] cost: serde_json::Value) {
         let row = serde_json::from_value::<SpendByResponseIdsRow>(json!({
-            "request_id": "request", "response_id": "response", "upstream_response_id": "",
+            "request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "",
             "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key",
             "user": "user", "spend": cost, "start_ms": 0
         }));

litellm-rust/crates/traces-clickhouse/src/span_row.rs

@@ -210,8 +210,8 @@ fn request_id(evidence: &CallEvidence) -> &str {
         .into_iter()
         .flatten()
         .find_map(|key| match key {
-            CallKey::LiteLlmRequest(id) | CallKey::ProviderResponse(id) => Some(id.as_str()),
-            CallKey::Transport => None,
+            CallKey::ProviderResponse(id) => Some(id.as_str()),
+            CallKey::LiteLlmRequest(_) | CallKey::Transport => None,
         })
         .unwrap_or_default()
 }

litellm-rust/crates/traces-clickhouse/tests/fixtures/README.md

@@ -25,3 +25,5 @@ The LlamaIndex captures contain provider IDs inside `output.value.raw.id`. Regre
 Curated SQL lives in `tests/queries/*.sql`. Each query has a matching `.expected.json` containing ordered result rows for `admin`, `team`, `key`, and `other_team` readers. Update the exports and expected results together. Add a named case in `tests/queries.rs` for each new query. Assertions compare only result data, excluding server statistics and execution timing
 
 Typed query tests execute the production SQL through `litellm_storage_clickhouse::fetch` using contracts from `litellm-traces`. The fixture projection is test setup, so this suite covers the Rust decoder, normalization, inserts, schema, readers, and queries. Python ingress transformations, including payload truncation and exception-event fallback, remain covered by the Python tests
+
+`crates/traces/tests/captures.rs` resolves every capture against its spend rows without ClickHouse and checks the unrelated-transport and redundant-response-ID invariants

litellm-rust/crates/traces-clickhouse/tests/fixtures/google_adk_billed_failure_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1CtycizaTWvf3rjCEenOd7gXLRl","response_id":"chatcmpl-EV1CtycizaTWvf3rjCEenOd7gXLRl","litellm_call_id":"5034129d-560a-4a8b-85da-4f0f3628e70b","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.37e-05,"prompt_tokens":32,"completion_tokens":41,"total_tokens":73,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"8110744290a0840bc6029401723da262","trace_id":"8110744290a0840bc6029401723da262","span_id":"43984ccbf7e93ffd","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"google_adk_billed_failure\",\"trace_id\":\"8110744290a0840bc6029401723da262\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"You are an agent. Your internal name is \\\"research_agent\\\".\"}, {\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1CtycizaTWvf3rjCEenOd7gXLRl\", \"created\": 1791061515, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s actions, decisions, and tool calls.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 41, \"prompt_tokens\": 32, \"total_tokens\": 73, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 18, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061515632,"end_time":1791061516529,"completion_start_time":1791061516529}

litellm-rust/crates/traces-clickhouse/tests/fixtures/google_adk_retry_spend_logs.jsonl

@@ -0,0 +1,2 @@
+{"request_id":"chatcmpl-EV1DqRSm66p9hLCDdlrh9loQ2slG2","response_id":"chatcmpl-EV1DqRSm66p9hLCDdlrh9loQ2slG2","litellm_call_id":"4d1e094d-24e7-49f6-8d33-656a7d7712ae","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":3.7199999999999996e-05,"prompt_tokens":32,"completion_tokens":68,"total_tokens":100,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"cdd888fb2417b10d57e2b2cee4dd428c","trace_id":"cdd888fb2417b10d57e2b2cee4dd428c","span_id":"977a62497f2dda9d","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"google_adk_retry\",\"trace_id\":\"cdd888fb2417b10d57e2b2cee4dd428c\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"You are an agent. Your internal name is \\\"research_agent\\\".\"}, {\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1DqRSm66p9hLCDdlrh9loQ2slG2\", \"created\": 1791061574, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s actions, tool calls, and outcomes.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 68, \"prompt_tokens\": 32, \"total_tokens\": 100, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 45, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061574518,"end_time":1791061576007,"completion_start_time":1791061576007}
+{"request_id":"chatcmpl-EV1Dsf8Gh5U6VZykhfciNU1lmsTgS","response_id":"chatcmpl-EV1Dsf8Gh5U6VZykhfciNU1lmsTgS","litellm_call_id":"38acfc26-d8f3-4adc-9528-0196c28b8640","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.67e-05,"prompt_tokens":32,"completion_tokens":47,"total_tokens":79,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"cdd888fb2417b10d57e2b2cee4dd428c","trace_id":"cdd888fb2417b10d57e2b2cee4dd428c","span_id":"2856caaa423beba1","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"google_adk_retry\",\"trace_id\":\"cdd888fb2417b10d57e2b2cee4dd428c\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"You are an agent. Your internal name is \\\"research_agent\\\".\"}, {\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1Dsf8Gh5U6VZykhfciNU1lmsTgS\", \"created\": 1791061576, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces show the steps an agent takes to complete a task.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 47, \"prompt_tokens\": 32, \"total_tokens\": 79, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 25, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061576510,"end_time":1791061577760,"completion_start_time":1791061577760}

litellm-rust/crates/traces-clickhouse/tests/fixtures/google_adk_stream_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1C3uQ69CnjfRcZy2MDlreaXkRVd","response_id":"chatcmpl-EV1C3uQ69CnjfRcZy2MDlreaXkRVd","litellm_call_id":"73d0b177-86e5-4e11-9704-bff2667a0a75","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":9.489999999999999e-05,"prompt_tokens":29,"completion_tokens":184,"total_tokens":213,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"59fb70ffc358dfe30a5e0d4615e4ca7f","trace_id":"59fb70ffc358dfe30a5e0d4615e4ca7f","span_id":"fc3bcda50363d82c","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"google_adk_stream\",\"trace_id\":\"59fb70ffc358dfe30a5e0d4615e4ca7f\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"You are an agent. Your internal name is \\\"research_agent\\\".\"}, {\"role\": \"user\", \"content\": \"What is an agent trace?\"}]","response":"{\"id\": \"chatcmpl-EV1C3uQ69CnjfRcZy2MDlreaXkRVd\", \"created\": 1791061464, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"An **agent trace** is a record of an AI agent\\u2019s execution: what it received, the actions it took (such as tool calls), the results it got back, and how the task ended.\\n\\nTraces help developers debug failures, understand behavior, and evaluate performance. They may include timestamps, inputs and outputs, tool errors, or state changes. A trace doesn\\u2019t have to include the model\\u2019s private reasoning; it can record only observable steps.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 184, \"prompt_tokens\": 29, \"total_tokens\": 213, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 84, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 9.489999999999999e-05}, \"service_tier\": \"default\"}","start_time":1791061463025,"end_time":1791061465613,"completion_start_time":1791061464712}

litellm-rust/crates/traces-clickhouse/tests/fixtures/mastra_simple_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV19kkSsaD6PMG5TGol2OVhYek5te","response_id":"chatcmpl-EV19kkSsaD6PMG5TGol2OVhYek5te","litellm_call_id":"afc4951d-7141-4476-b4d5-529b294e56b7","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":7.329999999999999e-05,"prompt_tokens":23,"completion_tokens":142,"total_tokens":165,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"6573bbd66c74e1ceb09fd173fe4cdc45","trace_id":"6573bbd66c74e1ceb09fd173fe4cdc45","span_id":"453659faa8fb4313","request_tags":["User-Agent: ai-sdk-openai-compatible","User-Agent: ai-sdk-openai-compatible/3.0.62 ai-sdk-provider-utils/5.0.53 node.js/25"],"metadata":"{\"fixture_capture\":{\"name\":\"mastra_simple\",\"trace_id\":\"6573bbd66c74e1ceb09fd173fe4cdc45\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"Answer the question concisely.\"}, {\"role\": \"user\", \"content\": \"What is an agent trace?\"}]","response":"{\"id\": \"chatcmpl-EV19kkSsaD6PMG5TGol2OVhYek5te\", \"created\": 1791061320, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"An **agent trace** is a record of an AI agent\\u2019s steps during a task\\u2014such as the inputs it received, actions or tool calls it made, results it got back, and its final response. Traces help people debug, evaluate, and audit agent behavior.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 142, \"prompt_tokens\": 23, \"total_tokens\": 165, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 78, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061319992,"end_time":1791061322400,"completion_start_time":1791061322400}

litellm-rust/crates/traces-clickhouse/tests/fixtures/pydantic_ai_billed_failure_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1DWkeu7A7n33K6iCaSxCDK1G71w","response_id":"chatcmpl-EV1DWkeu7A7n33K6iCaSxCDK1G71w","litellm_call_id":"27fa6242-6416-4749-9c1f-bc3ab3f83f35","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.45e-05,"prompt_tokens":15,"completion_tokens":46,"total_tokens":61,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"478c93b0465e5e11cc19e599738eadeb","trace_id":"478c93b0465e5e11cc19e599738eadeb","span_id":"877363da53522020","request_tags":["User-Agent: pydantic-ai","User-Agent: pydantic-ai/2.53.0"],"metadata":"{\"fixture_capture\":{\"name\":\"pydantic_ai_billed_failure\",\"trace_id\":\"478c93b0465e5e11cc19e599738eadeb\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1DWkeu7A7n33K6iCaSxCDK1G71w\", \"created\": 1791061554, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces show the steps an AI agent takes to complete a task.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 46, \"prompt_tokens\": 15, \"total_tokens\": 61, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 23, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061554409,"end_time":1791061555799,"completion_start_time":1791061555799}

litellm-rust/crates/traces-clickhouse/tests/fixtures/pydantic_ai_retry_spend_logs.jsonl

@@ -0,0 +1,2 @@
+{"request_id":"chatcmpl-EV1EcoG4Hwa0tTYV7gLus0fwAJ7wY","response_id":"chatcmpl-EV1EcoG4Hwa0tTYV7gLus0fwAJ7wY","litellm_call_id":"bbff9fcc-930d-47fe-b0eb-0ab1eba0eff0","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.8e-05,"prompt_tokens":15,"completion_tokens":53,"total_tokens":68,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"213790ddf7f01594d52d54818775b98d","trace_id":"213790ddf7f01594d52d54818775b98d","span_id":"10cd8cbd5f72b220","request_tags":["User-Agent: pydantic-ai","User-Agent: pydantic-ai/2.53.0"],"metadata":"{\"fixture_capture\":{\"name\":\"pydantic_ai_retry\",\"trace_id\":\"213790ddf7f01594d52d54818775b98d\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1EcoG4Hwa0tTYV7gLus0fwAJ7wY\", \"created\": 1791061622, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces reveal the steps an agent takes to complete a task.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 53, \"prompt_tokens\": 15, \"total_tokens\": 68, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 31, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061622557,"end_time":1791061623931,"completion_start_time":1791061623931}
+{"request_id":"chatcmpl-EV1Ee6ZsIx2AYZLIIcwgTLCt8rYAp","response_id":"chatcmpl-EV1Ee6ZsIx2AYZLIIcwgTLCt8rYAp","litellm_call_id":"b1b0392c-9022-4369-88ce-7d8bdc57c90c","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.55e-05,"prompt_tokens":15,"completion_tokens":48,"total_tokens":63,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"213790ddf7f01594d52d54818775b98d","trace_id":"213790ddf7f01594d52d54818775b98d","span_id":"c80b675f9912b72c","request_tags":["User-Agent: pydantic-ai","User-Agent: pydantic-ai/2.53.0"],"metadata":"{\"fixture_capture\":{\"name\":\"pydantic_ai_retry\",\"trace_id\":\"213790ddf7f01594d52d54818775b98d\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1Ee6ZsIx2AYZLIIcwgTLCt8rYAp\", \"created\": 1791061624, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s actions and decisions over time.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 48, \"prompt_tokens\": 15, \"total_tokens\": 63, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 27, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061624398,"end_time":1791061625775,"completion_start_time":1791061625775}

litellm-rust/crates/traces-clickhouse/tests/fixtures/pydantic_ai_stream_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1CVbDkWDEsratBuf9bhvEG3Av51","response_id":"chatcmpl-EV1CVbDkWDEsratBuf9bhvEG3Av51","litellm_call_id":"94605f85-b0da-4596-8538-9072fce7119e","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":0.0001117,"prompt_tokens":12,"completion_tokens":221,"total_tokens":233,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"f3f8852c4f360b03ce2a36e6d3ed6f51","trace_id":"f3f8852c4f360b03ce2a36e6d3ed6f51","span_id":"a383ecf3f8beb7a6","request_tags":["User-Agent: pydantic-ai","User-Agent: pydantic-ai/2.53.0"],"metadata":"{\"fixture_capture\":{\"name\":\"pydantic_ai_stream\",\"trace_id\":\"f3f8852c4f360b03ce2a36e6d3ed6f51\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"What is an agent trace?\"}]","response":"{\"id\": \"chatcmpl-EV1CVbDkWDEsratBuf9bhvEG3Av51\", \"created\": 1791061493, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"An **agent trace** is a chronological record of an AI agent\\u2019s activity while completing a task. It may include:\\n\\n- The task or input it received\\n- Actions it took, such as tool calls\\n- Results or observations it received\\n- Changes in state or decisions\\n- The final response or outcome\\n\\nFor example: *User asks for the weather \\u2192 agent calls a weather API \\u2192 API returns the forecast \\u2192 agent summarizes it.*\\n\\nA trace helps people debug or evaluate an agent. It doesn\\u2019t necessarily include the agent\\u2019s private reasoning; often it records only observable steps and tool interactions.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 221, \"prompt_tokens\": 12, \"total_tokens\": 233, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 93, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 0.0001117}, \"service_tier\": \"default\"}","start_time":1791061491444,"end_time":1791061494417,"completion_start_time":1791061493102}

litellm-rust/crates/traces-clickhouse/tests/fixtures/pydantic_ai_token_limit_swarm_spend_logs.jsonl

@@ -0,0 +1,2 @@
+{"request_id":"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDplNjhmYmQxY2UyNmFhN2E4OWIwNTljOWE2ZGYxMjAwN2U2MjRjNmEyMTA2Y2QzZTZhZjQ5MmFiN2EyNTlmMTE3O3Jlc3BvbnNlX2lkOnJlc3BfMDE5OTdjMDdlZDQ3NzkzNTAwNmFjMTVhMjVmNzc4ODdkMGFjNjQzZDM1M2E2Zjc1MmI=","response_id":"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDplNjhmYmQxY2UyNmFhN2E4OWIwNTljOWE2ZGYxMjAwN2U2MjRjNmEyMTA2Y2QzZTZhZjQ5MmFiN2EyNTlmMTE3O3Jlc3BvbnNlX2lkOnJlc3BfMDE5OTdjMDdlZDQ3NzkzNTAwNmFjMTVhMjVmNzc4ODdkMGFjNjQzZDM1M2E2Zjc1MmI=","litellm_call_id":"274a6199-ce75-4cbd-a1c8-a09f18b85de3","call_type":"responses","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1/responses","spend":2.18e-05,"prompt_tokens":73,"completion_tokens":29,"total_tokens":102,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"58534afa-54a4-48d3-be19-a4882953d3c6","trace_id":"","span_id":"","request_tags":["User-Agent: pydantic-ai","User-Agent: pydantic-ai/2.53.0"],"metadata":"{\"fixture_capture\":{\"name\":\"pydantic_ai_token_limit_swarm\",\"trace_id\":\"33551f56e73710cee91fb7beb1be6cc3\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"Call search first, then write with the facts, and return the written answer.\"}, {\"role\": \"user\", \"content\": \"What is an agent trace?\"}]","response":"{\"id\": \"resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDplNjhmYmQxY2UyNmFhN2E4OWIwNTljOWE2ZGYxMjAwN2U2MjRjNmEyMTA2Y2QzZTZhZjQ5MmFiN2EyNTlmMTE3O3Jlc3BvbnNlX2lkOnJlc3BfMDE5OTdjMDdlZDQ3NzkzNTAwNmFjMTVhMjVmNzc4ODdkMGFjNjQzZDM1M2E2Zjc1MmI=\", \"created_at\": 1791056421, \"error\": null, \"incomplete_details\": null, \"instructions\": \"Call search first, then write with the facts, and return the written answer.\", \"metadata\": {}, \"model\": \"gpt-6-luna\", \"object\": \"response\", \"output\": [{\"arguments\": \"{\\\"query\\\":\\\"definition of agent trace in AI agents execution trace tool calls observations steps\\\"}\", \"call_id\": \"call_hILz5R2pcOQ6GLcp907fCVeQ\", \"name\": \"search\", \"type\": \"function_call\", \"id\": \"fc_01997c07ed477935006ac15a267d9887d0adb7c782b1832764\", \"namespace\": null, \"status\": \"completed\"}], \"parallel_tool_calls\": true, \"temperature\": 1.0, \"tool_choice\": \"auto\", \"tools\": [{\"name\": \"search\", \"parameters\": {\"additionalProperties\": false, \"properties\": {\"query\": {\"type\": \"string\"}}, \"required\": [\"query\"], \"type\": \"object\"}, \"strict\": true, \"type\": \"function\", \"defer_loading\": null, \"description\": null, \"output_schema\": null}, {\"name\": \"write\", \"parameters\": {\"additionalProperties\": false, \"properties\": {\"facts\": {\"type\": \"string\"}}, \"required\": [\"facts\"], \"type\": \"object\"}, \"strict\": true, \"type\": \"function\", \"defer_loading\": null, \"description\": null, \"output_schema\": null}], \"top_p\": 0.98, \"max_output_tokens\": 256, \"previous_response_id\": null, \"reasoning\": {\"context\": \"all_turns\", \"effort\": \"medium\", \"mode\": \"standard\", \"summary\": null}, \"status\": \"completed\", \"text\": {\"format\": {\"type\": \"text\"}, \"verbosity\": \"medium\"}, \"truncation\": \"disabled\", \"usage\": {\"completion_tokens\": 29, \"prompt_tokens\": 73, \"total_tokens\": 102, \"completion_tokens_details\": {\"accepted_prediction_tokens\": null, \"audio_tokens\": null, \"reasoning_tokens\": 0, \"rejected_prediction_tokens\": null, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": null, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"user\": null, \"store\": true, \"access_programs\": {\"cyber\": \"daybreak_blue\"}, \"background\": false, \"billing\": {\"payer\": \"developer\"}, \"completed_at\": 1791056423, \"frequency_penalty\": 0.0, \"max_tool_calls\": null, \"moderation\": null, \"presence_penalty\": 0.0, \"prompt_cache_key\": null, \"prompt_cache_retention\": \"24h\", \"safety_identifier\": null, \"service_tier\": \"default\", \"tool_usage\": {\"image_gen\": {\"input_tokens\": 0, \"input_tokens_details\": {\"image_tokens\": 0, \"text_tokens\": 0}, \"output_tokens\": 0, \"output_tokens_details\": {\"image_tokens\": 0, \"text_tokens\": 0}, \"total_tokens\": 0}, \"web_search\": {\"num_requests\": 0}}, \"top_logprobs\": 0}","start_time":1791056421823,"end_time":1791056425435,"completion_start_time":1791056425435}
+{"request_id":"chatcmpl-EUzsnFpH3W2H1tbWvpBypOpEPupRA","response_id":"chatcmpl-EUzsnFpH3W2H1tbWvpBypOpEPupRA","litellm_call_id":"790f284d-7a98-4550-99ab-7374965c8365","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":0.00013099999999999999,"prompt_tokens":30,"completion_tokens":256,"total_tokens":286,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"71f6c352-9a97-47c2-89ed-9149c035b978","trace_id":"","span_id":"","request_tags":["User-Agent: pydantic-ai","User-Agent: pydantic-ai/2.53.0"],"metadata":"{\"fixture_capture\":{\"name\":\"pydantic_ai_token_limit_swarm\",\"trace_id\":\"33551f56e73710cee91fb7beb1be6cc3\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"system\", \"content\": \"Find key facts about the topic.\"}, {\"role\": \"user\", \"content\": \"definition of agent trace in AI agents execution trace tool calls observations steps\"}]","response":"{\"id\": \"chatcmpl-EUzsnFpH3W2H1tbWvpBypOpEPupRA\", \"created\": 1791056425, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"length\", \"index\": 0, \"message\": {\"content\": \"\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 256, \"prompt_tokens\": 30, \"total_tokens\": 286, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 256, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791056425493,"end_time":1791056428619,"completion_start_time":1791056428619}

litellm-rust/crates/traces-clickhouse/tests/fixtures/strands_billed_failure_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1Bc8dQIuPElIn5vaTQdxXaUzAiX","response_id":"chatcmpl-EV1Bc8dQIuPElIn5vaTQdxXaUzAiX","litellm_call_id":"c958e5e9-a1d6-411c-b070-8f5aab06de79","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":1.92e-05,"prompt_tokens":112,"completion_tokens":16,"total_tokens":128,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"951c8f6dbe7350974f563de3f282b3ac","trace_id":"951c8f6dbe7350974f563de3f282b3ac","span_id":"b6c34c8c5588fdf5","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"strands_billed_failure\",\"trace_id\":\"951c8f6dbe7350974f563de3f282b3ac\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": [{\"text\": \"Reply with one short sentence about agent traces.\", \"type\": \"text\"}]}]","response":"{\"id\": \"chatcmpl-EV1Bc8dQIuPElIn5vaTQdxXaUzAiX\", \"created\": 1791061436, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces show the steps an agent takes to complete a task.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 16, \"prompt_tokens\": 112, \"total_tokens\": 128, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 0, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 1.92e-05}, \"service_tier\": \"default\"}","start_time":1791061436369,"end_time":1791061437050,"completion_start_time":1791061436941}

litellm-rust/crates/traces-clickhouse/tests/fixtures/strands_retry_spend_logs.jsonl

@@ -0,0 +1,2 @@
+{"request_id":"chatcmpl-EV1BGShr9D1AfRCWu7DGwLNROCcTT","response_id":"chatcmpl-EV1BGShr9D1AfRCWu7DGwLNROCcTT","litellm_call_id":"cc203841-e4f8-42da-bd36-cdbe2bd9a4c0","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":1.9699999999999998e-05,"prompt_tokens":112,"completion_tokens":17,"total_tokens":129,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"d1eb10a194f225edbe397b5d11775b00","trace_id":"d1eb10a194f225edbe397b5d11775b00","span_id":"8364872d770021af","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"strands_retry\",\"trace_id\":\"d1eb10a194f225edbe397b5d11775b00\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": [{\"text\": \"Reply with one short sentence about agent traces.\", \"type\": \"text\"}]}]","response":"{\"id\": \"chatcmpl-EV1BGShr9D1AfRCWu7DGwLNROCcTT\", \"created\": 1791061414, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s actions, decisions, and tool interactions.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 17, \"prompt_tokens\": 112, \"total_tokens\": 129, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 0, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 1.9699999999999998e-05}, \"service_tier\": \"default\"}","start_time":1791061413879,"end_time":1791061414741,"completion_start_time":1791061414584}
+{"request_id":"chatcmpl-EV1BHJSXrImjpnUFx4dnRWjUmC9jV","response_id":"chatcmpl-EV1BHJSXrImjpnUFx4dnRWjUmC9jV","litellm_call_id":"400e4cf8-4aa5-4c4a-b998-b6f9aa175b56","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":1.92e-05,"prompt_tokens":112,"completion_tokens":16,"total_tokens":128,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"d1eb10a194f225edbe397b5d11775b00","trace_id":"d1eb10a194f225edbe397b5d11775b00","span_id":"99aad174e55e03b0","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 2.54.0"],"metadata":"{\"fixture_capture\":{\"name\":\"strands_retry\",\"trace_id\":\"d1eb10a194f225edbe397b5d11775b00\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": [{\"text\": \"Reply with one short sentence about agent traces.\", \"type\": \"text\"}]}]","response":"{\"id\": \"chatcmpl-EV1BHJSXrImjpnUFx4dnRWjUmC9jV\", \"created\": 1791061416, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces reveal the steps an AI takes to complete a task.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 16, \"prompt_tokens\": 112, \"total_tokens\": 128, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 0, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 1.92e-05}, \"service_tier\": \"default\"}","start_time":1791061415156,"end_time":1791061416292,"completion_start_time":1791061416186}

litellm-rust/crates/traces-clickhouse/tests/fixtures/vercel_ai_sdk_billed_failure_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1Be4YkRbpmqEp1VHkrj1vEGeerd","response_id":"chatcmpl-EV1Be4YkRbpmqEp1VHkrj1vEGeerd","litellm_call_id":"5d4283ed-39e1-4530-9200-24585098215f","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.1499999999999997e-05,"prompt_tokens":15,"completion_tokens":40,"total_tokens":55,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"da0164cd74393d84076fef5253c18441","trace_id":"da0164cd74393d84076fef5253c18441","span_id":"5a53a9fc62fda14c","request_tags":["User-Agent: ai-sdk-openai-compatible","User-Agent: ai-sdk-openai-compatible/3.0.62 ai-sdk-provider-utils/5.0.53 node.js/25"],"metadata":"{\"fixture_capture\":{\"name\":\"vercel_ai_sdk_billed_failure\",\"trace_id\":\"da0164cd74393d84076fef5253c18441\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1Be4YkRbpmqEp1VHkrj1vEGeerd\", \"created\": 1791061439, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s actions, decisions, and tool calls.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 40, \"prompt_tokens\": 15, \"total_tokens\": 55, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 17, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 2.1499999999999997e-05}, \"service_tier\": \"default\"}","start_time":1791061438594,"end_time":1791061439640,"completion_start_time":1791061439421}

litellm-rust/crates/traces-clickhouse/tests/fixtures/vercel_ai_sdk_py_simple_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV19oQg42x1rA1HfR9Biuy5ImkI7s","response_id":"chatcmpl-EV19oQg42x1rA1HfR9Biuy5ImkI7s","litellm_call_id":"2e7ef64f-35cd-432d-a684-14214cd9dd2d","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":9.82e-05,"prompt_tokens":12,"completion_tokens":194,"total_tokens":206,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"5b5f03a176b0b4fb74290922de689a21","trace_id":"5b5f03a176b0b4fb74290922de689a21","span_id":"6f8e118ee0c01d1a","request_tags":["User-Agent: AsyncOpenAI","User-Agent: AsyncOpenAI/Python 3.24.0"],"metadata":"{\"fixture_capture\":{\"name\":\"vercel_ai_sdk_py_simple\",\"trace_id\":\"5b5f03a176b0b4fb74290922de689a21\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"What is an agent trace?\"}]","response":"{\"id\": \"chatcmpl-EV19oQg42x1rA1HfR9Biuy5ImkI7s\", \"created\": 1791061325, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did while completing a task. It may include the user\\u2019s request, the agent\\u2019s actions and tool calls, the results it received, and its final response.\\n\\nFor example:\\n\\n1. User asks for the weather.\\n2. Agent calls a weather tool.\\n3. Tool returns the forecast.\\n4. Agent summarizes it for the user.\\n\\nTraces are useful for debugging, evaluating, and auditing an agent. The exact contents vary by system; a trace usually records observable steps and results, not necessarily the model\\u2019s private internal reasoning.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 194, \"prompt_tokens\": 12, \"total_tokens\": 206, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 65, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 9.82e-05}, \"service_tier\": \"default\"}","start_time":1791061324025,"end_time":1791061327041,"completion_start_time":1791061325560}

litellm-rust/crates/traces-clickhouse/tests/fixtures/vercel_ai_sdk_retry_spend_logs.jsonl

@@ -0,0 +1,2 @@
+{"request_id":"chatcmpl-EV1BG2mxTwq38bkzLa9xWiE472RFi","response_id":"chatcmpl-EV1BG2mxTwq38bkzLa9xWiE472RFi","litellm_call_id":"cd72f919-eb8e-4854-8c74-1a2d22fbe271","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":3.35e-05,"prompt_tokens":15,"completion_tokens":64,"total_tokens":79,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"18b74e8029c4c62d6ee2acbfaccd05d7","trace_id":"18b74e8029c4c62d6ee2acbfaccd05d7","span_id":"c2bdb1aefaca871b","request_tags":["User-Agent: ai","User-Agent: ai/7.0.127 ai-sdk-provider-utils/5.0.53 node.js/25"],"metadata":"{\"fixture_capture\":{\"name\":\"vercel_ai_sdk_retry\",\"trace_id\":\"18b74e8029c4c62d6ee2acbfaccd05d7\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1BG2mxTwq38bkzLa9xWiE472RFi\", \"created\": 1791061414, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s steps, decisions, and tool calls.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 64, \"prompt_tokens\": 15, \"total_tokens\": 79, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 41, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061414000,"end_time":1791061415440,"completion_start_time":1791061415440}
+{"request_id":"chatcmpl-EV1BJQ4BvQRI5X8f1m4LZ7uTX4sKt","response_id":"chatcmpl-EV1BJQ4BvQRI5X8f1m4LZ7uTX4sKt","litellm_call_id":"13406f69-913c-4ed9-87fd-03785496f128","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":2.2499999999999998e-05,"prompt_tokens":15,"completion_tokens":42,"total_tokens":57,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"18b74e8029c4c62d6ee2acbfaccd05d7","trace_id":"18b74e8029c4c62d6ee2acbfaccd05d7","span_id":"75cf929b3549bf74","request_tags":["User-Agent: ai","User-Agent: ai/7.0.127 ai-sdk-provider-utils/5.0.53 node.js/25"],"metadata":"{\"fixture_capture\":{\"name\":\"vercel_ai_sdk_retry\",\"trace_id\":\"18b74e8029c4c62d6ee2acbfaccd05d7\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1BJQ4BvQRI5X8f1m4LZ7uTX4sKt\", \"created\": 1791061417, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces record an agent\\u2019s actions, decisions, and tool calls over time.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null, \"provider_specific_fields\": {\"refusal\": null}, \"annotations\": []}, \"provider_specific_fields\": {}}], \"usage\": {\"completion_tokens\": 42, \"prompt_tokens\": 15, \"total_tokens\": 57, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 17, \"rejected_prediction_tokens\": 0}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}}, \"service_tier\": \"default\"}","start_time":1791061417451,"end_time":1791061418708,"completion_start_time":1791061418708}

litellm-rust/crates/traces-clickhouse/tests/fixtures/vercel_ai_sdk_stream_spend_logs.jsonl

@@ -0,0 +1 @@
+{"request_id":"chatcmpl-EV1AyMCFfzO4g0UVvSESWJ3Q5rFZK","response_id":"chatcmpl-EV1AyMCFfzO4g0UVvSESWJ3Q5rFZK","litellm_call_id":"d269912a-2203-4e1f-8fd8-640a09f9cb40","call_type":"acompletion","api_key":"fixture-key","key_alias":"","team_id":"fixture-team","team_alias":"","organization_id":"","user":"fixture-user","end_user":"","model":"openai/gpt-6-luna","model_group":"openai/gpt-6-luna","model_id":"e68fbd1ce26aa7a89b059c9a6df12007e624c6a2106cd3e6af492ab7a259f117","custom_llm_provider":"openai","api_base":"https://api.openai.com/v1","spend":9.5e-06,"prompt_tokens":15,"completion_tokens":16,"total_tokens":31,"cache_read_tokens":0,"cache_write_tokens":0,"status":"success","error_str":"","cache_hit":false,"session_id":"2acad168d77811f8fa89100e5e1fc0e7","trace_id":"2acad168d77811f8fa89100e5e1fc0e7","span_id":"a08a06551dfef63e","request_tags":["User-Agent: ai-sdk-openai-compatible","User-Agent: ai-sdk-openai-compatible/3.0.62 ai-sdk-provider-utils/5.0.53 node.js/25"],"metadata":"{\"fixture_capture\":{\"name\":\"vercel_ai_sdk_stream\",\"trace_id\":\"2acad168d77811f8fa89100e5e1fc0e7\",\"spend_linked\":true,\"spend_complete\":true}}","messages":"[{\"role\": \"user\", \"content\": \"Reply with one short sentence about agent traces.\"}]","response":"{\"id\": \"chatcmpl-EV1AyMCFfzO4g0UVvSESWJ3Q5rFZK\", \"created\": 1791061396, \"model\": \"openai/gpt-6-luna\", \"object\": \"chat.completion\", \"system_fingerprint\": null, \"choices\": [{\"finish_reason\": \"stop\", \"index\": 0, \"message\": {\"content\": \"Agent traces show the steps an agent took to complete a task.\", \"role\": \"assistant\", \"tool_calls\": null, \"function_call\": null}}], \"usage\": {\"completion_tokens\": 16, \"prompt_tokens\": 15, \"total_tokens\": 31, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 0, \"rejected_prediction_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cached_tokens\": 0, \"text_tokens\": null, \"image_tokens\": null, \"video_tokens\": null, \"cache_write_tokens\": 0, \"cache_creation_tokens\": 0}, \"cost\": 9.5e-06}, \"service_tier\": \"default\"}","start_time":1791061396267,"end_time":1791061397062,"completion_start_time":1791061396960}

litellm-rust/crates/traces-clickhouse/tests/migrations.rs

@@ -2132,7 +2132,7 @@ async fn nullable_spend_upgrade_preserves_existing_costs_and_unknown_new_costs(
     let writer = Connection::writer(&database.url)?;
     let timestamp = (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as i64;
     let statements = schema_statements("trace_test", 7)?;
-    for statement in &statements[..statements.len() - 1] {
+    for statement in &statements[..14] {
         execute_write(&database, statement).await?;
     }
     let legacy = serde_json::from_value(serde_json::json!({
@@ -2174,3 +2174,40 @@ async fn nullable_spend_upgrade_preserves_existing_costs_and_unknown_new_costs(
     );
     Ok(())
 }
+
+#[rstest]
+#[tokio::test]
+async fn gateway_id_upgrade_preserves_legacy_rows_and_accepts_new_ids(
+    #[future(awt)] database: TestResult<ClickHouseDatabase>,
+) -> TestResult {
+    let database = database?;
+    let writer = Connection::writer(&database.url)?;
+    let timestamp = (time::OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as i64;
+    let statements = schema_statements("trace_test", 7)?;
+    for statement in &statements[..15] {
+        execute_write(&database, statement).await?;
+    }
+    let legacy = serde_json::from_value(serde_json::json!({
+        "request_id": "legacy", "response_id": "response", "spend": 0.25,
+        "start_time": timestamp, "end_time": timestamp + 100
+    }))?;
+    insert_rows(&database, "spend_logs", vec![legacy]).await?;
+    ensure_schema(&database.client, &writer, "trace_test", 7).await?;
+    ensure_schema(&database.client, &writer, "trace_test", 7).await?;
+    let current = serde_json::from_value(serde_json::json!({
+        "request_id": "current", "response_id": "response", "litellm_call_id": "gateway",
+        "spend": null, "start_time": timestamp, "end_time": timestamp + 100
+    }))?;
+    insert_rows(&database, "spend_logs", vec![current]).await?;
+    let result = read_json(&database,
+        "SELECT request_id, litellm_call_id, spend FROM trace_test.spend_logs FINAL ORDER BY request_id"
+    ).await?;
+    assert_eq!(
+        result["data"],
+        serde_json::json!([
+            {"request_id": "current", "litellm_call_id": "gateway", "spend": null},
+            {"request_id": "legacy", "litellm_call_id": "", "spend": 0.25},
+        ])
+    );
+    Ok(())
+}

litellm-rust/crates/traces-clickhouse/tests/reads.rs

@@ -47,6 +47,7 @@ async fn list_costs_match_each_run_when_response_ids_are_reused(
                     ("UserId".into(), json!(user_id)),
                     ("Duration".into(), json!(1_000_000)),
                     ("LiteLLMRequestId".into(), json!("reused-response")),
+                    ("CallEvidence".into(), json!("complete")),
                 ])
             })
             .collect(),
@@ -152,6 +153,14 @@ async fn large_runs_remain_complete_under_default_reader_limits(
                     ("TeamId".into(), json!("team-a")),
                     ("ApiKeyHash".into(), json!("key-a")),
                     ("Duration".into(), json!(1000)),
+                    (
+                        "CallEvidence".into(),
+                        json!(if costed && step > 0 {
+                            "complete"
+                        } else {
+                            "unknown"
+                        }),
+                    ),
                     (
                         "LiteLLMRequestId".into(),
                         json!(if costed && step > 0 {
@@ -593,3 +602,111 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals(
     ));
     Ok(())
 }
+
+#[rstest]
+#[tokio::test]
+async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback(
+    #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+) -> TestResult {
+    let fixture = migrated_database?;
+    let client = &fixture.database.client;
+    let writer = Connection::writer(&fixture.database.url)?;
+    let start_ms = 1_790_000_000_000_i64;
+    let cases = [
+        (
+            "gateway",
+            "provider-request",
+            "gateway",
+            "team-a",
+            "key-a",
+            Some(0.25),
+        ),
+        ("legacy", "legacy", "", "team-a", "key-a", Some(0.25)),
+        ("conflict", "conflict", "different", "team-a", "key-a", None),
+        (
+            "foreign-team",
+            "request",
+            "foreign-team",
+            "team-b",
+            "key-a",
+            None,
+        ),
+        (
+            "foreign-key",
+            "request",
+            "foreign-key",
+            "team-a",
+            "key-b",
+            None,
+        ),
+    ];
+    insert_rows(
+        client,
+        &writer,
+        DATABASE,
+        InsertTable::OtelTraces,
+        cases
+            .iter()
+            .map(|(id, _, _, _, _, _)| {
+                BTreeMap::from([
+                    ("Timestamp".into(), json!(start_ms * 1_000_000)),
+                    ("Duration".into(), json!(1_000_000)),
+                    ("TraceId".into(), json!(id)),
+                    ("SpanId".into(), json!("call")),
+                    ("ObservationType".into(), json!("llm")),
+                    ("TeamId".into(), json!("team-a")),
+                    ("ApiKeyHash".into(), json!("key-a")),
+                    ("CallKeys".into(), json!([format!("litellm_request:{id}")])),
+                    ("CallEvidence".into(), json!("complete")),
+                ])
+            })
+            .collect(),
+    )
+    .await?;
+    insert_rows(
+        client,
+        &writer,
+        DATABASE,
+        InsertTable::SpendLogs,
+        cases
+            .iter()
+            .map(|(_, request, call_id, team, key, _)| {
+                BTreeMap::from([
+                    ("request_id".into(), json!(request)),
+                    ("response_id".into(), json!("provider-response")),
+                    ("litellm_call_id".into(), json!(call_id)),
+                    ("team_id".into(), json!(team)),
+                    ("api_key".into(), json!(key)),
+                    ("start_time".into(), json!(start_ms)),
+                    ("end_time".into(), json!(start_ms + 1)),
+                    ("spend".into(), json!(0.25)),
+                ])
+            })
+            .collect(),
+    )
+    .await?;
+    let reader = fixture
+        .readers
+        .connection(client, &QueryScope::All, "fixture-secret")
+        .await?;
+    let access = ReadAccessParams {
+        all_teams: false,
+        user_id: String::new(),
+        team_ids: vec!["team-a".into()],
+    };
+    let page = list_traces(client, &reader, &access, 0, 2_000_000_000_000, None, 50).await?;
+    assert_eq!(page.data.len(), cases.len());
+    for (id, _, _, _, _, expected) in cases {
+        let summary = page
+            .data
+            .iter()
+            .find(|summary| summary.trace_id == id)
+            .ok_or("missing run")?;
+        let detail = get_trace(client, &reader, &access, id, &summary.trace_ref)
+            .await?
+            .ok_or("missing trace")?;
+        assert_eq!(detail.summary.spend, expected, "{id}");
+        assert_eq!(summary.spend, expected, "{id}");
+    }
+    Ok(())
+}

litellm-rust/crates/traces-clickhouse/tests/span_rows.rs

@@ -214,3 +214,25 @@ fn absent_identity_fields_are_empty_only_in_storage(tenant: Tenant) {
     assert_eq!(row["CallKeys"], json!([]));
     assert_eq!(row["CallEvidence"], "unknown");
 }
+
+#[rstest]
+fn compatibility_id_keeps_provider_semantics_with_gateway_keys(tenant: Tenant) {
+    let body = export(vec![(
+        vec![],
+        vec![span(
+            &"02".repeat(8),
+            vec![
+                attribute("gen_ai.response.id", "response"),
+                attribute("litellm.call_id", "gateway"),
+            ],
+            json!({}),
+        )],
+    )]);
+    let stored = rows(&body, &tenant, MAX_VALUE_BYTES);
+    assert_eq!(stored[0]["LiteLLMRequestId"], "response");
+    assert_eq!(stored[0]["CallEvidence"], "partial");
+    assert_eq!(
+        stored[0]["CallKeys"],
+        json!(["litellm_request:gateway", "provider_response:response"])
+    );
+}

litellm-rust/crates/traces/Cargo.toml

@@ -23,6 +23,7 @@ thiserror.workspace = true
 time.workspace = true
 
 [dev-dependencies]
+base64.workspace = true
 criterion.workspace = true
 rstest.workspace = true
 

litellm-rust/crates/traces/src/normalize/format/genai.rs

@@ -2,8 +2,8 @@ use super::{Extraction, Format, Payload, SpanFacts};
 use crate::{
     Error,
     normalize::{
-        ObservationType, RoleEvidence, SpanContext, attr, messages, present, select_attribute,
-        usage_tokens,
+        CallEvidence, CallKey, ObservationType, RoleEvidence, SpanContext, attr, messages, present,
+        select_attribute, usage_tokens,
     },
 };
 
@@ -107,10 +107,17 @@ impl Format for GenAi {
         let (input_tokens, output_tokens) = usage_tokens(attributes)?;
         let input = payload(context, &INPUT_KEYS);
         let output = payload(context, &OUTPUT_KEYS);
+        let role = Operation::from_context(context).map(Operation::role);
+        let calls = match (role, present(attributes, &["gen_ai.response.id"])) {
+            (Some(ObservationType::Llm), Some(id)) => {
+                CallEvidence::complete(CallKey::ProviderResponse(id))
+            }
+            _ => CallEvidence::Unknown,
+        };
         Ok(Extraction {
             facts: SpanFacts {
-                role: Operation::from_context(context)
-                    .map(|operation| RoleEvidence::Declared(operation.role())),
+                role: role.map(RoleEvidence::Declared),
+                calls,
                 model: present(
                     attributes,
                     &["gen_ai.request.model", "gen_ai.response.model"],

litellm-rust/crates/traces/src/normalize/format/langsmith.rs

@@ -179,25 +179,52 @@ impl Format for LangSmith {
         let observation_type = ObservationType::try_from(attr(attributes, "langsmith.span.kind"))
             .unwrap_or(ObservationType::Chain);
         let io = span_io(observation_type, attributes);
+        let legacy_input = !attr(attributes, "gen_ai.prompt").is_empty();
+        let legacy_output = !attr(attributes, "gen_ai.completion").is_empty();
         Ok(Extraction {
             facts: SpanFacts {
                 role: Some(RoleEvidence::Declared(observation_type)),
-                input: if attr(attributes, "gen_ai.prompt").is_empty() {
-                    String::new()
-                } else {
+                input: if legacy_input {
                     io.input
-                },
-                output: if attr(attributes, "gen_ai.completion").is_empty() {
-                    String::new()
                 } else {
-                    io.output
+                    base.facts.input
                 },
-                calls: io.calls,
-                ..SpanFacts::default()
-            }
-            .or(base.facts),
+                output: if legacy_output {
+                    io.output
+                } else {
+                    base.facts.output
+                },
+                calls: match io.calls {
+                    CallEvidence::Unknown => base.facts.calls,
+                    calls => calls,
+                },
+                ..base.facts
+            },
             display_name: None,
-            consumed_attributes: base.consumed_attributes,
+            consumed_attributes: base
+                .consumed_attributes
+                .into_iter()
+                .filter(|source| {
+                    !(legacy_input
+                        && matches!(
+                            *source,
+                            "gen_ai.input.messages"
+                                | "gen_ai.tool.call.arguments"
+                                | "gen_ai.retrieval.query.text"
+                                | "gen_ai.prompt"
+                        ))
+                        && !(legacy_output
+                            && matches!(
+                                *source,
+                                "gen_ai.output.messages"
+                                    | "gen_ai.tool.call.result"
+                                    | "gen_ai.retrieval.documents"
+                                    | "gen_ai.completion"
+                            ))
+                })
+                .chain(legacy_input.then_some("gen_ai.prompt"))
+                .chain(legacy_output.then_some("gen_ai.completion"))
+                .collect(),
         })
     }
 }

litellm-rust/crates/traces/src/normalize/instrumentation/http_client.rs

@@ -13,6 +13,16 @@ const SCOPES: [&str; 7] = [
 
 pub(super) fn matches(context: &SpanContext<'_>) -> bool {
     SCOPES.contains(&context.scope)
+        || (context.scope == "litellm.gateway.client"
+            && context.name == "gateway.request"
+            && context
+                .attributes
+                .get("litellm.gateway.attempt")
+                .is_some_and(|value| value == "true")
+            && context
+                .attributes
+                .get("http.request.method")
+                .is_some_and(|value| value == "POST"))
 }
 
 pub(super) fn adjust(facts: SpanFacts) -> SpanFacts {

litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs

@@ -147,10 +147,8 @@ impl Instrumentation {
             facts,
             display_name,
             consumed_attributes,
-        } = self.adjust(
-            context,
-            prepared.map_facts(|facts| with_response_id(context, facts)),
-        );
+        } = self.adjust(context, prepared);
+        let facts = with_call_ids(context, facts);
         let role = match (facts.role, metadata.ls_agent_type) {
             (
                 None
@@ -216,15 +214,15 @@ impl Instrumentation {
     }
 }
 
-/// `gen_ai.response.id` names one provider response, whichever convention recorded it.
-fn with_response_id(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts {
-    match present(context.attributes, &["gen_ai.response.id"]) {
-        Some(id) => SpanFacts {
-            calls: facts.calls.with(CallKey::ProviderResponse(id)),
-            ..facts
-        },
-        None => facts,
-    }
+fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts {
+    let calls = [
+        present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse),
+        present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest),
+    ]
+    .into_iter()
+    .flatten()
+    .fold(facts.calls, CallEvidence::with);
+    SpanFacts { calls, ..facts }
 }
 
 fn recorded_agent_name(

litellm-rust/crates/traces/src/normalize/mod.rs

@@ -47,7 +47,7 @@ pub enum ObservationType {
 #[derive(Clone, Debug, Deserialize, Eq, Ord, PartialEq, PartialOrd)]
 #[serde(try_from = "String")]
 pub enum CallKey {
-    /// LiteLLM's own id for the request (`spend_logs.request_id`).
+    /// LiteLLM's gateway call id, with a fallback to legacy spend request ids.
     LiteLlmRequest(String),
     /// The provider response id returned to the caller (`spend_logs.response_id`).
     ProviderResponse(String),
@@ -126,7 +126,7 @@ impl CallEvidence {
     pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self {
         let kind = row
             .call_evidence
-            .unwrap_or(if row.litellm_request_id.is_empty() {
+            .unwrap_or(if Self::row_keys(row).is_empty() {
                 CallEvidenceKind::Unknown
             } else {
                 CallEvidenceKind::Complete
@@ -146,7 +146,7 @@ impl CallEvidence {
     /// convention found, but says nothing about completeness.
     fn with(self, key: CallKey) -> Self {
         match self {
-            Self::Unknown => Self::complete(key),
+            Self::Unknown => Self::Partial(BTreeSet::from([key])),
             Self::Partial(keys) => Self::Partial(keys.into_iter().chain([key]).collect()),
             Self::Complete(keys) => Self::Complete(keys.into_iter().chain([key]).collect()),
         }

litellm-rust/crates/traces/src/query/named.rs

@@ -64,7 +64,7 @@ pub struct TraceSpansParams {
     pub trace_ref: String,
 }
 
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct TraceSpansRow {
     #[serde(default)]
     pub trace_id: String,
@@ -172,6 +172,7 @@ pub struct SpendByResponseIdsParams {
 #[derive(Debug, Deserialize, Serialize)]
 pub struct SpendByResponseIdsRow {
     pub request_id: String,
+    pub litellm_call_id: String,
     pub response_id: String,
     pub upstream_response_id: String,
     pub trace_id: String,
@@ -183,6 +184,12 @@ pub struct SpendByResponseIdsRow {
     pub start_ms: i64,
 }
 
+impl SpendByResponseIdsRow {
+    pub(crate) fn identity(&self) -> (&str, i64, &str) {
+        (&self.team_id, self.start_ms, &self.request_id)
+    }
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct TraceIdentityParams {
     #[serde(flatten)]

litellm-rust/crates/traces/src/resolve/graph.rs

@@ -46,6 +46,13 @@ impl<'a> Graph<'a> {
         parent.is_empty() || !self.by_id.contains_key(parent.as_str())
     }
 
+    pub(super) fn children(&self, index: usize) -> Vec<usize> {
+        self.children
+            .get(self.id(index))
+            .map(|children| children.to_vec())
+            .unwrap_or_default()
+    }
+
     pub(super) fn ancestors(&self, index: usize) -> Vec<usize> {
         let mut seen = HashSet::from([self.id(index)]);
         let mut found = Vec::new();

litellm-rust/crates/traces/src/resolve/resolution.rs

@@ -102,14 +102,8 @@ impl<'a> Resolution<'a> {
             .map(|source| self.requests(source))
             .collect();
         let transports: Vec<_> = self
-            .graph
-            .descendants(call)
+            .transports(call)
             .into_iter()
-            .filter(|descendant| {
-                self.row(*descendant)
-                    .call_keys
-                    .contains(&CallKey::Transport)
-            })
             .map(|transport| self.requests(transport))
             .collect();
         let transport_requests: Option<Vec<Requests<'a>>> = (!transports.is_empty())
@@ -133,13 +127,51 @@ impl<'a> Resolution<'a> {
         Some(
             selected
                 .into_iter()
-                .map(|request| (request.request_id.as_str(), request))
+                .map(|request| (request.identity(), request))
                 .collect::<IndexMap<_, _>>()
                 .into_values()
                 .collect(),
         )
     }
 
+    /// The request attempts a model call made: its transport descendants, or, for bridges that
+    /// emit the request beside the call instead of under it, transport siblings inside the call's
+    /// time window when the call is the only model call under that parent.
+    fn transports(&self, call: usize) -> Vec<usize> {
+        let is_transport = |index: &usize| self.row(*index).call_keys.contains(&CallKey::Transport);
+        let nested: Vec<usize> = self
+            .graph
+            .descendants(call)
+            .into_iter()
+            .filter(is_transport)
+            .collect();
+        let Some(parent) = self.graph.parent(call).filter(|_| nested.is_empty()) else {
+            return nested;
+        };
+        let siblings = self.graph.children(parent);
+        let lone_call = siblings
+            .iter()
+            .filter(|sibling| self.kind(**sibling) == ObservationType::Llm)
+            .count()
+            == 1;
+        if !lone_call {
+            return nested;
+        }
+        let call_row = self.row(call);
+        let call_start_ns = i128::from(call_row.start_ns);
+        let call_end_ns = call_start_ns + i128::from(call_row.duration_ns);
+        siblings
+            .into_iter()
+            .filter(is_transport)
+            .filter(|sibling| {
+                let transport = self.row(*sibling);
+                let transport_start_ns = i128::from(transport.start_ns);
+                let transport_end_ns = transport_start_ns + i128::from(transport.duration_ns);
+                transport_start_ns >= call_start_ns && transport_end_ns <= call_end_ns
+            })
+            .collect()
+    }
+
     pub(super) fn unique_tools(&self) -> Vec<usize> {
         let mut by_call: IndexMap<&str, usize> = IndexMap::new();
         for index in

litellm-rust/crates/traces/src/resolve/spend.rs

@@ -107,14 +107,14 @@ impl<'a> KeyMatch<'a> {
             Self::Missing => false,
             Self::Unique(request) => selected
                 .iter()
-                .any(|row| row.request_id == request.request_id),
+                .any(|row| row.identity() == request.identity()),
             Self::Ambiguous(requests) => {
                 requests
                     .iter()
                     .filter(|request| {
                         selected
                             .iter()
-                            .any(|row| row.request_id == request.request_id)
+                            .any(|row| row.identity() == request.identity())
                     })
                     .count()
                     == 1
@@ -136,7 +136,7 @@ impl<'a> SpendEvidence<'a> {
                 let requests: Requests<'a> = matches
                     .iter()
                     .filter_map(KeyMatch::unique)
-                    .map(|request| (request.request_id.as_str(), request))
+                    .map(|request| (request.identity(), request))
                     .collect::<IndexMap<_, _>>()
                     .into_values()
                     .collect();
@@ -164,12 +164,16 @@ fn matches<'a>(
     spend_rows: &'a [SpendRow],
     key: &CallKey,
     row: &TraceSpansRow,
-) -> IndexMap<&'a str, &'a SpendRow> {
+) -> IndexMap<(&'a str, i64, &'a str), &'a SpendRow> {
     let matches = |spend: &SpendRow| match key {
         CallKey::ProviderResponse(id) => {
             !id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id)
         }
-        CallKey::LiteLlmRequest(id) => !id.is_empty() && spend.request_id == *id,
+        CallKey::LiteLlmRequest(id) => {
+            !id.is_empty()
+                && (spend.litellm_call_id == *id
+                    || (spend.litellm_call_id.is_empty() && spend.request_id == *id))
+        }
         CallKey::Transport => {
             !row.trace_id.is_empty()
                 && !row.span_id.is_empty()
@@ -180,7 +184,7 @@ fn matches<'a>(
     spend_rows
         .iter()
         .filter(|spend| ownership.owns(spend) && matches(spend))
-        .map(|spend| (spend.request_id.as_str(), spend))
+        .map(|spend| (spend.identity(), spend))
         .collect()
 }
 
@@ -190,18 +194,35 @@ pub(super) fn requests<'a>(
     spend_rows: &'a [SpendRow],
 ) -> SpendEvidence<'a> {
     let evidence = CallEvidence::from_row(row);
-    let matches = evidence
+    let keyed: Vec<(&CallKey, Requests<'a>)> = evidence
         .key_set()
         .into_iter()
         .flatten()
         .map(|key| {
-            KeyMatch::new(
+            (
+                key,
                 matches(ownership, spend_rows, key, row)
                     .into_values()
                     .collect(),
             )
         })
         .collect();
+    let anchored: Vec<&SpendRow> = keyed
+        .iter()
+        .filter(|(key, _)| !matches!(key, CallKey::LiteLlmRequest(_)))
+        .flat_map(|(_, requests)| requests.iter().copied())
+        .collect();
+    let legacy_rows = !anchored.is_empty()
+        && anchored
+            .iter()
+            .all(|request| request.litellm_call_id.is_empty());
+    let matches = keyed
+        .into_iter()
+        .filter(|(key, requests)| {
+            !(legacy_rows && requests.is_empty() && matches!(key, CallKey::LiteLlmRequest(_)))
+        })
+        .map(|(_, requests)| KeyMatch::new(requests))
+        .collect();
     match evidence.kind() {
         CallEvidenceKind::Complete => SpendEvidence::Complete(matches),
         CallEvidenceKind::Partial => SpendEvidence::Partial(matches),
@@ -226,9 +247,9 @@ pub(super) fn total(calls: &[Option<Requests<'_>>]) -> Option<f64> {
         .map(|requests| requests.as_ref())
         .collect::<Option<Vec<_>>>()
         .map(|calls| calls.into_iter().flatten().copied().collect());
-    let unique: IndexMap<&str, &SpendRow> = requests?
+    let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests?
         .into_iter()
-        .map(|request| (request.request_id.as_str(), request))
+        .map(|request| (request.identity(), request))
         .collect();
     request_cost(&unique.into_values().collect::<Vec<_>>())
 }

litellm-rust/crates/traces/tests/captures.rs

@@ -0,0 +1,445 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::path::{Path, PathBuf};
+
+use base64::{Engine as _, engine::general_purpose::STANDARD};
+use litellm_traces::{
+    CallEvidence, CallEvidenceKind, CallKey, DecodedSpan, ObservationType, SpanStatus, decode_otlp,
+    query::named::{SpendByResponseIdsRow, TraceSpansRow},
+    resolve_trace,
+};
+use rstest::rstest;
+use serde::Deserialize;
+use serde_json::{Value, json};
+
+struct CaptureData {
+    otlp: Vec<u8>,
+}
+
+fn capture_name(spend_log_path: &Path) -> &str {
+    spend_log_path
+        .file_stem()
+        .and_then(|stem| stem.to_str())
+        .and_then(|stem| stem.strip_suffix("_spend_logs"))
+        .unwrap_or_else(|| {
+            panic!(
+                "spend log filename must end with _spend_logs: {}",
+                spend_log_path.display()
+            )
+        })
+}
+
+fn manifest_path(path: &Path) -> PathBuf {
+    if path.is_absolute() {
+        path.to_path_buf()
+    } else {
+        Path::new(env!("CARGO_MANIFEST_DIR")).join(path)
+    }
+}
+
+fn capture_data(spend_log_path: &Path) -> CaptureData {
+    let name = capture_name(spend_log_path);
+    let otlp_path = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures")
+        .join(format!("{name}.json"));
+    let otlp = std::fs::read(&otlp_path).unwrap_or_else(|error| {
+        panic!(
+            "missing OTLP export for spend capture {name} at {}: {error}",
+            otlp_path.display()
+        )
+    });
+    CaptureData { otlp }
+}
+
+#[derive(Deserialize)]
+struct CapturedSpend {
+    request_id: String,
+    #[serde(default)]
+    litellm_call_id: String,
+    response_id: String,
+    trace_id: String,
+    span_id: String,
+    team_id: String,
+    api_key: String,
+    user: String,
+    spend: Option<f64>,
+    start_time: i64,
+    metadata: String,
+}
+
+#[derive(Deserialize)]
+struct SpendMetadata {
+    fixture_capture: FixtureCapture,
+}
+
+#[derive(Deserialize)]
+struct FixtureCapture {
+    name: String,
+    trace_id: String,
+    spend_linked: bool,
+    #[serde(default = "true_value")]
+    spend_complete: bool,
+}
+
+fn true_value() -> bool {
+    true
+}
+
+fn upstream_response_id(response_id: &str) -> String {
+    let Some(encoded) = response_id.strip_prefix("resp_") else {
+        return String::new();
+    };
+    STANDARD
+        .decode(encoded)
+        .ok()
+        .and_then(|bytes| String::from_utf8(bytes).ok())
+        .and_then(|decoded| {
+            let (_, response_id) = decoded.split_once("response_id:")?;
+            Some(response_id.split(';').next()?.to_owned())
+        })
+        .unwrap_or_default()
+}
+
+fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec<SpendByResponseIdsRow>) {
+    let records: Vec<CapturedSpend> = spend_logs
+        .lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid spend fixture row"))
+        .collect();
+    let metadata: SpendMetadata =
+        serde_json::from_str(&records.first().expect("spend fixture rows").metadata)
+            .expect("valid spend fixture metadata");
+    let spends = records
+        .into_iter()
+        .map(|record| {
+            let upstream_response_id = upstream_response_id(&record.response_id);
+            SpendByResponseIdsRow {
+                request_id: record.request_id,
+                litellm_call_id: record.litellm_call_id,
+                response_id: record.response_id,
+                upstream_response_id,
+                trace_id: record.trace_id,
+                span_id: record.span_id,
+                team_id: record.team_id,
+                api_key: record.api_key,
+                user: record.user,
+                spend: record.spend,
+                start_ms: record.start_time,
+            }
+        })
+        .collect();
+    (metadata.fixture_capture, spends)
+}
+
+fn status_message(span: &DecodedSpan) -> &str {
+    if !span.status_message.is_empty() {
+        return &span.status_message;
+    }
+    span.events
+        .iter()
+        .find(|event| event.name == "exception")
+        .and_then(|event| {
+            event
+                .attributes
+                .get("exception.message")
+                .filter(|message| !message.is_empty())
+                .or_else(|| event.attributes.get("exception.type"))
+        })
+        .map(String::as_str)
+        .unwrap_or_default()
+}
+
+fn trace_span(span: DecodedSpan) -> TraceSpansRow {
+    let service = span
+        .resource_attributes
+        .get("service.name")
+        .cloned()
+        .unwrap_or_default();
+    let message = status_message(&span);
+    let error_truncated = message.chars().count() > 128;
+    let status_message = message.chars().take(128).collect();
+    let normalized = span.normalized;
+    let call_keys = normalized
+        .calls
+        .key_set()
+        .into_iter()
+        .flatten()
+        .cloned()
+        .collect();
+    let litellm_request_id = normalized
+        .calls
+        .key_set()
+        .into_iter()
+        .flatten()
+        .find_map(|key| match key {
+            CallKey::ProviderResponse(id) => Some(id.clone()),
+            CallKey::LiteLlmRequest(_) | CallKey::Transport => None,
+        })
+        .unwrap_or_default();
+    TraceSpansRow {
+        trace_id: span.trace_id,
+        span_id: span.span_id,
+        parent_span_id: span.parent_span_id,
+        name: span.name,
+        kind: normalized.observation_type,
+        wrapper_candidate: normalized.wrapper_candidate,
+        agent: normalized.agent_name.unwrap_or_default(),
+        framework: normalized
+            .framework
+            .map(|framework| framework.to_string())
+            .unwrap_or_default(),
+        status: match span.status_code.as_str() {
+            "STATUS_CODE_OK" => SpanStatus::Ok,
+            "STATUS_CODE_ERROR" => SpanStatus::Error,
+            _ => SpanStatus::Unset,
+        },
+        status_message,
+        error_truncated,
+        start_ns: i64::try_from(span.start_ns).expect("valid trace start timestamp"),
+        duration_ns: span.end_ns - span.start_ns,
+        service,
+        input_preview: normalized.input_preview,
+        model: normalized.model.unwrap_or_default(),
+        input_tokens: normalized.input_tokens,
+        output_tokens: normalized.output_tokens,
+        litellm_request_id,
+        call_keys,
+        call_evidence: Some(normalized.calls.kind()),
+        tool_call_id: normalized.tool_call_id.unwrap_or_default(),
+        team_id: "fixture-team".into(),
+        api_key_hash: "fixture-key".into(),
+        user_id: "fixture-user".into(),
+    }
+}
+
+fn trace_rows(otlp: &[u8]) -> Vec<TraceSpansRow> {
+    let mut seen = BTreeSet::new();
+    decode_otlp(otlp, Some("application/json"))
+        .expect("valid OTLP fixture")
+        .into_iter()
+        .filter_map(|span| seen.insert(span.span_id.clone()).then(|| trace_span(span)))
+        .collect()
+}
+
+fn fixture(
+    spend_log_path: &Path,
+) -> (
+    CaptureData,
+    FixtureCapture,
+    Vec<TraceSpansRow>,
+    Vec<SpendByResponseIdsRow>,
+) {
+    let spend_log_path = manifest_path(spend_log_path);
+    let name = capture_name(&spend_log_path);
+    let spend_log_contents = std::fs::read_to_string(&spend_log_path).unwrap_or_else(|error| {
+        panic!(
+            "unable to read spend log fixture {}: {error}",
+            spend_log_path.display()
+        )
+    });
+    let (capture, spends) = captured_spend_rows(&spend_log_contents);
+    assert_eq!(capture.name, name);
+    let data = capture_data(&spend_log_path);
+    let rows = trace_rows(&data.otlp);
+    (data, capture, rows, spends)
+}
+
+fn assert_spend_close(actual: Option<f64>, expected: Option<f64>, capture: &str) {
+    match (actual, expected) {
+        (Some(actual), Some(expected)) => assert!(
+            (actual - expected).abs() <= 1e-12,
+            "{capture}: expected {expected}, got {actual}"
+        ),
+        _ => assert_eq!(actual, expected, "{capture}"),
+    }
+}
+
+fn agent_spends(trace: &litellm_traces::Trace) -> BTreeMap<String, Option<f64>> {
+    trace
+        .agents
+        .iter()
+        .map(|agent| (agent.name.clone(), agent.spend))
+        .collect()
+}
+
+fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow {
+    let start_ns =
+        i64::try_from(i128::from(call.start_ns) + i128::from(call.duration_ns) + 1_000_000)
+            .expect("valid unrelated transport timestamp");
+    TraceSpansRow {
+        trace_id: call.trace_id.clone(),
+        span_id: format!("unrelated-transport-{}", call.span_id),
+        parent_span_id: call.parent_span_id.clone(),
+        name: "unrelated-http".into(),
+        kind: ObservationType::Framework,
+        wrapper_candidate: false,
+        agent: String::new(),
+        framework: String::new(),
+        status: SpanStatus::Ok,
+        status_message: String::new(),
+        error_truncated: false,
+        start_ns,
+        duration_ns: 1_000_000,
+        service: call.service.clone(),
+        input_preview: String::new(),
+        model: String::new(),
+        input_tokens: 0,
+        output_tokens: 0,
+        litellm_request_id: String::new(),
+        call_keys: vec![CallKey::Transport],
+        call_evidence: Some(CallEvidenceKind::Complete),
+        tool_call_id: String::new(),
+        team_id: call.team_id.clone(),
+        api_key_hash: call.api_key_hash.clone(),
+        user_id: call.user_id.clone(),
+    }
+}
+
+fn append_response_id(document: &mut Value, trace_id: &str, span_id: &str, response_id: &str) {
+    let resources = document["resourceSpans"]
+        .as_array_mut()
+        .expect("OTLP resource spans");
+    for resource in resources {
+        let scopes = resource["scopeSpans"]
+            .as_array_mut()
+            .expect("OTLP scope spans");
+        for scope in scopes {
+            let spans = scope["spans"].as_array_mut().expect("OTLP spans");
+            for span in spans {
+                let matches = span.get("traceId").and_then(Value::as_str) == Some(trace_id)
+                    && span.get("spanId").and_then(Value::as_str) == Some(span_id);
+                if !matches {
+                    continue;
+                }
+                let attributes = span
+                    .as_object_mut()
+                    .expect("OTLP span object")
+                    .entry("attributes")
+                    .or_insert_with(|| json!([]))
+                    .as_array_mut()
+                    .expect("OTLP span attributes");
+                attributes.push(json!({
+                    "key": "gen_ai.response.id",
+                    "value": { "stringValue": response_id },
+                }));
+                return;
+            }
+        }
+    }
+    panic!("missing OTLP span {trace_id}/{span_id}");
+}
+
+#[rstest]
+fn captured_trace_cost_matches_spend_logs(
+    #[files("../traces-clickhouse/tests/fixtures/*_spend_logs.jsonl")]
+    #[exclude("deeplite_swarm")]
+    spend_logs: PathBuf,
+) {
+    let name = capture_name(&spend_logs);
+    let (_, capture, rows, spends) = fixture(&spend_logs);
+    let trace = resolve_trace(&capture.trace_id, "", &rows, &spends).expect("captured trace");
+    let expected = if capture.spend_linked && capture.spend_complete {
+        Some(spends.iter().map(|row| row.spend.unwrap_or(0.0)).sum())
+    } else {
+        None
+    };
+    assert_spend_close(trace.summary.spend, expected, name);
+}
+
+#[rstest]
+fn unrelated_sibling_transport_leaves_cost_unchanged(
+    #[files("../traces-clickhouse/tests/fixtures/*_spend_logs.jsonl")]
+    #[exclude("deeplite_swarm")]
+    spend_logs: PathBuf,
+) {
+    let name = capture_name(&spend_logs);
+    let (_, capture, rows, spends) = fixture(&spend_logs);
+    let baseline = resolve_trace(&capture.trace_id, "", &rows, &spends).expect("captured trace");
+    let baseline_spend = baseline.summary.spend;
+    let baseline_agent_spends = agent_spends(&baseline);
+    let calls: Vec<_> = rows
+        .iter()
+        .filter(|row| {
+            row.kind == ObservationType::Llm && !row.call_keys.contains(&CallKey::Transport)
+        })
+        .cloned()
+        .collect();
+    assert!(!calls.is_empty(), "{name} has no model call rows");
+    for call in calls {
+        let augmented_rows = rows
+            .iter()
+            .cloned()
+            .chain([unrelated_transport(&call)])
+            .collect::<Vec<_>>();
+        let augmented =
+            resolve_trace(&capture.trace_id, "", &augmented_rows, &spends).expect("captured trace");
+        assert_eq!(
+            augmented.summary.spend, baseline_spend,
+            "{name}, model span {}",
+            call.span_id
+        );
+        assert_eq!(
+            agent_spends(&augmented),
+            baseline_agent_spends,
+            "{name}, model span {}",
+            call.span_id
+        );
+    }
+}
+
+#[rstest]
+fn redundant_genai_response_id_keeps_call_evidence(
+    #[files("../traces-clickhouse/tests/fixtures/*_spend_logs.jsonl")]
+    #[exclude("deeplite_swarm")]
+    spend_logs: PathBuf,
+) {
+    let (data, capture, _, _) = fixture(&spend_logs);
+    let name = capture.name;
+    let decoded = decode_otlp(&data.otlp, Some("application/json")).expect("valid OTLP fixture");
+    let targets: Vec<_> = decoded
+        .iter()
+        .filter_map(|span| {
+            let CallEvidence::Complete(keys) = &span.normalized.calls else {
+                return None;
+            };
+            if span.attributes.contains_key("gen_ai.response.id") {
+                return None;
+            }
+            let response_ids: Vec<_> = keys
+                .iter()
+                .filter_map(|key| match key {
+                    CallKey::ProviderResponse(id) => Some(id.clone()),
+                    CallKey::LiteLlmRequest(_) | CallKey::Transport => None,
+                })
+                .collect();
+            (!response_ids.is_empty()).then(|| {
+                (
+                    span.trace_id.clone(),
+                    span.span_id.clone(),
+                    span.normalized.calls.clone(),
+                    response_ids,
+                )
+            })
+        })
+        .collect();
+    for (trace_id, span_id, expected, response_ids) in targets {
+        for response_id in response_ids {
+            let mut document: Value =
+                serde_json::from_slice(&data.otlp).expect("valid OTLP JSON fixture");
+            append_response_id(&mut document, &trace_id, &span_id, &response_id);
+            let modified = serde_json::to_vec(&document).expect("serializable OTLP JSON");
+            let spans = decode_otlp(&modified, Some("application/json"))
+                .expect("OTLP with redundant response ID");
+            let actual = spans
+                .iter()
+                .find(|span| span.trace_id == trace_id && span.span_id == span_id)
+                .expect("modified span")
+                .normalized
+                .calls
+                .clone();
+            assert_eq!(
+                actual, expected,
+                "{name}, span {span_id}, response id {response_id}"
+            );
+        }
+    }
+}

litellm-rust/crates/traces/tests/fixtures/claude_agent_sdk_simple.json

@@ -9,12 +9,6 @@
               "stringValue": "research_agent"
             }
           },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "claude-agent-sdk-simple-linked"
-            }
-          },
           {
             "key": "host.arch",
             "value": {
@@ -24,13 +18,19 @@
           {
             "key": "os.type",
             "value": {
-              "stringValue": "linux"
+              "stringValue": "darwin"
             }
           },
           {
             "key": "os.version",
             "value": {
-              "stringValue": "7.0.11-orbstack-00360-gc9bc4d96ac70"
+              "stringValue": "25.6.0"
+            }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "claude-code"
             }
           },
           {
@@ -49,13 +49,13 @@
           },
           "spans": [
             {
-              "traceId": "68d4ab5c1bb4cdff9f7fa72ce5e360d4",
-              "spanId": "2202f91fa2679814",
-              "parentSpanId": "f0281548ccd4d661",
+              "traceId": "207bf859f653150fd4346add5cb11ad3",
+              "spanId": "1825d4fb63cffcec",
+              "parentSpanId": "8a420dd58d6ff237",
               "name": "claude_code.llm_request",
               "kind": 1,
-              "startTimeUnixNano": "1791013731536000000",
-              "endTimeUnixNano": "1791013738679125375",
+              "startTimeUnixNano": "1791061376589000000",
+              "endTimeUnixNano": "1791061384361645125",
               "attributes": [
                 {
                   "key": "gen_ai.agent.name",
@@ -66,19 +66,19 @@
                 {
                   "key": "user.id",
                   "value": {
-                    "stringValue": "4493a11fb6084c04be89c081b455b16d3b2df792ebf4dc2014e12d078d4732e8"
+                    "stringValue": "999dc5afd40b65ac74b33bdc01c83510d935a57dfc4b1c232b51504bb8caed2e"
                   }
                 },
                 {
                   "key": "session.id",
                   "value": {
-                    "stringValue": "7b36c5c7-8eb5-45ad-8ffc-2966f64389b7"
+                    "stringValue": "0e5bb47c-6b08-4a1b-bde4-9ab142589572"
                   }
                 },
                 {
                   "key": "terminal.type",
                   "value": {
-                    "stringValue": "non-interactive"
+                    "stringValue": "ghostty"
                   }
                 },
                 {
@@ -168,7 +168,7 @@
                 {
                   "key": "system_reminders_count",
                   "value": {
-                    "intValue": "1"
+                    "intValue": "2"
                   }
                 },
                 {
@@ -180,25 +180,25 @@
                 {
                   "key": "system_reminders",
                   "value": {
-                    "stringValue": "# Environment\nYou have been invoked in the following environment: \n - Primary working directory: /fixtures/claude-agent-sdk\n - Is a git repository: true\n - Platform: linux\n - Shell: unknown\n - OS Version: Linux 7.0.11-orbstack-00360-gc9bc4d96ac70\n\nYou are powered by the model openai/gpt-6-luna.\n\n<total_tokens>15000000 tokens left</total_tokens>\n\nToday's date is 2026-10-03."
+                    "stringValue": "Codebase and user instructions are shown below. Be sure to adhere to these instructions. IMPORTANT: These instructions OVERRIDE any default behavior and you MUST follow them exactly as written.\n\nContents of /home/user/.claude/projects/-home-user-dev-litellm-lens-example/memory/MEMORY.md (user's auto-memory, persists across conversations):\n\n- [LiteLLM spend correlation contract](litellm-spend-correlation-contract.md) \u2014 source PR #44421, offline validation with mock gateway + recorder\n- [SDK wiring limits](litellm-lens-example-sdk-wiring-limits.md) \u2014 which SDKs cannot take the shared gateway transport and why\n\n---\n\n# Environment\nYou have been invoked in the following environment: \n - Primary working directory: /home/user/dev/litellm-lens-example/claude-agent-sdk\n - Is a git repository: true\n - Platform: darwin\n - Shell: zsh\n - OS Version: Darwin 25.6.0\n\nYou are powered by the model openai/gpt-6-luna.\n\n<total_tokens>15000000 tokens left</total_tokens>\n\nToday's date is 2026-10-03."
                   }
                 },
                 {
                   "key": "duration_ms",
                   "value": {
-                    "intValue": "7143"
+                    "intValue": "7772"
                   }
                 },
                 {
                   "key": "input_tokens",
                   "value": {
-                    "intValue": "172"
+                    "intValue": "323"
                   }
                 },
                 {
                   "key": "output_tokens",
                   "value": {
-                    "intValue": "665"
+                    "intValue": "671"
                   }
                 },
                 {
@@ -234,25 +234,25 @@
                 {
                   "key": "request_id",
                   "value": {
-                    "stringValue": "msg_7117e61b-cb2a-4e2f-8155-9b8bab62a4b9"
+                    "stringValue": "msg_b65eb4b6-da48-4eb3-a564-0f65202f4797"
                   }
                 },
                 {
                   "key": "gen_ai.response.id",
                   "value": {
-                    "stringValue": "msg_7117e61b-cb2a-4e2f-8155-9b8bab62a4b9"
+                    "stringValue": "msg_b65eb4b6-da48-4eb3-a564-0f65202f4797"
                   }
                 },
                 {
                   "key": "ttft_ms",
                   "value": {
-                    "intValue": "540"
+                    "intValue": "769"
                   }
                 },
                 {
                   "key": "first_content_ms",
                   "value": {
-                    "intValue": "5584"
+                    "intValue": "6113"
                   }
                 },
                 {
@@ -264,7 +264,7 @@
                 {
                   "key": "response.model_output",
                   "value": {
-                    "stringValue": "An **agent trace** is the chronological record of an agent run: the input it received, the model’s intermediate messages, any tools it called and their results, and how the run ended.\n\nIt shows **how** the agent reached its final answer—not just the answer itself—and is useful for debugging and monitoring. In the Claude Agent SDK, you can follow a run through its streamed messages and events. Traces may contain prompts or other sensitive data, so handle them accordingly."
+                    "stringValue": "An **agent trace** is the connected record of one agent task from start to finish\u2014not just a single LLM request.\n\nFor example, a task might produce a trace containing:\n1. The agent\u2019s model call\n2. A weather-tool call and its result\n3. A follow-up model call that writes the answer\n\nThose steps are linked as spans in a timeline, often with timestamps, status, token usage, and cost. Traces help you understand what the agent did, where it failed or spent time, and which calls contributed to the cost.\n\nA trace may contain prompts and tool inputs or outputs, depending on the instrumentation, but it **doesn\u2019t mean the agent\u2019s hidden chain-of-thought is being recorded**."
                   }
                 },
                 {
@@ -288,7 +288,7 @@
               ],
               "events": [
                 {
-                  "timeUnixNano": "1791013731538265232",
+                  "timeUnixNano": "1791061376591988250",
                   "name": "gen_ai.request.attempt",
                   "attributes": [
                     {
@@ -304,13 +304,13 @@
               "flags": 257
             },
             {
-              "traceId": "68d4ab5c1bb4cdff9f7fa72ce5e360d4",
-              "spanId": "f0281548ccd4d661",
-              "parentSpanId": "d8aa87f2b774735d",
+              "traceId": "207bf859f653150fd4346add5cb11ad3",
+              "spanId": "8a420dd58d6ff237",
+              "parentSpanId": "c2c6b0971076f31c",
               "name": "claude_code.interaction",
               "kind": 1,
-              "startTimeUnixNano": "1791013731512000000",
-              "endTimeUnixNano": "1791013738692801566",
+              "startTimeUnixNano": "1791061376541000000",
+              "endTimeUnixNano": "1791061384366705375",
               "attributes": [
                 {
                   "key": "gen_ai.agent.name",
@@ -321,19 +321,19 @@
                 {
                   "key": "user.id",
                   "value": {
-                    "stringValue": "4493a11fb6084c04be89c081b455b16d3b2df792ebf4dc2014e12d078d4732e8"
+                    "stringValue": "999dc5afd40b65ac74b33bdc01c83510d935a57dfc4b1c232b51504bb8caed2e"
                   }
                 },
                 {
                   "key": "session.id",
                   "value": {
-                    "stringValue": "7b36c5c7-8eb5-45ad-8ffc-2966f64389b7"
+                    "stringValue": "0e5bb47c-6b08-4a1b-bde4-9ab142589572"
                   }
                 },
                 {
                   "key": "terminal.type",
                   "value": {
-                    "stringValue": "non-interactive"
+                    "stringValue": "ghostty"
                   }
                 },
                 {
@@ -381,7 +381,7 @@
                 {
                   "key": "interaction.duration_ms",
                   "value": {
-                    "intValue": "7181"
+                    "intValue": "7826"
                   }
                 }
               ],
@@ -416,7 +416,7 @@
           {
             "key": "service.instance.id",
             "value": {
-              "stringValue": "ff3ae854-67ab-4a61-98b3-8840846062f8"
+              "stringValue": "5ac3598e-aed9-4cda-ac0e-1ab2f82a2673"
             }
           },
           {
@@ -425,17 +425,17 @@
               "stringValue": "research_agent"
             }
           },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "claude-agent-sdk-simple-linked"
-            }
-          },
           {
             "key": "telemetry.auto.version",
             "value": {
               "stringValue": "0.66b0"
             }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service:python3"
+            }
           }
         ]
       },
@@ -447,12 +447,12 @@
           },
           "spans": [
             {
-              "traceId": "68d4ab5c1bb4cdff9f7fa72ce5e360d4",
-              "spanId": "d8aa87f2b774735d",
+              "traceId": "207bf859f653150fd4346add5cb11ad3",
+              "spanId": "c2c6b0971076f31c",
               "name": "ClaudeAgentSDK.query",
               "kind": 1,
-              "startTimeUnixNano": "1791013731360778595",
-              "endTimeUnixNano": "1791013738788489656",
+              "startTimeUnixNano": "1791061376098056000",
+              "endTimeUnixNano": "1791061384413648000",
               "attributes": [
                 {
                   "key": "llm.system",
@@ -481,7 +481,7 @@
                 {
                   "key": "llm.output_messages.0.message.content.0",
                   "value": {
-                    "stringValue": "An **agent trace** is the chronological record of an agent run: the input it received, the model’s intermediate messages, any tools it called and their results, and how the run ended.\n\nIt shows **how** the agent reached its final answer—not just the answer itself—and is useful for debugging and monitoring. In the Claude Agent SDK, you can follow a run through its streamed messages and events. Traces may contain prompts or other sensitive data, so handle them accordingly."
+                    "stringValue": "An **agent trace** is the connected record of one agent task from start to finish\u2014not just a single LLM request.\n\nFor example, a task might produce a trace containing:\n1. The agent\u2019s model call\n2. A weather-tool call and its result\n3. A follow-up model call that writes the answer\n\nThose steps are linked as spans in a timeline, often with timestamps, status, token usage, and cost. Traces help you understand what the agent did, where it failed or spent time, and which calls contributed to the cost.\n\nA trace may contain prompts and tool inputs or outputs, depending on the instrumentation, but it **doesn\u2019t mean the agent\u2019s hidden chain-of-thought is being recorded**."
                   }
                 },
                 {
@@ -499,7 +499,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "An **agent trace** is the chronological record of an agent run: the input it received, the model’s intermediate messages, any tools it called and their results, and how the run ended.\n\nIt shows **how** the agent reached its final answer—not just the answer itself—and is useful for debugging and monitoring. In the Claude Agent SDK, you can follow a run through its streamed messages and events. Traces may contain prompts or other sensitive data, so handle them accordingly."
+                    "stringValue": "An **agent trace** is the connected record of one agent task from start to finish\u2014not just a single LLM request.\n\nFor example, a task might produce a trace containing:\n1. The agent\u2019s model call\n2. A weather-tool call and its result\n3. A follow-up model call that writes the answer\n\nThose steps are linked as spans in a timeline, often with timestamps, status, token usage, and cost. Traces help you understand what the agent did, where it failed or spent time, and which calls contributed to the cost.\n\nA trace may contain prompts and tool inputs or outputs, depending on the instrumentation, but it **doesn\u2019t mean the agent\u2019s hidden chain-of-thought is being recorded**."
                   }
                 },
                 {
@@ -517,19 +517,19 @@
                 {
                   "key": "llm.token_count.prompt",
                   "value": {
-                    "intValue": "172"
+                    "intValue": "323"
                   }
                 },
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "665"
+                    "intValue": "671"
                   }
                 },
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "837"
+                    "intValue": "994"
                   }
                 },
                 {
@@ -547,13 +547,13 @@
                 {
                   "key": "llm.cost.total",
                   "value": {
-                    "doubleValue": 0.013987999999999999
+                    "doubleValue": 0.014712000000000001
                   }
                 },
                 {
                   "key": "session.id",
                   "value": {
-                    "stringValue": "7b36c5c7-8eb5-45ad-8ffc-2966f64389b7"
+                    "stringValue": "0e5bb47c-6b08-4a1b-bde4-9ab142589572"
                   }
                 },
                 {

litellm-rust/crates/traces/tests/fixtures/crewai_simple.json

@@ -24,13 +24,7 @@
           {
             "key": "service.instance.id",
             "value": {
-              "stringValue": "ce3d1e9d-9ad8-4e9c-baa1-790de8270f2a"
-            }
-          },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "crewai-simple"
+              "stringValue": "474ef02c-24f0-4fe4-84e3-568c08278bf7"
             }
           },
           {
@@ -38,6 +32,12 @@
             "value": {
               "stringValue": "0.66b0"
             }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service:python3"
+            }
           }
         ]
       },
@@ -49,13 +49,13 @@
           },
           "spans": [
             {
-              "traceId": "110c44d444b7742cfa57fc70cef424d8",
-              "spanId": "29ed447ec5f9b6e8",
-              "parentSpanId": "97f30ba7a431f8e7",
+              "traceId": "9423efa0faa4a873d896222c6b33aacc",
+              "spanId": "742c3f48055313df",
+              "parentSpanId": "3af673c4fc56a4d3",
               "name": "ChatCompletion",
               "kind": 1,
-              "startTimeUnixNano": "1791012936617707896",
-              "endTimeUnixNano": "1791012938235971272",
+              "startTimeUnixNano": "1791061313593128000",
+              "endTimeUnixNano": "1791061315819218000",
               "attributes": [
                 {
                   "key": "llm.system",
@@ -66,7 +66,7 @@
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"role\":\"system\",\"content\":\"You are research_agent. You explain technical concepts.\\nYour personal goal is: Answer questions clearly\"},{\"role\":\"user\",\"content\":\"\\nCurrent Task: What is an agent trace?\\n\\nThis is the expected criteria for your final answer: A short answer\\nyou MUST return the actual complete content as the final answer, not a summary.\\n\\nProvide your complete response:\"}],\"model\":\"openai/gpt-6-luna\"}"
+                    "stringValue": "{\"messages\": [{\"role\": \"system\", \"content\": \"You are research_agent. You explain technical concepts.\\nYour personal goal is: Answer questions clearly\"}, {\"role\": \"user\", \"content\": \"\\nCurrent Task: What is an agent trace?\\n\\nThis is the expected criteria for your final answer: A short answer\\nyou MUST return the actual complete content as the final answer, not a summary.\\n\\nProvide your complete response:\"}], \"model\": \"openai/gpt-6-luna\"}"
                   }
                 },
                 {
@@ -78,7 +78,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"id\":\"chatcmpl-EUoZMwGeJM9ul6B0NsHufReuUYcEa\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"content\":\"An **agent trace** is a chronological record of an AI agent’s actions and observations while completing a task—such as its inputs, tool calls, tool results, and final response. It helps explain, debug, and evaluate the agent’s behavior.\",\"role\":\"assistant\",\"annotations\":[],\"provider_specific_fields\":{\"refusal\":null}},\"provider_specific_fields\":{}}],\"created\":1791012936,\"model\":\"openai/gpt-6-luna\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"usage\":{\"completion_tokens\":86,\"prompt_tokens\":73,\"total_tokens\":159,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":27,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"cache_creation_tokens\":0}}}"
+                    "stringValue": "{\"id\":\"chatcmpl-EV19e7sf5xeFVr05xBNOmEygQ8mTd\",\"choices\":[{\"finish_reason\":\"stop\",\"index\":0,\"message\":{\"content\":\"An **agent trace** is a chronological record of an AI agent\u2019s execution: the steps it took, such as receiving input, calling tools, observing results, and producing an answer. Traces help developers understand, debug, and evaluate an agent\u2019s behavior.\",\"role\":\"assistant\",\"annotations\":[],\"provider_specific_fields\":{\"refusal\":null}},\"provider_specific_fields\":{}}],\"created\":1791061314,\"model\":\"openai/gpt-6-luna\",\"object\":\"chat.completion\",\"service_tier\":\"default\",\"usage\":{\"completion_tokens\":122,\"prompt_tokens\":73,\"total_tokens\":195,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":61,\"rejected_prediction_tokens\":0},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"cache_creation_tokens\":0}}}"
                   }
                 },
                 {
@@ -90,7 +90,7 @@
                 {
                   "key": "llm.invocation_parameters",
                   "value": {
-                    "stringValue": "{\"model\":\"openai/gpt-6-luna\"}"
+                    "stringValue": "{\"model\": \"openai/gpt-6-luna\"}"
                   }
                 },
                 {
@@ -126,7 +126,7 @@
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "159"
+                    "intValue": "195"
                   }
                 },
                 {
@@ -138,7 +138,7 @@
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "86"
+                    "intValue": "122"
                   }
                 },
                 {
@@ -162,7 +162,7 @@
                 {
                   "key": "llm.token_count.completion_details.reasoning",
                   "value": {
-                    "intValue": "27"
+                    "intValue": "61"
                   }
                 },
                 {
@@ -180,7 +180,7 @@
                 {
                   "key": "llm.output_messages.0.message.content",
                   "value": {
-                    "stringValue": "An **agent trace** is a chronological record of an AI agent’s actions and observations while completing a task—such as its inputs, tool calls, tool results, and final response. It helps explain, debug, and evaluate the agent’s behavior."
+                    "stringValue": "An **agent trace** is a chronological record of an AI agent\u2019s execution: the steps it took, such as receiving input, calling tools, observing results, and producing an answer. Traces help developers understand, debug, and evaluate an agent\u2019s behavior."
                   }
                 },
                 {
@@ -210,18 +210,18 @@
           },
           "spans": [
             {
-              "traceId": "110c44d444b7742cfa57fc70cef424d8",
-              "spanId": "97f30ba7a431f8e7",
-              "parentSpanId": "b2637dab2e2fc2ca",
+              "traceId": "9423efa0faa4a873d896222c6b33aacc",
+              "spanId": "3af673c4fc56a4d3",
+              "parentSpanId": "1edfe2e36e867117",
               "name": "research_agent._execute_core",
               "kind": 1,
-              "startTimeUnixNano": "1791012936360020229",
-              "endTimeUnixNano": "1791012938250502005",
+              "startTimeUnixNano": "1791061313403894000",
+              "endTimeUnixNano": "1791061315825553000",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"agent\":{\"entity_type\":\"agent\",\"id\":\"af3484b5-92a5-4a01-acf7-0e759af2f968\",\"role\":\"research_agent\",\"goal\":\"Answer questions clearly\",\"backstory\":\"You explain technical concepts.\",\"cache\":true,\"verbose\":false,\"max_rpm\":null,\"allow_delegation\":false,\"tools\":[],\"max_iter\":25,\"tool_failure_policy\":null,\"i18n\":{\"prompt_file\":null},\"cache_handler\":null,\"tools_results\":[],\"max_tokens\":null,\"knowledge\":null,\"knowledge_sources\":null,\"knowledge_storage\":null,\"security_config\":{\"fingerprint\":{\"metadata\":{}}},\"checkpoint\":null,\"adapted_agent\":false,\"knowledge_config\":null,\"apps\":null,\"mcps\":null,\"memory\":null,\"skills\":null,\"execution_context\":null,\"checkpoint_kickoff_event_id\":null,\"max_execution_time\":null,\"use_system_prompt\":true,\"system_template\":null,\"prompt_template\":null,\"response_template\":null,\"allow_code_execution\":false,\"respect_context_window\":true,\"max_retry_limit\":2,\"multimodal\":false,\"inject_date\":false,\"date_format\":\"%Y-%m-%d\",\"code_execution_mode\":\"safe\",\"planning_config\":null,\"planning\":false,\"reasoning\":false,\"max_reasoning_attempts\":null,\"embedder\":null,\"agent_knowledge_context\":null,\"crew_knowledge_context\":null,\"knowledge_search_query\":null,\"from_repository\":null,\"guardrail_max_retries\":3,\"a2a\":null,\"key\":\"4dc9e33a8c7925f0f322ab0a8886aef0\"},\"context\":\"\",\"tools\":[]}"
+                    "stringValue": "{\"agent\": {\"entity_type\": \"agent\", \"id\": \"85ad8f68-68d9-4510-9367-af4d6697cb24\", \"role\": \"research_agent\", \"goal\": \"Answer questions clearly\", \"backstory\": \"You explain technical concepts.\", \"cache\": true, \"verbose\": false, \"max_rpm\": null, \"allow_delegation\": false, \"tools\": [], \"max_iter\": 25, \"tool_failure_policy\": null, \"i18n\": {\"prompt_file\": null}, \"cache_handler\": null, \"tools_results\": [], \"max_tokens\": null, \"knowledge\": null, \"knowledge_sources\": null, \"knowledge_storage\": null, \"security_config\": {\"fingerprint\": {\"metadata\": {}}}, \"checkpoint\": null, \"adapted_agent\": false, \"knowledge_config\": null, \"apps\": null, \"mcps\": null, \"memory\": null, \"skills\": null, \"execution_context\": null, \"checkpoint_kickoff_event_id\": null, \"max_execution_time\": null, \"use_system_prompt\": true, \"system_template\": null, \"prompt_template\": null, \"response_template\": null, \"allow_code_execution\": false, \"respect_context_window\": true, \"max_retry_limit\": 2, \"multimodal\": false, \"inject_date\": false, \"date_format\": \"%Y-%m-%d\", \"code_execution_mode\": \"safe\", \"planning_config\": null, \"planning\": false, \"reasoning\": false, \"max_reasoning_attempts\": null, \"embedder\": null, \"agent_knowledge_context\": null, \"crew_knowledge_context\": null, \"knowledge_search_query\": null, \"from_repository\": null, \"guardrail_max_retries\": 3, \"a2a\": null, \"key\": \"4dc9e33a8c7925f0f322ab0a8886aef0\"}, \"context\": \"\", \"tools\": []}"
                   }
                 },
                 {
@@ -239,7 +239,7 @@
                 {
                   "key": "task_id",
                   "value": {
-                    "stringValue": "583ee87b-ad8f-42b0-9354-78036e8c3875"
+                    "stringValue": "be50aff3-19df-4772-99d7-5bd96218c0f6"
                   }
                 },
                 {
@@ -257,13 +257,13 @@
                 {
                   "key": "crew_id",
                   "value": {
-                    "stringValue": "75706cf9-6de1-475e-8e6b-dc145c5d54d6"
+                    "stringValue": "e829105c-c95a-4f8e-aea4-4a98b6c84b13"
                   }
                 },
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"description\":\"What is an agent trace?\",\"name\":\"What is an agent trace?\",\"expected_output\":\"A short answer\",\"summary\":\"What is an agent trace?...\",\"raw\":\"An **agent trace** is a chronological record of an AI agent’s actions and observations while completing a task—such as its inputs, tool calls, tool results, and final response. It helps explain, debug, and evaluate the agent’s behavior.\",\"pydantic\":null,\"json_dict\":null,\"agent\":\"research_agent\",\"output_format\":\"raw\",\"messages\":[{\"role\":\"system\",\"content\":\"You are research_agent. You explain technical concepts.\\nYour personal goal is: Answer questions clearly\"},{\"role\":\"user\",\"content\":\"\\nCurrent Task: What is an agent trace?\\n\\nThis is the expected criteria for your final answer: A short answer\\nyou MUST return the actual complete content as the final answer, not a summary.\\n\\nProvide your complete response:\"}],\"tool_failures\":[]}"
+                    "stringValue": "{\"description\": \"What is an agent trace?\", \"name\": \"What is an agent trace?\", \"expected_output\": \"A short answer\", \"summary\": \"What is an agent trace?...\", \"raw\": \"An **agent trace** is a chronological record of an AI agent\u2019s execution: the steps it took, such as receiving input, calling tools, observing results, and producing an answer. Traces help developers understand, debug, and evaluate an agent\u2019s behavior.\", \"pydantic\": null, \"json_dict\": null, \"agent\": \"research_agent\", \"output_format\": \"raw\", \"messages\": [{\"role\": \"system\", \"content\": \"You are research_agent. You explain technical concepts.\\nYour personal goal is: Answer questions clearly\"}, {\"role\": \"user\", \"content\": \"\\nCurrent Task: What is an agent trace?\\n\\nThis is the expected criteria for your final answer: A short answer\\nyou MUST return the actual complete content as the final answer, not a summary.\\n\\nProvide your complete response:\"}], \"tool_failures\": []}"
                   }
                 },
                 {
@@ -285,17 +285,17 @@
               "flags": 256
             },
             {
-              "traceId": "110c44d444b7742cfa57fc70cef424d8",
-              "spanId": "b2637dab2e2fc2ca",
+              "traceId": "9423efa0faa4a873d896222c6b33aacc",
+              "spanId": "1edfe2e36e867117",
               "name": "research_crew.kickoff",
               "kind": 1,
-              "startTimeUnixNano": "1791012936330034002",
-              "endTimeUnixNano": "1791012938279104760",
+              "startTimeUnixNano": "1791061313393493000",
+              "endTimeUnixNano": "1791061315827947000",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"question\":\"What is an agent trace?\"}"
+                    "stringValue": "{\"question\": \"What is an agent trace?\"}"
                   }
                 },
                 {
@@ -313,31 +313,31 @@
                 {
                   "key": "crew_id",
                   "value": {
-                    "stringValue": "75706cf9-6de1-475e-8e6b-dc145c5d54d6"
+                    "stringValue": "e829105c-c95a-4f8e-aea4-4a98b6c84b13"
                   }
                 },
                 {
                   "key": "crew_inputs",
                   "value": {
-                    "stringValue": "{\"question\":\"What is an agent trace?\"}"
+                    "stringValue": "{\"question\": \"What is an agent trace?\"}"
                   }
                 },
                 {
                   "key": "crew_agents",
                   "value": {
-                    "stringValue": "[{\"key\":\"4dc9e33a8c7925f0f322ab0a8886aef0\",\"id\":\"af3484b5-92a5-4a01-acf7-0e759af2f968\",\"role\":\"research_agent\",\"goal\":\"Answer questions clearly\",\"backstory\":\"You explain technical concepts.\",\"verbose?\":false,\"max_iter\":25,\"max_rpm\":null,\"delegation_enabled\":false,\"tools_names\":[]}]"
+                    "stringValue": "[{\"key\": \"4dc9e33a8c7925f0f322ab0a8886aef0\", \"id\": \"85ad8f68-68d9-4510-9367-af4d6697cb24\", \"role\": \"research_agent\", \"goal\": \"Answer questions clearly\", \"backstory\": \"You explain technical concepts.\", \"verbose?\": false, \"max_iter\": 25, \"max_rpm\": null, \"delegation_enabled\": false, \"tools_names\": []}]"
                   }
                 },
                 {
                   "key": "crew_tasks",
                   "value": {
-                    "stringValue": "[{\"id\":\"583ee87b-ad8f-42b0-9354-78036e8c3875\",\"description\":\"{question}\",\"expected_output\":\"A short answer\",\"async_execution?\":false,\"human_input?\":false,\"agent_role\":\"research_agent\",\"agent_key\":\"4dc9e33a8c7925f0f322ab0a8886aef0\",\"context\":null,\"tools_names\":[]}]"
+                    "stringValue": "[{\"id\": \"be50aff3-19df-4772-99d7-5bd96218c0f6\", \"description\": \"{question}\", \"expected_output\": \"A short answer\", \"async_execution?\": false, \"human_input?\": false, \"agent_role\": \"research_agent\", \"agent_key\": \"4dc9e33a8c7925f0f322ab0a8886aef0\", \"context\": null, \"tools_names\": []}]"
                   }
                 },
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"raw\":\"An **agent trace** is a chronological record of an AI agent’s actions and observations while completing a task—such as its inputs, tool calls, tool results, and final response. It helps explain, debug, and evaluate the agent’s behavior.\",\"pydantic\":null,\"json_dict\":null,\"tasks_output\":[{\"description\":\"What is an agent trace?\",\"name\":\"What is an agent trace?\",\"expected_output\":\"A short answer\",\"summary\":\"What is an agent trace?...\",\"raw\":\"An **agent trace** is a chronological record of an AI agent’s actions and observations while completing a task—such as its inputs, tool calls, tool results, and final response. It helps explain, debug, and evaluate the agent’s behavior.\",\"pydantic\":null,\"json_dict\":null,\"agent\":\"research_agent\",\"output_format\":\"raw\",\"messages\":[{\"role\":\"system\",\"content\":\"You are research_agent. You explain technical concepts.\\nYour personal goal is: Answer questions clearly\"},{\"role\":\"user\",\"content\":\"\\nCurrent Task: What is an agent trace?\\n\\nThis is the expected criteria for your final answer: A short answer\\nyou MUST return the actual complete content as the final answer, not a summary.\\n\\nProvide your complete response:\"}],\"tool_failures\":[]}],\"token_usage\":{\"total_tokens\":159,\"prompt_tokens\":73,\"cached_prompt_tokens\":0,\"completion_tokens\":86,\"reasoning_tokens\":27,\"cache_creation_tokens\":0,\"successful_requests\":1}}"
+                    "stringValue": "{\"raw\": \"An **agent trace** is a chronological record of an AI agent\u2019s execution: the steps it took, such as receiving input, calling tools, observing results, and producing an answer. Traces help developers understand, debug, and evaluate an agent\u2019s behavior.\", \"pydantic\": null, \"json_dict\": null, \"tasks_output\": [{\"description\": \"What is an agent trace?\", \"name\": \"What is an agent trace?\", \"expected_output\": \"A short answer\", \"summary\": \"What is an agent trace?...\", \"raw\": \"An **agent trace** is a chronological record of an AI agent\u2019s execution: the steps it took, such as receiving input, calling tools, observing results, and producing an answer. Traces help developers understand, debug, and evaluate an agent\u2019s behavior.\", \"pydantic\": null, \"json_dict\": null, \"agent\": \"research_agent\", \"output_format\": \"raw\", \"messages\": [{\"role\": \"system\", \"content\": \"You are research_agent. You explain technical concepts.\\nYour personal goal is: Answer questions clearly\"}, {\"role\": \"user\", \"content\": \"\\nCurrent Task: What is an agent trace?\\n\\nThis is the expected criteria for your final answer: A short answer\\nyou MUST return the actual complete content as the final answer, not a summary.\\n\\nProvide your complete response:\"}], \"tool_failures\": []}], \"token_usage\": {\"total_tokens\": 195, \"prompt_tokens\": 73, \"cached_prompt_tokens\": 0, \"completion_tokens\": 122, \"reasoning_tokens\": 61, \"cache_creation_tokens\": 0, \"successful_requests\": 1}}"
                   }
                 },
                 {

litellm-rust/crates/traces/tests/fixtures/google_adk_retry.json

@@ -0,0 +1,576 @@
+{
+  "resourceSpans": [
+    {
+      "resource": {
+        "attributes": [
+          {
+            "key": "telemetry.sdk.language",
+            "value": {
+              "stringValue": "python"
+            }
+          },
+          {
+            "key": "telemetry.sdk.name",
+            "value": {
+              "stringValue": "opentelemetry"
+            }
+          },
+          {
+            "key": "telemetry.sdk.version",
+            "value": {
+              "stringValue": "1.42.1"
+            }
+          },
+          {
+            "key": "telemetry.auto.version",
+            "value": {
+              "stringValue": "0.63b1"
+            }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service"
+            }
+          }
+        ]
+      },
+      "scopeSpans": [
+        {
+          "scope": {
+            "name": "litellm.gateway.client"
+          },
+          "spans": [
+            {
+              "traceId": "cdd888fb2417b10d57e2b2cee4dd428c",
+              "spanId": "977a62497f2dda9d",
+              "parentSpanId": "1a4892aa3fae3307",
+              "name": "gateway.request",
+              "kind": 3,
+              "startTimeUnixNano": "1791061574509820000",
+              "endTimeUnixNano": "1791061576009220000",
+              "attributes": [
+                {
+                  "key": "litellm.gateway.attempt",
+                  "value": {
+                    "boolValue": true
+                  }
+                },
+                {
+                  "key": "http.request.method",
+                  "value": {
+                    "stringValue": "POST"
+                  }
+                },
+                {
+                  "key": "url.full",
+                  "value": {
+                    "stringValue": "http://localhost:4002/v1/chat/completions"
+                  }
+                },
+                {
+                  "key": "server.address",
+                  "value": {
+                    "stringValue": "localhost"
+                  }
+                },
+                {
+                  "key": "http.response.status_code",
+                  "value": {
+                    "intValue": "503"
+                  }
+                },
+                {
+                  "key": "litellm.call_id",
+                  "value": {
+                    "stringValue": "4d1e094d-24e7-49f6-8d33-656a7d7712ae"
+                  }
+                }
+              ],
+              "status": {
+                "code": 2
+              },
+              "flags": 256
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "resource": {
+        "attributes": [
+          {
+            "key": "telemetry.sdk.language",
+            "value": {
+              "stringValue": "python"
+            }
+          },
+          {
+            "key": "telemetry.sdk.name",
+            "value": {
+              "stringValue": "opentelemetry"
+            }
+          },
+          {
+            "key": "telemetry.sdk.version",
+            "value": {
+              "stringValue": "1.42.1"
+            }
+          },
+          {
+            "key": "telemetry.auto.version",
+            "value": {
+              "stringValue": "0.63b1"
+            }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service"
+            }
+          }
+        ]
+      },
+      "scopeSpans": [
+        {
+          "scope": {
+            "name": "litellm.gateway.client"
+          },
+          "spans": [
+            {
+              "traceId": "cdd888fb2417b10d57e2b2cee4dd428c",
+              "spanId": "2856caaa423beba1",
+              "parentSpanId": "1a4892aa3fae3307",
+              "name": "gateway.request",
+              "kind": 3,
+              "startTimeUnixNano": "1791061576507921000",
+              "endTimeUnixNano": "1791061577761649000",
+              "attributes": [
+                {
+                  "key": "litellm.gateway.attempt",
+                  "value": {
+                    "boolValue": true
+                  }
+                },
+                {
+                  "key": "http.request.method",
+                  "value": {
+                    "stringValue": "POST"
+                  }
+                },
+                {
+                  "key": "url.full",
+                  "value": {
+                    "stringValue": "http://localhost:4002/v1/chat/completions"
+                  }
+                },
+                {
+                  "key": "server.address",
+                  "value": {
+                    "stringValue": "localhost"
+                  }
+                },
+                {
+                  "key": "http.response.status_code",
+                  "value": {
+                    "intValue": "200"
+                  }
+                },
+                {
+                  "key": "litellm.call_id",
+                  "value": {
+                    "stringValue": "38acfc26-d8f3-4adc-9528-0196c28b8640"
+                  }
+                }
+              ],
+              "status": {},
+              "flags": 256
+            }
+          ]
+        },
+        {
+          "scope": {
+            "name": "openinference.instrumentation.google_adk",
+            "version": "1.0.2"
+          },
+          "spans": [
+            {
+              "traceId": "cdd888fb2417b10d57e2b2cee4dd428c",
+              "spanId": "1a4892aa3fae3307",
+              "parentSpanId": "dff96cfbe5a5cb73",
+              "name": "call_llm",
+              "kind": 1,
+              "startTimeUnixNano": "1791061573142809000",
+              "endTimeUnixNano": "1791061577770765000",
+              "attributes": [
+                {
+                  "key": "session.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "debug_user_id"
+                  }
+                },
+                {
+                  "key": "gen_ai.operation.name",
+                  "value": {
+                    "stringValue": "generate_content"
+                  }
+                },
+                {
+                  "key": "gen_ai.agent.name",
+                  "value": {
+                    "stringValue": "research_agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.conversation.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "gen_ai.system",
+                  "value": {
+                    "stringValue": "gcp.vertex.agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.request.model",
+                  "value": {
+                    "stringValue": "openai/openai/gpt-6-luna"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.invocation_id",
+                  "value": {
+                    "stringValue": "e-de6dbbd8-e192-415a-a15c-24e216046237"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.session_id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.event_id",
+                  "value": {
+                    "stringValue": "e5d9e1dd-20e8-4715-96db-dcd96a5f4e35"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.llm_request",
+                  "value": {
+                    "stringValue": "{\"model\": \"openai/openai/gpt-6-luna\", \"config\": {\"system_instruction\": \"You are an agent. Your internal name is \\\"research_agent\\\".\", \"labels\": {\"adk_agent_name\": \"research_agent\"}}, \"contents\": [{\"parts\": [{\"text\": \"Reply with one short sentence about agent traces.\"}], \"role\": \"user\"}]}"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.llm_response",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"Agent traces show the steps an agent takes to complete a task.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":47,\"prompt_token_count\":32,\"thoughts_token_count\":25,\"total_token_count\":79}}"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.input_tokens",
+                  "value": {
+                    "intValue": "32"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.output_tokens",
+                  "value": {
+                    "intValue": "72"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.cache_read.input_tokens",
+                  "value": {
+                    "intValue": "0"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.reasoning.output_tokens",
+                  "value": {
+                    "intValue": "25"
+                  }
+                },
+                {
+                  "key": "gen_ai.response.finish_reasons",
+                  "value": {
+                    "arrayValue": {
+                      "values": [
+                        {
+                          "stringValue": "stop"
+                        }
+                      ]
+                    }
+                  }
+                },
+                {
+                  "key": "llm.provider",
+                  "value": {
+                    "stringValue": "google"
+                  }
+                },
+                {
+                  "key": "input.value",
+                  "value": {
+                    "stringValue": "{\"model\":\"openai/openai/gpt-6-luna\",\"contents\":[{\"parts\":[{\"text\":\"Reply with one short sentence about agent traces.\"}],\"role\":\"user\"}],\"config\":{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}},\"live_connect_config\":{\"input_audio_transcription\":{},\"output_audio_transcription\":{}}}"
+                  }
+                },
+                {
+                  "key": "input.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "llm.model_name",
+                  "value": {
+                    "stringValue": "openai/openai/gpt-6-luna"
+                  }
+                },
+                {
+                  "key": "llm.invocation_parameters",
+                  "value": {
+                    "stringValue": "{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}}"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.0.message.role",
+                  "value": {
+                    "stringValue": "system"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.0.message.content",
+                  "value": {
+                    "stringValue": "You are an agent. Your internal name is \"research_agent\"."
+                  }
+                },
+                {
+                  "key": "llm.input_messages.1.message.role",
+                  "value": {
+                    "stringValue": "user"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.1.message.contents.0.message_content.text",
+                  "value": {
+                    "stringValue": "Reply with one short sentence about agent traces."
+                  }
+                },
+                {
+                  "key": "llm.input_messages.1.message.contents.0.message_content.type",
+                  "value": {
+                    "stringValue": "text"
+                  }
+                },
+                {
+                  "key": "output.value",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"Agent traces show the steps an agent takes to complete a task.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":47,\"prompt_token_count\":32,\"thoughts_token_count\":25,\"total_token_count\":79}}"
+                  }
+                },
+                {
+                  "key": "output.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "llm.token_count.total",
+                  "value": {
+                    "intValue": "79"
+                  }
+                },
+                {
+                  "key": "llm.token_count.prompt",
+                  "value": {
+                    "intValue": "32"
+                  }
+                },
+                {
+                  "key": "llm.token_count.completion_details.reasoning",
+                  "value": {
+                    "intValue": "25"
+                  }
+                },
+                {
+                  "key": "llm.token_count.completion",
+                  "value": {
+                    "intValue": "47"
+                  }
+                },
+                {
+                  "key": "llm.output_messages.0.message.role",
+                  "value": {
+                    "stringValue": "model"
+                  }
+                },
+                {
+                  "key": "llm.output_messages.0.message.contents.0.message_content.text",
+                  "value": {
+                    "stringValue": "Agent traces show the steps an agent takes to complete a task."
+                  }
+                },
+                {
+                  "key": "llm.output_messages.0.message.contents.0.message_content.type",
+                  "value": {
+                    "stringValue": "text"
+                  }
+                },
+                {
+                  "key": "openinference.span.kind",
+                  "value": {
+                    "stringValue": "LLM"
+                  }
+                }
+              ],
+              "status": {
+                "code": 1
+              },
+              "flags": 256
+            },
+            {
+              "traceId": "cdd888fb2417b10d57e2b2cee4dd428c",
+              "spanId": "dff96cfbe5a5cb73",
+              "parentSpanId": "1effa85408e9e4f7",
+              "name": "agent_run [research_agent]",
+              "kind": 1,
+              "startTimeUnixNano": "1791061573129211000",
+              "endTimeUnixNano": "1791061577770883000",
+              "attributes": [
+                {
+                  "key": "agent.name",
+                  "value": {
+                    "stringValue": "research_agent"
+                  }
+                },
+                {
+                  "key": "session.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "debug_user_id"
+                  }
+                },
+                {
+                  "key": "gen_ai.operation.name",
+                  "value": {
+                    "stringValue": "invoke_agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.agent.description",
+                  "value": {
+                    "stringValue": ""
+                  }
+                },
+                {
+                  "key": "gen_ai.agent.name",
+                  "value": {
+                    "stringValue": "research_agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.conversation.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "output.value",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"Agent traces show the steps an agent takes to complete a task.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":47,\"prompt_token_count\":32,\"thoughts_token_count\":25,\"total_token_count\":79},\"invocation_id\":\"e-de6dbbd8-e192-415a-a15c-24e216046237\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"\"},\"id\":\"e5d9e1dd-20e8-4715-96db-dcd96a5f4e35\",\"timestamp\":1791061573.14271}"
+                  }
+                },
+                {
+                  "key": "output.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "openinference.span.kind",
+                  "value": {
+                    "stringValue": "AGENT"
+                  }
+                }
+              ],
+              "status": {
+                "code": 1
+              },
+              "flags": 256
+            },
+            {
+              "traceId": "cdd888fb2417b10d57e2b2cee4dd428c",
+              "spanId": "1effa85408e9e4f7",
+              "name": "invocation [research_app]",
+              "kind": 1,
+              "startTimeUnixNano": "1791061573075243000",
+              "endTimeUnixNano": "1791061577771095000",
+              "attributes": [
+                {
+                  "key": "input.value",
+                  "value": {
+                    "stringValue": "{\"user_id\": \"debug_user_id\", \"session_id\": \"debug_session_id\", \"invocation_id\": null, \"new_message\": {\"parts\": [{\"text\": \"Reply with one short sentence about agent traces.\"}], \"role\": \"user\"}, \"state_delta\": null, \"run_config\": {\"save_input_blobs_as_artifacts\": false, \"support_cfc\": false, \"streaming_mode\": \"StreamingMode.NONE\", \"output_audio_transcription\": {}, \"input_audio_transcription\": {}, \"save_live_blob\": false, \"save_live_audio\": false, \"max_llm_calls\": 500, \"include_thoughts_from_other_agents\": false}, \"yield_user_message\": false, \"abort_signal\": null}"
+                  }
+                },
+                {
+                  "key": "input.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "debug_user_id"
+                  }
+                },
+                {
+                  "key": "session.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "output.value",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"Agent traces show the steps an agent takes to complete a task.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":47,\"prompt_token_count\":32,\"thoughts_token_count\":25,\"total_token_count\":79},\"invocation_id\":\"e-de6dbbd8-e192-415a-a15c-24e216046237\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"research_agent@1\"},\"id\":\"e5d9e1dd-20e8-4715-96db-dcd96a5f4e35\",\"timestamp\":1791061573.14271}"
+                  }
+                },
+                {
+                  "key": "output.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "openinference.span.kind",
+                  "value": {
+                    "stringValue": "CHAIN"
+                  }
+                }
+              ],
+              "status": {
+                "code": 1
+              },
+              "flags": 256
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

litellm-rust/crates/traces/tests/fixtures/google_adk_simple.json

@@ -21,21 +21,77 @@
               "stringValue": "1.42.1"
             }
           },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "google-adk-simple"
-            }
-          },
           {
             "key": "telemetry.auto.version",
             "value": {
               "stringValue": "0.63b1"
             }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service"
+            }
           }
         ]
       },
       "scopeSpans": [
+        {
+          "scope": {
+            "name": "litellm.gateway.client"
+          },
+          "spans": [
+            {
+              "traceId": "247bc18e08a66f0b54a024928fd70ccb",
+              "spanId": "6c636a3189b1dc12",
+              "parentSpanId": "69c1827a773fcdf9",
+              "name": "gateway.request",
+              "kind": 3,
+              "startTimeUnixNano": "1791061383698325000",
+              "endTimeUnixNano": "1791061386697226000",
+              "attributes": [
+                {
+                  "key": "litellm.gateway.attempt",
+                  "value": {
+                    "boolValue": true
+                  }
+                },
+                {
+                  "key": "http.request.method",
+                  "value": {
+                    "stringValue": "POST"
+                  }
+                },
+                {
+                  "key": "url.full",
+                  "value": {
+                    "stringValue": "http://localhost:4002/v1/chat/completions"
+                  }
+                },
+                {
+                  "key": "server.address",
+                  "value": {
+                    "stringValue": "localhost"
+                  }
+                },
+                {
+                  "key": "http.response.status_code",
+                  "value": {
+                    "intValue": "200"
+                  }
+                },
+                {
+                  "key": "litellm.call_id",
+                  "value": {
+                    "stringValue": "0a5203cf-e0f4-4131-b2ec-310740539396"
+                  }
+                }
+              ],
+              "status": {},
+              "flags": 256
+            }
+          ]
+        },
         {
           "scope": {
             "name": "openinference.instrumentation.google_adk",
@@ -43,13 +99,13 @@
           },
           "spans": [
             {
-              "traceId": "df61d220386ef57406d1eebb19dd6599",
-              "spanId": "cb6d07f7e2960614",
-              "parentSpanId": "52ac80deae53913e",
+              "traceId": "247bc18e08a66f0b54a024928fd70ccb",
+              "spanId": "69c1827a773fcdf9",
+              "parentSpanId": "74dcd13708e96513",
               "name": "call_llm",
               "kind": 1,
-              "startTimeUnixNano": "1791012833504174130",
-              "endTimeUnixNano": "1791012836291324943",
+              "startTimeUnixNano": "1791061382042896000",
+              "endTimeUnixNano": "1791061386709208000",
               "attributes": [
                 {
                   "key": "session.id",
@@ -90,13 +146,13 @@
                 {
                   "key": "gen_ai.request.model",
                   "value": {
-                    "stringValue": "litellm_proxy/openai/gpt-6-luna"
+                    "stringValue": "openai/openai/gpt-6-luna"
                   }
                 },
                 {
                   "key": "gcp.vertex.agent.invocation_id",
                   "value": {
-                    "stringValue": "e-01b1f385-a6b8-4122-a535-1cdfc38d3c6f"
+                    "stringValue": "e-87a3570c-297c-4a96-8f12-2edcc67b26ff"
                   }
                 },
                 {
@@ -108,19 +164,19 @@
                 {
                   "key": "gcp.vertex.agent.event_id",
                   "value": {
-                    "stringValue": "eb613b2d-4ceb-4949-89a5-730ca9a71b9a"
+                    "stringValue": "02b35dc6-0c26-4ecb-81a3-e2ca9947b98e"
                   }
                 },
                 {
                   "key": "gcp.vertex.agent.llm_request",
                   "value": {
-                    "stringValue": "{\"model\":\"litellm_proxy/openai/gpt-6-luna\",\"config\":{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}},\"contents\":[{\"parts\":[{\"text\":\"What is an agent trace?\"}],\"role\":\"user\"}]}"
+                    "stringValue": "{\"model\": \"openai/openai/gpt-6-luna\", \"config\": {\"system_instruction\": \"You are an agent. Your internal name is \\\"research_agent\\\".\", \"labels\": {\"adk_agent_name\": \"research_agent\"}}, \"contents\": [{\"parts\": [{\"text\": \"What is an agent trace?\"}], \"role\": \"user\"}]}"
                   }
                 },
                 {
                   "key": "gcp.vertex.agent.llm_response",
                   "value": {
-                    "stringValue": "{\"model_version\":\"litellm_proxy/openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent’s run: the sequence of steps it took and what happened at each step. It may include the input, tool calls and their results, intermediate outputs, timing, and errors.\\n\\nTraces help developers debug, evaluate, and monitor an agent. They don’t necessarily contain the agent’s private reasoning; a trace can record observable actions and brief summaries instead.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":177,\"prompt_token_count\":29,\"thoughts_token_count\":85,\"total_token_count\":206}}"
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a chronological record of an AI agent\u2019s work on a task. It may include the user\u2019s request, the agent\u2019s actions, tool calls and their results, and the final response.\\n\\nTraces help developers debug behavior, evaluate performance, and audit what happened. They don\u2019t necessarily include the agent\u2019s full internal reasoning; often they contain only observable steps and outputs.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":168,\"prompt_token_count\":29,\"thoughts_token_count\":80,\"total_token_count\":197}}"
                   }
                 },
                 {
@@ -132,7 +188,7 @@
                 {
                   "key": "gen_ai.usage.output_tokens",
                   "value": {
-                    "intValue": "262"
+                    "intValue": "248"
                   }
                 },
                 {
@@ -144,7 +200,7 @@
                 {
                   "key": "gen_ai.usage.reasoning.output_tokens",
                   "value": {
-                    "intValue": "85"
+                    "intValue": "80"
                   }
                 },
                 {
@@ -168,7 +224,7 @@
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"model\":\"litellm_proxy/openai/gpt-6-luna\",\"contents\":[{\"parts\":[{\"text\":\"What is an agent trace?\"}],\"role\":\"user\"}],\"config\":{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}},\"live_connect_config\":{\"input_audio_transcription\":{},\"output_audio_transcription\":{}}}"
+                    "stringValue": "{\"model\":\"openai/openai/gpt-6-luna\",\"contents\":[{\"parts\":[{\"text\":\"What is an agent trace?\"}],\"role\":\"user\"}],\"config\":{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}},\"live_connect_config\":{\"input_audio_transcription\":{},\"output_audio_transcription\":{}}}"
                   }
                 },
                 {
@@ -180,7 +236,7 @@
                 {
                   "key": "llm.model_name",
                   "value": {
-                    "stringValue": "litellm_proxy/openai/gpt-6-luna"
+                    "stringValue": "openai/openai/gpt-6-luna"
                   }
                 },
                 {
@@ -222,7 +278,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"model_version\":\"litellm_proxy/openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent’s run: the sequence of steps it took and what happened at each step. It may include the input, tool calls and their results, intermediate outputs, timing, and errors.\\n\\nTraces help developers debug, evaluate, and monitor an agent. They don’t necessarily contain the agent’s private reasoning; a trace can record observable actions and brief summaries instead.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":177,\"prompt_token_count\":29,\"thoughts_token_count\":85,\"total_token_count\":206}}"
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a chronological record of an AI agent\u2019s work on a task. It may include the user\u2019s request, the agent\u2019s actions, tool calls and their results, and the final response.\\n\\nTraces help developers debug behavior, evaluate performance, and audit what happened. They don\u2019t necessarily include the agent\u2019s full internal reasoning; often they contain only observable steps and outputs.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":168,\"prompt_token_count\":29,\"thoughts_token_count\":80,\"total_token_count\":197}}"
                   }
                 },
                 {
@@ -234,7 +290,7 @@
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "206"
+                    "intValue": "197"
                   }
                 },
                 {
@@ -246,13 +302,13 @@
                 {
                   "key": "llm.token_count.completion_details.reasoning",
                   "value": {
-                    "intValue": "85"
+                    "intValue": "80"
                   }
                 },
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "177"
+                    "intValue": "168"
                   }
                 },
                 {
@@ -264,7 +320,7 @@
                 {
                   "key": "llm.output_messages.0.message.contents.0.message_content.text",
                   "value": {
-                    "stringValue": "An **agent trace** is a record of an AI agent’s run: the sequence of steps it took and what happened at each step. It may include the input, tool calls and their results, intermediate outputs, timing, and errors.\n\nTraces help developers debug, evaluate, and monitor an agent. They don’t necessarily contain the agent’s private reasoning; a trace can record observable actions and brief summaries instead."
+                    "stringValue": "An **agent trace** is a chronological record of an AI agent\u2019s work on a task. It may include the user\u2019s request, the agent\u2019s actions, tool calls and their results, and the final response.\n\nTraces help developers debug behavior, evaluate performance, and audit what happened. They don\u2019t necessarily include the agent\u2019s full internal reasoning; often they contain only observable steps and outputs."
                   }
                 },
                 {
@@ -273,12 +329,6 @@
                     "stringValue": "text"
                   }
                 },
-                {
-                  "key": "gen_ai.response.id",
-                  "value": {
-                    "stringValue": "chatcmpl-EUoXinBzYIEZupgGwtpL85Z1kunV8"
-                  }
-                },
                 {
                   "key": "openinference.span.kind",
                   "value": {
@@ -292,13 +342,13 @@
               "flags": 256
             },
             {
-              "traceId": "df61d220386ef57406d1eebb19dd6599",
-              "spanId": "52ac80deae53913e",
-              "parentSpanId": "b304248bca94d81a",
+              "traceId": "247bc18e08a66f0b54a024928fd70ccb",
+              "spanId": "74dcd13708e96513",
+              "parentSpanId": "e6d2e44b3d2d9d6d",
               "name": "agent_run [research_agent]",
               "kind": 1,
-              "startTimeUnixNano": "1791012833480201860",
-              "endTimeUnixNano": "1791012836291519273",
+              "startTimeUnixNano": "1791061382027984000",
+              "endTimeUnixNano": "1791061386709354000",
               "attributes": [
                 {
                   "key": "agent.name",
@@ -345,7 +395,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"model_version\":\"litellm_proxy/openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent’s run: the sequence of steps it took and what happened at each step. It may include the input, tool calls and their results, intermediate outputs, timing, and errors.\\n\\nTraces help developers debug, evaluate, and monitor an agent. They don’t necessarily contain the agent’s private reasoning; a trace can record observable actions and brief summaries instead.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":177,\"prompt_token_count\":29,\"thoughts_token_count\":85,\"total_token_count\":206},\"invocation_id\":\"e-01b1f385-a6b8-4122-a535-1cdfc38d3c6f\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"\"},\"id\":\"eb613b2d-4ceb-4949-89a5-730ca9a71b9a\",\"timestamp\":1791012833.504099}"
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a chronological record of an AI agent\u2019s work on a task. It may include the user\u2019s request, the agent\u2019s actions, tool calls and their results, and the final response.\\n\\nTraces help developers debug behavior, evaluate performance, and audit what happened. They don\u2019t necessarily include the agent\u2019s full internal reasoning; often they contain only observable steps and outputs.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":168,\"prompt_token_count\":29,\"thoughts_token_count\":80,\"total_token_count\":197},\"invocation_id\":\"e-87a3570c-297c-4a96-8f12-2edcc67b26ff\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"\"},\"id\":\"02b35dc6-0c26-4ecb-81a3-e2ca9947b98e\",\"timestamp\":1791061382.04284}"
                   }
                 },
                 {
@@ -367,17 +417,17 @@
               "flags": 256
             },
             {
-              "traceId": "df61d220386ef57406d1eebb19dd6599",
-              "spanId": "b304248bca94d81a",
+              "traceId": "247bc18e08a66f0b54a024928fd70ccb",
+              "spanId": "e6d2e44b3d2d9d6d",
               "name": "invocation [research_app]",
               "kind": 1,
-              "startTimeUnixNano": "1791012833360078724",
-              "endTimeUnixNano": "1791012836291729271",
+              "startTimeUnixNano": "1791061381946497000",
+              "endTimeUnixNano": "1791061386709575000",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"user_id\":\"debug_user_id\",\"session_id\":\"debug_session_id\",\"invocation_id\":null,\"new_message\":{\"parts\":[{\"text\":\"What is an agent trace?\"}],\"role\":\"user\"},\"state_delta\":null,\"run_config\":{\"save_input_blobs_as_artifacts\":false,\"support_cfc\":false,\"streaming_mode\":\"StreamingMode.NONE\",\"output_audio_transcription\":{},\"input_audio_transcription\":{},\"save_live_blob\":false,\"save_live_audio\":false,\"max_llm_calls\":500,\"include_thoughts_from_other_agents\":false},\"yield_user_message\":false,\"abort_signal\":null}"
+                    "stringValue": "{\"user_id\": \"debug_user_id\", \"session_id\": \"debug_session_id\", \"invocation_id\": null, \"new_message\": {\"parts\": [{\"text\": \"What is an agent trace?\"}], \"role\": \"user\"}, \"state_delta\": null, \"run_config\": {\"save_input_blobs_as_artifacts\": false, \"support_cfc\": false, \"streaming_mode\": \"StreamingMode.NONE\", \"output_audio_transcription\": {}, \"input_audio_transcription\": {}, \"save_live_blob\": false, \"save_live_audio\": false, \"max_llm_calls\": 500, \"include_thoughts_from_other_agents\": false}, \"yield_user_message\": false, \"abort_signal\": null}"
                   }
                 },
                 {
@@ -401,7 +451,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"model_version\":\"litellm_proxy/openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent’s run: the sequence of steps it took and what happened at each step. It may include the input, tool calls and their results, intermediate outputs, timing, and errors.\\n\\nTraces help developers debug, evaluate, and monitor an agent. They don’t necessarily contain the agent’s private reasoning; a trace can record observable actions and brief summaries instead.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":177,\"prompt_token_count\":29,\"thoughts_token_count\":85,\"total_token_count\":206},\"invocation_id\":\"e-01b1f385-a6b8-4122-a535-1cdfc38d3c6f\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"research_agent@1\"},\"id\":\"eb613b2d-4ceb-4949-89a5-730ca9a71b9a\",\"timestamp\":1791012833.504099}"
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a chronological record of an AI agent\u2019s work on a task. It may include the user\u2019s request, the agent\u2019s actions, tool calls and their results, and the final response.\\n\\nTraces help developers debug behavior, evaluate performance, and audit what happened. They don\u2019t necessarily include the agent\u2019s full internal reasoning; often they contain only observable steps and outputs.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":168,\"prompt_token_count\":29,\"thoughts_token_count\":80,\"total_token_count\":197},\"invocation_id\":\"e-87a3570c-297c-4a96-8f12-2edcc67b26ff\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"research_agent@1\"},\"id\":\"02b35dc6-0c26-4ecb-81a3-e2ca9947b98e\",\"timestamp\":1791061382.04284}"
                   }
                 },
                 {

litellm-rust/crates/traces/tests/fixtures/google_adk_stream.json

@@ -0,0 +1,480 @@
+{
+  "resourceSpans": [
+    {
+      "resource": {
+        "attributes": [
+          {
+            "key": "telemetry.sdk.language",
+            "value": {
+              "stringValue": "python"
+            }
+          },
+          {
+            "key": "telemetry.sdk.name",
+            "value": {
+              "stringValue": "opentelemetry"
+            }
+          },
+          {
+            "key": "telemetry.sdk.version",
+            "value": {
+              "stringValue": "1.42.1"
+            }
+          },
+          {
+            "key": "telemetry.auto.version",
+            "value": {
+              "stringValue": "0.63b1"
+            }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service"
+            }
+          }
+        ]
+      },
+      "scopeSpans": [
+        {
+          "scope": {
+            "name": "litellm.gateway.client"
+          },
+          "spans": [
+            {
+              "traceId": "59fb70ffc358dfe30a5e0d4615e4ca7f",
+              "spanId": "fc3bcda50363d82c",
+              "parentSpanId": "90b23ff6b4199a4b",
+              "name": "gateway.request",
+              "kind": 3,
+              "startTimeUnixNano": "1791061463015671000",
+              "endTimeUnixNano": "1791061465614543000",
+              "attributes": [
+                {
+                  "key": "litellm.gateway.attempt",
+                  "value": {
+                    "boolValue": true
+                  }
+                },
+                {
+                  "key": "http.request.method",
+                  "value": {
+                    "stringValue": "POST"
+                  }
+                },
+                {
+                  "key": "url.full",
+                  "value": {
+                    "stringValue": "http://localhost:4002/v1/chat/completions"
+                  }
+                },
+                {
+                  "key": "server.address",
+                  "value": {
+                    "stringValue": "localhost"
+                  }
+                },
+                {
+                  "key": "http.response.status_code",
+                  "value": {
+                    "intValue": "200"
+                  }
+                },
+                {
+                  "key": "litellm.call_id",
+                  "value": {
+                    "stringValue": "73d0b177-86e5-4e11-9704-bff2667a0a75"
+                  }
+                }
+              ],
+              "status": {},
+              "flags": 256
+            }
+          ]
+        },
+        {
+          "scope": {
+            "name": "openinference.instrumentation.google_adk",
+            "version": "1.0.2"
+          },
+          "spans": [
+            {
+              "traceId": "59fb70ffc358dfe30a5e0d4615e4ca7f",
+              "spanId": "90b23ff6b4199a4b",
+              "parentSpanId": "a8c9e8a324ce58ff",
+              "name": "call_llm",
+              "kind": 1,
+              "startTimeUnixNano": "1791061461482820000",
+              "endTimeUnixNano": "1791061465621507000",
+              "attributes": [
+                {
+                  "key": "session.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "debug_user_id"
+                  }
+                },
+                {
+                  "key": "gen_ai.operation.name",
+                  "value": {
+                    "stringValue": "generate_content"
+                  }
+                },
+                {
+                  "key": "gen_ai.agent.name",
+                  "value": {
+                    "stringValue": "research_agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.conversation.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "gen_ai.system",
+                  "value": {
+                    "stringValue": "gcp.vertex.agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.request.model",
+                  "value": {
+                    "stringValue": "openai/openai/gpt-6-luna"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.invocation_id",
+                  "value": {
+                    "stringValue": "e-135e3af9-1874-442b-a1f3-d0601dd85c31"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.session_id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.event_id",
+                  "value": {
+                    "stringValue": "a4c320a1-6fae-4364-9978-5082db9799ef"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.llm_request",
+                  "value": {
+                    "stringValue": "{\"model\": \"openai/openai/gpt-6-luna\", \"config\": {\"system_instruction\": \"You are an agent. Your internal name is \\\"research_agent\\\".\", \"labels\": {\"adk_agent_name\": \"research_agent\"}}, \"contents\": [{\"parts\": [{\"text\": \"What is an agent trace?\"}], \"role\": \"user\"}]}"
+                  }
+                },
+                {
+                  "key": "gcp.vertex.agent.llm_response",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent\u2019s execution: what it received, the actions it took (such as tool calls), the results it got back, and how the task ended.\\n\\nTraces help developers debug failures, understand behavior, and evaluate performance. They may include timestamps, inputs and outputs, tool errors, or state changes. A trace doesn\u2019t have to include the model\u2019s private reasoning; it can record only observable steps.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":184,\"prompt_token_count\":29,\"thoughts_token_count\":84,\"total_token_count\":213}}"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.input_tokens",
+                  "value": {
+                    "intValue": "29"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.output_tokens",
+                  "value": {
+                    "intValue": "268"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.cache_read.input_tokens",
+                  "value": {
+                    "intValue": "0"
+                  }
+                },
+                {
+                  "key": "gen_ai.usage.reasoning.output_tokens",
+                  "value": {
+                    "intValue": "84"
+                  }
+                },
+                {
+                  "key": "gen_ai.response.finish_reasons",
+                  "value": {
+                    "arrayValue": {
+                      "values": [
+                        {
+                          "stringValue": "stop"
+                        }
+                      ]
+                    }
+                  }
+                },
+                {
+                  "key": "llm.provider",
+                  "value": {
+                    "stringValue": "google"
+                  }
+                },
+                {
+                  "key": "input.value",
+                  "value": {
+                    "stringValue": "{\"model\":\"openai/openai/gpt-6-luna\",\"contents\":[{\"parts\":[{\"text\":\"What is an agent trace?\"}],\"role\":\"user\"}],\"config\":{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}},\"live_connect_config\":{\"input_audio_transcription\":{},\"output_audio_transcription\":{}}}"
+                  }
+                },
+                {
+                  "key": "input.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "llm.model_name",
+                  "value": {
+                    "stringValue": "openai/openai/gpt-6-luna"
+                  }
+                },
+                {
+                  "key": "llm.invocation_parameters",
+                  "value": {
+                    "stringValue": "{\"system_instruction\":\"You are an agent. Your internal name is \\\"research_agent\\\".\",\"labels\":{\"adk_agent_name\":\"research_agent\"}}"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.0.message.role",
+                  "value": {
+                    "stringValue": "system"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.0.message.content",
+                  "value": {
+                    "stringValue": "You are an agent. Your internal name is \"research_agent\"."
+                  }
+                },
+                {
+                  "key": "llm.input_messages.1.message.role",
+                  "value": {
+                    "stringValue": "user"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.1.message.contents.0.message_content.text",
+                  "value": {
+                    "stringValue": "What is an agent trace?"
+                  }
+                },
+                {
+                  "key": "llm.input_messages.1.message.contents.0.message_content.type",
+                  "value": {
+                    "stringValue": "text"
+                  }
+                },
+                {
+                  "key": "output.value",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent\u2019s execution: what it received, the actions it took (such as tool calls), the results it got back, and how the task ended.\\n\\nTraces help developers debug failures, understand behavior, and evaluate performance. They may include timestamps, inputs and outputs, tool errors, or state changes. A trace doesn\u2019t have to include the model\u2019s private reasoning; it can record only observable steps.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":184,\"prompt_token_count\":29,\"thoughts_token_count\":84,\"total_token_count\":213}}"
+                  }
+                },
+                {
+                  "key": "output.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "llm.token_count.total",
+                  "value": {
+                    "intValue": "213"
+                  }
+                },
+                {
+                  "key": "llm.token_count.prompt",
+                  "value": {
+                    "intValue": "29"
+                  }
+                },
+                {
+                  "key": "llm.token_count.completion_details.reasoning",
+                  "value": {
+                    "intValue": "84"
+                  }
+                },
+                {
+                  "key": "llm.token_count.completion",
+                  "value": {
+                    "intValue": "184"
+                  }
+                },
+                {
+                  "key": "llm.output_messages.0.message.role",
+                  "value": {
+                    "stringValue": "model"
+                  }
+                },
+                {
+                  "key": "llm.output_messages.0.message.contents.0.message_content.text",
+                  "value": {
+                    "stringValue": "An **agent trace** is a record of an AI agent\u2019s execution: what it received, the actions it took (such as tool calls), the results it got back, and how the task ended.\n\nTraces help developers debug failures, understand behavior, and evaluate performance. They may include timestamps, inputs and outputs, tool errors, or state changes. A trace doesn\u2019t have to include the model\u2019s private reasoning; it can record only observable steps."
+                  }
+                },
+                {
+                  "key": "llm.output_messages.0.message.contents.0.message_content.type",
+                  "value": {
+                    "stringValue": "text"
+                  }
+                },
+                {
+                  "key": "openinference.span.kind",
+                  "value": {
+                    "stringValue": "LLM"
+                  }
+                }
+              ],
+              "status": {
+                "code": 1
+              },
+              "flags": 256
+            },
+            {
+              "traceId": "59fb70ffc358dfe30a5e0d4615e4ca7f",
+              "spanId": "a8c9e8a324ce58ff",
+              "parentSpanId": "3f575c3cdbccf331",
+              "name": "agent_run [research_agent]",
+              "kind": 1,
+              "startTimeUnixNano": "1791061461465979000",
+              "endTimeUnixNano": "1791061465621633000",
+              "attributes": [
+                {
+                  "key": "agent.name",
+                  "value": {
+                    "stringValue": "research_agent"
+                  }
+                },
+                {
+                  "key": "session.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "debug_user_id"
+                  }
+                },
+                {
+                  "key": "gen_ai.operation.name",
+                  "value": {
+                    "stringValue": "invoke_agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.agent.description",
+                  "value": {
+                    "stringValue": ""
+                  }
+                },
+                {
+                  "key": "gen_ai.agent.name",
+                  "value": {
+                    "stringValue": "research_agent"
+                  }
+                },
+                {
+                  "key": "gen_ai.conversation.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "output.value",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent\u2019s execution: what it received, the actions it took (such as tool calls), the results it got back, and how the task ended.\\n\\nTraces help developers debug failures, understand behavior, and evaluate performance. They may include timestamps, inputs and outputs, tool errors, or state changes. A trace doesn\u2019t have to include the model\u2019s private reasoning; it can record only observable steps.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":184,\"prompt_token_count\":29,\"thoughts_token_count\":84,\"total_token_count\":213},\"invocation_id\":\"e-135e3af9-1874-442b-a1f3-d0601dd85c31\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"research_agent@1\"},\"id\":\"a4c320a1-6fae-4364-9978-5082db9799ef\",\"timestamp\":1791061465.604408}"
+                  }
+                },
+                {
+                  "key": "output.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "openinference.span.kind",
+                  "value": {
+                    "stringValue": "AGENT"
+                  }
+                }
+              ],
+              "status": {
+                "code": 1
+              },
+              "flags": 256
+            },
+            {
+              "traceId": "59fb70ffc358dfe30a5e0d4615e4ca7f",
+              "spanId": "3f575c3cdbccf331",
+              "name": "invocation [research_app]",
+              "kind": 1,
+              "startTimeUnixNano": "1791061461360524000",
+              "endTimeUnixNano": "1791061465621830000",
+              "attributes": [
+                {
+                  "key": "input.value",
+                  "value": {
+                    "stringValue": "{\"user_id\": \"debug_user_id\", \"session_id\": \"debug_session_id\", \"invocation_id\": null, \"new_message\": {\"parts\": [{\"text\": \"What is an agent trace?\"}], \"role\": \"user\"}, \"state_delta\": null, \"run_config\": {\"save_input_blobs_as_artifacts\": false, \"support_cfc\": false, \"streaming_mode\": \"StreamingMode.SSE\", \"output_audio_transcription\": {}, \"input_audio_transcription\": {}, \"save_live_blob\": false, \"save_live_audio\": false, \"max_llm_calls\": 500, \"include_thoughts_from_other_agents\": false}, \"yield_user_message\": false, \"abort_signal\": null}"
+                  }
+                },
+                {
+                  "key": "input.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "user.id",
+                  "value": {
+                    "stringValue": "debug_user_id"
+                  }
+                },
+                {
+                  "key": "session.id",
+                  "value": {
+                    "stringValue": "debug_session_id"
+                  }
+                },
+                {
+                  "key": "output.value",
+                  "value": {
+                    "stringValue": "{\"model_version\":\"openai/gpt-6-luna\",\"content\":{\"parts\":[{\"text\":\"An **agent trace** is a record of an AI agent\u2019s execution: what it received, the actions it took (such as tool calls), the results it got back, and how the task ended.\\n\\nTraces help developers debug failures, understand behavior, and evaluate performance. They may include timestamps, inputs and outputs, tool errors, or state changes. A trace doesn\u2019t have to include the model\u2019s private reasoning; it can record only observable steps.\"}],\"role\":\"model\"},\"partial\":false,\"finish_reason\":\"STOP\",\"usage_metadata\":{\"cached_content_token_count\":0,\"candidates_token_count\":184,\"prompt_token_count\":29,\"thoughts_token_count\":84,\"total_token_count\":213},\"invocation_id\":\"e-135e3af9-1874-442b-a1f3-d0601dd85c31\",\"author\":\"research_agent\",\"actions\":{\"state_delta\":{},\"artifact_delta\":{},\"requested_auth_configs\":{},\"requested_tool_confirmations\":{}},\"node_info\":{\"path\":\"research_agent@1\"},\"id\":\"a4c320a1-6fae-4364-9978-5082db9799ef\",\"timestamp\":1791061465.604408}"
+                  }
+                },
+                {
+                  "key": "output.mime_type",
+                  "value": {
+                    "stringValue": "application/json"
+                  }
+                },
+                {
+                  "key": "openinference.span.kind",
+                  "value": {
+                    "stringValue": "CHAIN"
+                  }
+                }
+              ],
+              "status": {
+                "code": 1
+              },
+              "flags": 256
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}

litellm-rust/crates/traces/tests/fixtures/langchain_simple.json

@@ -24,13 +24,7 @@
           {
             "key": "service.instance.id",
             "value": {
-              "stringValue": "857d5035-73a2-443d-a3b3-beda907e8e08"
-            }
-          },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "langchain-simple"
+              "stringValue": "3ecc2492-218f-4536-8bb8-b2c1e5ee234b"
             }
           },
           {
@@ -38,6 +32,12 @@
             "value": {
               "stringValue": "0.66b0"
             }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service:python3"
+            }
           }
         ]
       },
@@ -49,18 +49,18 @@
           },
           "spans": [
             {
-              "traceId": "fff422e2eaff0db64132f26efe387a6c",
-              "spanId": "de7f6f2c980f1dd9",
-              "parentSpanId": "462247f1c7f18034",
+              "traceId": "d9a080b530fb7f3d642b1608d157ba70",
+              "spanId": "1965258dc3bdbc38",
+              "parentSpanId": "3d757465aa89a16c",
               "name": "ChatOpenAI",
               "kind": 1,
-              "startTimeUnixNano": "1791012713671619840",
-              "endTimeUnixNano": "1791012718164809984",
+              "startTimeUnixNano": "1791061313991373056",
+              "endTimeUnixNano": "1791061316664692992",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[[{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"HumanMessage\"],\"kwargs\":{\"content\":\"What is an agent trace?\",\"type\":\"human\",\"id\":\"83d6b4d7-b3ca-4058-a516-7c884a44ac2e\"}}]]}"
+                    "stringValue": "{\"messages\": [[{\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"HumanMessage\"], \"kwargs\": {\"content\": \"What is an agent trace?\", \"type\": \"human\", \"id\": \"52e7344a-4b38-4789-8147-2a277f6c775f\"}}]]}"
                   }
                 },
                 {
@@ -72,7 +72,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"generations\":[[{\"text\":\"An **agent trace** is a record of an AI agent’s run: what it received, which actions or tools it used, what results came back, and what it ultimately produced.\\n\\nFor example:\\n\\n1. User asks for the weather.\\n2. Agent calls a weather API.\\n3. API returns the forecast.\\n4. Agent summarizes it for the user.\\n\\nTraces help developers debug and evaluate agents. They usually capture observable steps and tool interactions—not necessarily the agent’s private internal reasoning. The exact contents depend on the system.\",\"generation_info\":{\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ChatGeneration\",\"message\":{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"AIMessage\"],\"kwargs\":{\"content\":\"An **agent trace** is a record of an AI agent’s run: what it received, which actions or tools it used, what results came back, and what it ultimately produced.\\n\\nFor example:\\n\\n1. User asks for the weather.\\n2. Agent calls a weather API.\\n3. API returns the forecast.\\n4. Agent summarizes it for the user.\\n\\nTraces help developers debug and evaluate agents. They usually capture observable steps and tool interactions—not necessarily the agent’s private internal reasoning. The exact contents depend on the system.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":233,\"prompt_tokens\":12,\"total_tokens\":245,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":118,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoVmBRgBU4JGnF3vUCi895jENkSZ\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"id\":\"lc_run--01a100ad-34c7-7293-9fc1-65444c4a94ce-0\",\"usage_metadata\":{\"input_tokens\":12,\"output_tokens\":233,\"total_tokens\":245,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":118}},\"tool_calls\":[],\"invalid_tool_calls\":[]}}}]],\"llm_output\":{\"token_usage\":{\"completion_tokens\":233,\"prompt_tokens\":12,\"total_tokens\":245,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":118,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoVmBRgBU4JGnF3vUCi895jENkSZ\",\"service_tier\":\"default\"},\"run\":null,\"type\":\"LLMResult\"}"
+                    "stringValue": "{\"generations\": [[{\"text\": \"An **agent trace** is a recorded timeline of what an AI agent did to complete a task. It may include:\\n\\n- The user\u2019s request and the agent\u2019s intermediate reasoning or decisions\\n- Calls to tools, APIs, or other agents, including their inputs and results\\n- Model responses, errors, and retries\\n- Timing, token usage, and other metadata\\n\\nTraces help developers understand how an agent reached an outcome, find failures or bottlenecks, and evaluate its behavior. In many systems, a trace is made up of smaller **spans**, each representing one step, such as a model call or tool call.\", \"generation_info\": {\"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ChatGeneration\", \"message\": {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"AIMessage\"], \"kwargs\": {\"content\": \"An **agent trace** is a recorded timeline of what an AI agent did to complete a task. It may include:\\n\\n- The user\u2019s request and the agent\u2019s intermediate reasoning or decisions\\n- Calls to tools, APIs, or other agents, including their inputs and results\\n- Model responses, errors, and retries\\n- Timing, token usage, and other metadata\\n\\nTraces help developers understand how an agent reached an outcome, find failures or bottlenecks, and evaluate its behavior. In many systems, a trace is made up of smaller **spans**, each representing one step, such as a model call or tool call.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 191, \"prompt_tokens\": 12, \"total_tokens\": 203, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 55, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19ewJlEkpZw5ySD3IcmX746sKS3\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"id\": \"lc_run--01a10392-c9c7-7263-a600-2b79f287e58b-0\", \"usage_metadata\": {\"input_tokens\": 12, \"output_tokens\": 191, \"total_tokens\": 203, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 55}}, \"tool_calls\": [], \"invalid_tool_calls\": []}}}]], \"llm_output\": {\"token_usage\": {\"completion_tokens\": 191, \"prompt_tokens\": 12, \"total_tokens\": 203, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 55, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19ewJlEkpZw5ySD3IcmX746sKS3\", \"service_tier\": \"default\"}, \"run\": null, \"type\": \"LLMResult\"}"
                   }
                 },
                 {
@@ -102,13 +102,13 @@
                 {
                   "key": "llm.output_messages.0.message.content",
                   "value": {
-                    "stringValue": "An **agent trace** is a record of an AI agent’s run: what it received, which actions or tools it used, what results came back, and what it ultimately produced.\n\nFor example:\n\n1. User asks for the weather.\n2. Agent calls a weather API.\n3. API returns the forecast.\n4. Agent summarizes it for the user.\n\nTraces help developers debug and evaluate agents. They usually capture observable steps and tool interactions—not necessarily the agent’s private internal reasoning. The exact contents depend on the system."
+                    "stringValue": "An **agent trace** is a recorded timeline of what an AI agent did to complete a task. It may include:\n\n- The user\u2019s request and the agent\u2019s intermediate reasoning or decisions\n- Calls to tools, APIs, or other agents, including their inputs and results\n- Model responses, errors, and retries\n- Timing, token usage, and other metadata\n\nTraces help developers understand how an agent reached an outcome, find failures or bottlenecks, and evaluate its behavior. In many systems, a trace is made up of smaller **spans**, each representing one step, such as a model call or tool call."
                   }
                 },
                 {
                   "key": "llm.invocation_parameters",
                   "value": {
-                    "stringValue": "{\"model\":\"openai/gpt-6-luna\",\"model_name\":\"openai/gpt-6-luna\",\"stream\":false,\"_type\":\"openai-chat\",\"stop\":null}"
+                    "stringValue": "{\"model\": \"openai/gpt-6-luna\", \"model_name\": \"openai/gpt-6-luna\", \"stream\": false, \"_type\": \"openai-chat\", \"stop\": null}"
                   }
                 },
                 {
@@ -138,13 +138,13 @@
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "233"
+                    "intValue": "191"
                   }
                 },
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "245"
+                    "intValue": "203"
                   }
                 },
                 {
@@ -156,7 +156,7 @@
                 {
                   "key": "llm.token_count.completion_details.reasoning",
                   "value": {
-                    "intValue": "118"
+                    "intValue": "55"
                   }
                 },
                 {
@@ -180,7 +180,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langchain_chat_model\",\"lc_agent_name\":\"research_agent\",\"langgraph_step\":1,\"langgraph_node\":\"model\",\"langgraph_triggers\":[\"branch:to:model\"],\"langgraph_path\":[\"__pregel_pull\",\"model\"],\"langgraph_checkpoint_ns\":\"model:5d073867-3e88-7b69-7e22-507bf15134a9\",\"checkpoint_ns\":\"model:5d073867-3e88-7b69-7e22-507bf15134a9\",\"ls_provider\":\"openai\",\"ls_model_name\":\"openai/gpt-6-luna\",\"ls_model_type\":\"chat\",\"ls_temperature\":null,\"lc_versions\":{\"langchain-core\":\"1.6.6\",\"langchain\":\"1.4.3\",\"langchain-openai\":\"1.6.7\"}}"
+                    "stringValue": "{\"ls_integration\": \"langchain_chat_model\", \"lc_agent_name\": \"research_agent\", \"langgraph_step\": 1, \"langgraph_node\": \"model\", \"langgraph_triggers\": [\"branch:to:model\"], \"langgraph_path\": [\"__pregel_pull\", \"model\"], \"langgraph_checkpoint_ns\": \"model:cc9bfebe-a965-ae0a-e6f2-9e3e3c0d6f14\", \"checkpoint_ns\": \"model:cc9bfebe-a965-ae0a-e6f2-9e3e3c0d6f14\", \"ls_provider\": \"openai\", \"ls_model_name\": \"openai/gpt-6-luna\", \"ls_model_type\": \"chat\", \"ls_temperature\": null, \"lc_versions\": {\"langchain-core\": \"1.6.6\", \"langchain\": \"1.4.3\", \"langchain-openai\": \"1.6.7\"}}"
                   }
                 },
                 {
@@ -196,18 +196,18 @@
               "flags": 256
             },
             {
-              "traceId": "fff422e2eaff0db64132f26efe387a6c",
-              "spanId": "462247f1c7f18034",
-              "parentSpanId": "b2609fcd461d1097",
+              "traceId": "d9a080b530fb7f3d642b1608d157ba70",
+              "spanId": "3d757465aa89a16c",
+              "parentSpanId": "2c19b55e7c4b5fd8",
               "name": "model",
               "kind": 1,
-              "startTimeUnixNano": "1791012713671066112",
-              "endTimeUnixNano": "1791012718166048000",
+              "startTimeUnixNano": "1791061313991044096",
+              "endTimeUnixNano": "1791061316665323008",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"83d6b4d7-b3ca-4058-a516-7c884a44ac2e\"}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"52e7344a-4b38-4789-8147-2a277f6c775f\"}}]}"
                   }
                 },
                 {
@@ -219,7 +219,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "[{\"graph\":null,\"update\":{\"messages\":[{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of an AI agent’s run: what it received, which actions or tools it used, what results came back, and what it ultimately produced.\\n\\nFor example:\\n\\n1. User asks for the weather.\\n2. Agent calls a weather API.\\n3. API returns the forecast.\\n4. Agent summarizes it for the user.\\n\\nTraces help developers debug and evaluate agents. They usually capture observable steps and tool interactions—not necessarily the agent’s private internal reasoning. The exact contents depend on the system.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":233,\"prompt_tokens\":12,\"total_tokens\":245,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":118,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoVmBRgBU4JGnF3vUCi895jENkSZ\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":\"research_agent\",\"id\":\"lc_run--01a100ad-34c7-7293-9fc1-65444c4a94ce-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":12,\"output_tokens\":233,\"total_tokens\":245,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":118}}}}]},\"resume\":null,\"goto\":[]}]"
+                    "stringValue": "[{\"graph\": null, \"update\": {\"messages\": [{\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a recorded timeline of what an AI agent did to complete a task. It may include:\\n\\n- The user\u2019s request and the agent\u2019s intermediate reasoning or decisions\\n- Calls to tools, APIs, or other agents, including their inputs and results\\n- Model responses, errors, and retries\\n- Timing, token usage, and other metadata\\n\\nTraces help developers understand how an agent reached an outcome, find failures or bottlenecks, and evaluate its behavior. In many systems, a trace is made up of smaller **spans**, each representing one step, such as a model call or tool call.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 191, \"prompt_tokens\": 12, \"total_tokens\": 203, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 55, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19ewJlEkpZw5ySD3IcmX746sKS3\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": \"research_agent\", \"id\": \"lc_run--01a10392-c9c7-7263-a600-2b79f287e58b-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 12, \"output_tokens\": 191, \"total_tokens\": 203, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 55}}}}]}, \"resume\": null, \"goto\": []}]"
                   }
                 },
                 {
@@ -243,7 +243,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langchain_create_agent\",\"lc_agent_name\":\"research_agent\",\"langgraph_step\":1,\"langgraph_node\":\"model\",\"langgraph_triggers\":[\"branch:to:model\"],\"langgraph_path\":[\"__pregel_pull\",\"model\"],\"langgraph_checkpoint_ns\":\"model:5d073867-3e88-7b69-7e22-507bf15134a9\"}"
+                    "stringValue": "{\"ls_integration\": \"langchain_create_agent\", \"lc_agent_name\": \"research_agent\", \"langgraph_step\": 1, \"langgraph_node\": \"model\", \"langgraph_triggers\": [\"branch:to:model\"], \"langgraph_path\": [\"__pregel_pull\", \"model\"], \"langgraph_checkpoint_ns\": \"model:cc9bfebe-a965-ae0a-e6f2-9e3e3c0d6f14\"}"
                   }
                 },
                 {
@@ -259,17 +259,17 @@
               "flags": 256
             },
             {
-              "traceId": "fff422e2eaff0db64132f26efe387a6c",
-              "spanId": "b2609fcd461d1097",
+              "traceId": "d9a080b530fb7f3d642b1608d157ba70",
+              "spanId": "2c19b55e7c4b5fd8",
               "name": "research_agent",
               "kind": 1,
-              "startTimeUnixNano": "1791012713670352896",
-              "endTimeUnixNano": "1791012718166877952",
+              "startTimeUnixNano": "1791061313989900032",
+              "endTimeUnixNano": "1791061316666109184",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is an agent trace?\"}]}"
+                    "stringValue": "{\"messages\": [{\"role\": \"user\", \"content\": \"What is an agent trace?\"}]}"
                   }
                 },
                 {
@@ -281,7 +281,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"83d6b4d7-b3ca-4058-a516-7c884a44ac2e\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of an AI agent’s run: what it received, which actions or tools it used, what results came back, and what it ultimately produced.\\n\\nFor example:\\n\\n1. User asks for the weather.\\n2. Agent calls a weather API.\\n3. API returns the forecast.\\n4. Agent summarizes it for the user.\\n\\nTraces help developers debug and evaluate agents. They usually capture observable steps and tool interactions—not necessarily the agent’s private internal reasoning. The exact contents depend on the system.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":233,\"prompt_tokens\":12,\"total_tokens\":245,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":118,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoVmBRgBU4JGnF3vUCi895jENkSZ\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":\"research_agent\",\"id\":\"lc_run--01a100ad-34c7-7293-9fc1-65444c4a94ce-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":12,\"output_tokens\":233,\"total_tokens\":245,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":118}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"52e7344a-4b38-4789-8147-2a277f6c775f\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a recorded timeline of what an AI agent did to complete a task. It may include:\\n\\n- The user\u2019s request and the agent\u2019s intermediate reasoning or decisions\\n- Calls to tools, APIs, or other agents, including their inputs and results\\n- Model responses, errors, and retries\\n- Timing, token usage, and other metadata\\n\\nTraces help developers understand how an agent reached an outcome, find failures or bottlenecks, and evaluate its behavior. In many systems, a trace is made up of smaller **spans**, each representing one step, such as a model call or tool call.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 191, \"prompt_tokens\": 12, \"total_tokens\": 203, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 55, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19ewJlEkpZw5ySD3IcmX746sKS3\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": \"research_agent\", \"id\": \"lc_run--01a10392-c9c7-7263-a600-2b79f287e58b-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 12, \"output_tokens\": 191, \"total_tokens\": 203, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 55}}}}]}"
                   }
                 },
                 {
@@ -299,7 +299,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langchain_create_agent\",\"lc_agent_name\":\"research_agent\"}"
+                    "stringValue": "{\"ls_integration\": \"langchain_create_agent\", \"lc_agent_name\": \"research_agent\"}"
                   }
                 },
                 {

litellm-rust/crates/traces/tests/fixtures/langgraph_simple.json

@@ -24,13 +24,7 @@
           {
             "key": "service.instance.id",
             "value": {
-              "stringValue": "ea95702f-4404-4f09-b8b6-19515d10fb73"
-            }
-          },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "langgraph-simple"
+              "stringValue": "5ae85caa-bb64-4218-94f6-0066f5d01eba"
             }
           },
           {
@@ -38,6 +32,12 @@
             "value": {
               "stringValue": "0.66b0"
             }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service:python3"
+            }
           }
         ]
       },
@@ -49,18 +49,18 @@
           },
           "spans": [
             {
-              "traceId": "af9e61052268f1da3133f29cace994e7",
-              "spanId": "258df8df18b0c4d0",
-              "parentSpanId": "6e090feb0298b338",
+              "traceId": "a2b7bdfc7ce2168e05f7298ee11ca7d3",
+              "spanId": "be5e5e817f6a71a5",
+              "parentSpanId": "91fbc8f8c9a10da7",
               "name": "ChatOpenAI",
               "kind": 1,
-              "startTimeUnixNano": "1791012813994045952",
-              "endTimeUnixNano": "1791012817659385088",
+              "startTimeUnixNano": "1791061316831124992",
+              "endTimeUnixNano": "1791061320049913856",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[[{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"HumanMessage\"],\"kwargs\":{\"content\":\"What is an agent trace?\",\"type\":\"human\",\"id\":\"c2ac14fc-4fb2-4fc9-9814-729cacc38b74\"}}]]}"
+                    "stringValue": "{\"messages\": [[{\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"HumanMessage\"], \"kwargs\": {\"content\": \"What is an agent trace?\", \"type\": \"human\", \"id\": \"2a09435d-ae05-4259-9d4a-5a788d07464d\"}}]]}"
                   }
                 },
                 {
@@ -72,7 +72,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"generations\":[[{\"text\":\"An **agent trace** is a record of an AI agent’s run: what it received, what steps it took, which tools or services it called, what results came back, and how it produced its final response.\\n\\nA trace might include:\\n\\n- The user’s request and relevant inputs\\n- The agent’s steps or decisions\\n- Tool calls and their results\\n- Errors, retries, and timing\\n- The final output\\n\\nTraces help developers understand, debug, and evaluate an agent’s behavior. They don’t necessarily contain the model’s private internal reasoning; often they show only observable steps, such as tool calls and outputs.\",\"generation_info\":{\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ChatGeneration\",\"message\":{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"AIMessage\"],\"kwargs\":{\"content\":\"An **agent trace** is a record of an AI agent’s run: what it received, what steps it took, which tools or services it called, what results came back, and how it produced its final response.\\n\\nA trace might include:\\n\\n- The user’s request and relevant inputs\\n- The agent’s steps or decisions\\n- Tool calls and their results\\n- Errors, retries, and timing\\n- The final output\\n\\nTraces help developers understand, debug, and evaluate an agent’s behavior. They don’t necessarily contain the model’s private internal reasoning; often they show only observable steps, such as tool calls and outputs.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":195,\"prompt_tokens\":12,\"total_tokens\":207,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":59,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXOfkaG93LZigoStAzPDsRzU4wZ\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"id\":\"lc_run--01a100ae-bcaa-7321-ae7a-432658009c65-0\",\"usage_metadata\":{\"input_tokens\":12,\"output_tokens\":195,\"total_tokens\":207,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":59}},\"tool_calls\":[],\"invalid_tool_calls\":[]}}}]],\"llm_output\":{\"token_usage\":{\"completion_tokens\":195,\"prompt_tokens\":12,\"total_tokens\":207,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":59,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXOfkaG93LZigoStAzPDsRzU4wZ\",\"service_tier\":\"default\"},\"run\":null,\"type\":\"LLMResult\"}"
+                    "stringValue": "{\"generations\": [[{\"text\": \"An **agent trace** is a record of what an AI agent did during a task, step by step. It commonly includes the agent\u2019s inputs and outputs, tool calls and their results, and timing or errors.\\n\\nFor example, a trace might show: *user asks a question \u2192 agent searches the web \u2192 receives results \u2192 summarizes them.*\\n\\nTraces help people debug, evaluate, and audit an agent. They usually capture observable actions\u2014not necessarily the model\u2019s private internal reasoning.\", \"generation_info\": {\"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ChatGeneration\", \"message\": {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"AIMessage\"], \"kwargs\": {\"content\": \"An **agent trace** is a record of what an AI agent did during a task, step by step. It commonly includes the agent\u2019s inputs and outputs, tool calls and their results, and timing or errors.\\n\\nFor example, a trace might show: *user asks a question \u2192 agent searches the web \u2192 receives results \u2192 summarizes them.*\\n\\nTraces help people debug, evaluate, and audit an agent. They usually capture observable actions\u2014not necessarily the model\u2019s private internal reasoning.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 187, \"prompt_tokens\": 12, \"total_tokens\": 199, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 83, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19hpMRm3ctM96rCV2Noc7Dp5Wcm\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"id\": \"lc_run--01a10392-d4df-72b2-aa09-53e30f1c2e3c-0\", \"usage_metadata\": {\"input_tokens\": 12, \"output_tokens\": 187, \"total_tokens\": 199, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 83}}, \"tool_calls\": [], \"invalid_tool_calls\": []}}}]], \"llm_output\": {\"token_usage\": {\"completion_tokens\": 187, \"prompt_tokens\": 12, \"total_tokens\": 199, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 83, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19hpMRm3ctM96rCV2Noc7Dp5Wcm\", \"service_tier\": \"default\"}, \"run\": null, \"type\": \"LLMResult\"}"
                   }
                 },
                 {
@@ -102,13 +102,13 @@
                 {
                   "key": "llm.output_messages.0.message.content",
                   "value": {
-                    "stringValue": "An **agent trace** is a record of an AI agent’s run: what it received, what steps it took, which tools or services it called, what results came back, and how it produced its final response.\n\nA trace might include:\n\n- The user’s request and relevant inputs\n- The agent’s steps or decisions\n- Tool calls and their results\n- Errors, retries, and timing\n- The final output\n\nTraces help developers understand, debug, and evaluate an agent’s behavior. They don’t necessarily contain the model’s private internal reasoning; often they show only observable steps, such as tool calls and outputs."
+                    "stringValue": "An **agent trace** is a record of what an AI agent did during a task, step by step. It commonly includes the agent\u2019s inputs and outputs, tool calls and their results, and timing or errors.\n\nFor example, a trace might show: *user asks a question \u2192 agent searches the web \u2192 receives results \u2192 summarizes them.*\n\nTraces help people debug, evaluate, and audit an agent. They usually capture observable actions\u2014not necessarily the model\u2019s private internal reasoning."
                   }
                 },
                 {
                   "key": "llm.invocation_parameters",
                   "value": {
-                    "stringValue": "{\"model\":\"openai/gpt-6-luna\",\"model_name\":\"openai/gpt-6-luna\",\"stream\":false,\"_type\":\"openai-chat\",\"stop\":null}"
+                    "stringValue": "{\"model\": \"openai/gpt-6-luna\", \"model_name\": \"openai/gpt-6-luna\", \"stream\": false, \"_type\": \"openai-chat\", \"stop\": null}"
                   }
                 },
                 {
@@ -138,13 +138,13 @@
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "195"
+                    "intValue": "187"
                   }
                 },
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "207"
+                    "intValue": "199"
                   }
                 },
                 {
@@ -156,7 +156,7 @@
                 {
                   "key": "llm.token_count.completion_details.reasoning",
                   "value": {
-                    "intValue": "59"
+                    "intValue": "83"
                   }
                 },
                 {
@@ -180,7 +180,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langchain_chat_model\",\"langgraph_step\":1,\"langgraph_node\":\"call_model\",\"langgraph_triggers\":[\"branch:to:call_model\"],\"langgraph_path\":[\"__pregel_pull\",\"call_model\"],\"langgraph_checkpoint_ns\":\"call_model:9cdca8af-0105-48c1-a90c-e733d0eda7d0\",\"checkpoint_ns\":\"call_model:9cdca8af-0105-48c1-a90c-e733d0eda7d0\",\"ls_provider\":\"openai\",\"ls_model_name\":\"openai/gpt-6-luna\",\"ls_model_type\":\"chat\",\"ls_temperature\":null,\"lc_versions\":{\"langchain-core\":\"1.6.6\",\"langchain-openai\":\"1.6.7\"}}"
+                    "stringValue": "{\"ls_integration\": \"langchain_chat_model\", \"langgraph_step\": 1, \"langgraph_node\": \"call_model\", \"langgraph_triggers\": [\"branch:to:call_model\"], \"langgraph_path\": [\"__pregel_pull\", \"call_model\"], \"langgraph_checkpoint_ns\": \"call_model:4756011d-1080-f746-eefc-23cf69bce566\", \"checkpoint_ns\": \"call_model:4756011d-1080-f746-eefc-23cf69bce566\", \"ls_provider\": \"openai\", \"ls_model_name\": \"openai/gpt-6-luna\", \"ls_model_type\": \"chat\", \"ls_temperature\": null, \"lc_versions\": {\"langchain-core\": \"1.6.6\", \"langchain-openai\": \"1.6.7\"}}"
                   }
                 },
                 {
@@ -196,18 +196,18 @@
               "flags": 256
             },
             {
-              "traceId": "af9e61052268f1da3133f29cace994e7",
-              "spanId": "6e090feb0298b338",
-              "parentSpanId": "a5857e5f6e1fee75",
+              "traceId": "a2b7bdfc7ce2168e05f7298ee11ca7d3",
+              "spanId": "91fbc8f8c9a10da7",
+              "parentSpanId": "4ed2d903d4625b4a",
               "name": "call_model",
               "kind": 1,
-              "startTimeUnixNano": "1791012813993732096",
-              "endTimeUnixNano": "1791012817660307968",
+              "startTimeUnixNano": "1791061316830797056",
+              "endTimeUnixNano": "1791061320050427904",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"c2ac14fc-4fb2-4fc9-9814-729cacc38b74\"}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"2a09435d-ae05-4259-9d4a-5a788d07464d\"}}]}"
                   }
                 },
                 {
@@ -219,7 +219,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of an AI agent’s run: what it received, what steps it took, which tools or services it called, what results came back, and how it produced its final response.\\n\\nA trace might include:\\n\\n- The user’s request and relevant inputs\\n- The agent’s steps or decisions\\n- Tool calls and their results\\n- Errors, retries, and timing\\n- The final output\\n\\nTraces help developers understand, debug, and evaluate an agent’s behavior. They don’t necessarily contain the model’s private internal reasoning; often they show only observable steps, such as tool calls and outputs.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":195,\"prompt_tokens\":12,\"total_tokens\":207,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":59,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXOfkaG93LZigoStAzPDsRzU4wZ\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-bcaa-7321-ae7a-432658009c65-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":12,\"output_tokens\":195,\"total_tokens\":207,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":59}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a record of what an AI agent did during a task, step by step. It commonly includes the agent\u2019s inputs and outputs, tool calls and their results, and timing or errors.\\n\\nFor example, a trace might show: *user asks a question \u2192 agent searches the web \u2192 receives results \u2192 summarizes them.*\\n\\nTraces help people debug, evaluate, and audit an agent. They usually capture observable actions\u2014not necessarily the model\u2019s private internal reasoning.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 187, \"prompt_tokens\": 12, \"total_tokens\": 199, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 83, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19hpMRm3ctM96rCV2Noc7Dp5Wcm\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10392-d4df-72b2-aa09-53e30f1c2e3c-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 12, \"output_tokens\": 187, \"total_tokens\": 199, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 83}}}}]}"
                   }
                 },
                 {
@@ -243,7 +243,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":1,\"langgraph_node\":\"call_model\",\"langgraph_triggers\":[\"branch:to:call_model\"],\"langgraph_path\":[\"__pregel_pull\",\"call_model\"],\"langgraph_checkpoint_ns\":\"call_model:9cdca8af-0105-48c1-a90c-e733d0eda7d0\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 1, \"langgraph_node\": \"call_model\", \"langgraph_triggers\": [\"branch:to:call_model\"], \"langgraph_path\": [\"__pregel_pull\", \"call_model\"], \"langgraph_checkpoint_ns\": \"call_model:4756011d-1080-f746-eefc-23cf69bce566\"}"
                   }
                 },
                 {
@@ -259,17 +259,17 @@
               "flags": 256
             },
             {
-              "traceId": "af9e61052268f1da3133f29cace994e7",
-              "spanId": "a5857e5f6e1fee75",
+              "traceId": "a2b7bdfc7ce2168e05f7298ee11ca7d3",
+              "spanId": "4ed2d903d4625b4a",
               "name": "research_agent",
               "kind": 1,
-              "startTimeUnixNano": "1791012813992832000",
-              "endTimeUnixNano": "1791012817661214976",
+              "startTimeUnixNano": "1791061316829106944",
+              "endTimeUnixNano": "1791061320050825984",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is an agent trace?\"}]}"
+                    "stringValue": "{\"messages\": [{\"role\": \"user\", \"content\": \"What is an agent trace?\"}]}"
                   }
                 },
                 {
@@ -281,7 +281,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"c2ac14fc-4fb2-4fc9-9814-729cacc38b74\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of an AI agent’s run: what it received, what steps it took, which tools or services it called, what results came back, and how it produced its final response.\\n\\nA trace might include:\\n\\n- The user’s request and relevant inputs\\n- The agent’s steps or decisions\\n- Tool calls and their results\\n- Errors, retries, and timing\\n- The final output\\n\\nTraces help developers understand, debug, and evaluate an agent’s behavior. They don’t necessarily contain the model’s private internal reasoning; often they show only observable steps, such as tool calls and outputs.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":195,\"prompt_tokens\":12,\"total_tokens\":207,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":59,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXOfkaG93LZigoStAzPDsRzU4wZ\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-bcaa-7321-ae7a-432658009c65-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":12,\"output_tokens\":195,\"total_tokens\":207,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":59}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"2a09435d-ae05-4259-9d4a-5a788d07464d\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a record of what an AI agent did during a task, step by step. It commonly includes the agent\u2019s inputs and outputs, tool calls and their results, and timing or errors.\\n\\nFor example, a trace might show: *user asks a question \u2192 agent searches the web \u2192 receives results \u2192 summarizes them.*\\n\\nTraces help people debug, evaluate, and audit an agent. They usually capture observable actions\u2014not necessarily the model\u2019s private internal reasoning.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 187, \"prompt_tokens\": 12, \"total_tokens\": 199, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 83, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV19hpMRm3ctM96rCV2Noc7Dp5Wcm\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10392-d4df-72b2-aa09-53e30f1c2e3c-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 12, \"output_tokens\": 187, \"total_tokens\": 199, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 83}}}}]}"
                   }
                 },
                 {
@@ -299,7 +299,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\"}"
                   }
                 },
                 {

litellm-rust/crates/traces/tests/fixtures/langgraph_swarm.json

@@ -24,13 +24,7 @@
           {
             "key": "service.instance.id",
             "value": {
-              "stringValue": "b392f5f6-8bde-4100-8615-85406c69532f"
-            }
-          },
-          {
-            "key": "service.name",
-            "value": {
-              "stringValue": "langgraph-swarm"
+              "stringValue": "7008af59-8740-44f0-afdc-ff9d33f91deb"
             }
           },
           {
@@ -38,6 +32,12 @@
             "value": {
               "stringValue": "0.66b0"
             }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service:python3"
+            }
           }
         ]
       },
@@ -49,18 +49,18 @@
           },
           "spans": [
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "fe69d78d633b09ba",
-              "parentSpanId": "f1d7b2e38e4a297a",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "08253eda5ea23268",
+              "parentSpanId": "0c8d577859bdf794",
               "name": "ChatOpenAI",
               "kind": 1,
-              "startTimeUnixNano": "1791012830407246080",
-              "endTimeUnixNano": "1791012832857249024",
+              "startTimeUnixNano": "1791061349479399936",
+              "endTimeUnixNano": "1791061352921875968",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[[{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"SystemMessage\"],\"kwargs\":{\"content\":\"Gather the key facts about the user's question.\",\"type\":\"system\"}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"HumanMessage\"],\"kwargs\":{\"content\":\"What is an agent trace?\",\"type\":\"human\",\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}}]]}"
+                    "stringValue": "{\"messages\": [[{\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"SystemMessage\"], \"kwargs\": {\"content\": \"Gather the key facts about the user's question.\", \"type\": \"system\"}}, {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"HumanMessage\"], \"kwargs\": {\"content\": \"What is an agent trace?\", \"type\": \"human\", \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}]]}"
                   }
                 },
                 {
@@ -72,7 +72,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"generations\":[[{\"text\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"generation_info\":{\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ChatGeneration\",\"message\":{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"AIMessage\"],\"kwargs\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}},\"tool_calls\":[],\"invalid_tool_calls\":[]}}}]],\"llm_output\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\"},\"run\":null,\"type\":\"LLMResult\"}"
+                    "stringValue": "{\"generations\": [[{\"text\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"generation_info\": {\"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ChatGeneration\", \"message\": {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"AIMessage\"], \"kwargs\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}, \"tool_calls\": [], \"invalid_tool_calls\": []}}}]], \"llm_output\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\"}, \"run\": null, \"type\": \"LLMResult\"}"
                   }
                 },
                 {
@@ -114,13 +114,13 @@
                 {
                   "key": "llm.output_messages.0.message.content",
                   "value": {
-                    "stringValue": "An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\n\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\n\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning."
+                    "stringValue": "An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\n\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\n\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system."
                   }
                 },
                 {
                   "key": "llm.invocation_parameters",
                   "value": {
-                    "stringValue": "{\"model\":\"openai/gpt-6-luna\",\"model_name\":\"openai/gpt-6-luna\",\"stream\":false,\"_type\":\"openai-chat\",\"stop\":null}"
+                    "stringValue": "{\"model\": \"openai/gpt-6-luna\", \"model_name\": \"openai/gpt-6-luna\", \"stream\": false, \"_type\": \"openai-chat\", \"stop\": null}"
                   }
                 },
                 {
@@ -150,13 +150,13 @@
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "191"
+                    "intValue": "223"
                   }
                 },
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "216"
+                    "intValue": "248"
                   }
                 },
                 {
@@ -168,7 +168,7 @@
                 {
                   "key": "llm.token_count.completion_details.reasoning",
                   "value": {
-                    "intValue": "88"
+                    "intValue": "96"
                   }
                 },
                 {
@@ -192,7 +192,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langchain_chat_model\",\"langgraph_step\":1,\"langgraph_node\":\"call_model\",\"langgraph_triggers\":[\"branch:to:call_model\"],\"langgraph_path\":[\"__pregel_pull\",\"call_model\"],\"langgraph_checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f|call_model:c7cf85d6-2beb-b779-6379-65712fdb9128\",\"checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f\",\"ls_provider\":\"openai\",\"ls_model_name\":\"openai/gpt-6-luna\",\"ls_model_type\":\"chat\",\"ls_temperature\":null,\"lc_versions\":{\"langchain-core\":\"1.6.6\",\"langchain-openai\":\"1.6.7\"}}"
+                    "stringValue": "{\"ls_integration\": \"langchain_chat_model\", \"langgraph_step\": 1, \"langgraph_node\": \"call_model\", \"langgraph_triggers\": [\"branch:to:call_model\"], \"langgraph_path\": [\"__pregel_pull\", \"call_model\"], \"langgraph_checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545|call_model:8efbb99d-9a7f-0fed-6a01-200ce39b60d2\", \"checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545\", \"ls_provider\": \"openai\", \"ls_model_name\": \"openai/gpt-6-luna\", \"ls_model_type\": \"chat\", \"ls_temperature\": null, \"lc_versions\": {\"langchain-core\": \"1.6.6\", \"langchain-openai\": \"1.6.7\"}}"
                   }
                 },
                 {
@@ -208,18 +208,18 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "f1d7b2e38e4a297a",
-              "parentSpanId": "ace3bc964d644662",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "0c8d577859bdf794",
+              "parentSpanId": "310d05438d077e5d",
               "name": "call_model",
               "kind": 1,
-              "startTimeUnixNano": "1791012830406982912",
-              "endTimeUnixNano": "1791012832857625856",
+              "startTimeUnixNano": "1791061349479172096",
+              "endTimeUnixNano": "1791061352922414080",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}]}"
                   }
                 },
                 {
@@ -231,7 +231,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}]}"
                   }
                 },
                 {
@@ -255,7 +255,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":1,\"langgraph_node\":\"call_model\",\"langgraph_triggers\":[\"branch:to:call_model\"],\"langgraph_path\":[\"__pregel_pull\",\"call_model\"],\"langgraph_checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f|call_model:c7cf85d6-2beb-b779-6379-65712fdb9128\",\"checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 1, \"langgraph_node\": \"call_model\", \"langgraph_triggers\": [\"branch:to:call_model\"], \"langgraph_path\": [\"__pregel_pull\", \"call_model\"], \"langgraph_checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545|call_model:8efbb99d-9a7f-0fed-6a01-200ce39b60d2\", \"checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545\"}"
                   }
                 },
                 {
@@ -271,18 +271,18 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "ace3bc964d644662",
-              "parentSpanId": "6976ae7fb7fc8e90",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "310d05438d077e5d",
+              "parentSpanId": "ecb303041155e83e",
               "name": "search_agent",
               "kind": 1,
-              "startTimeUnixNano": "1791012830406665984",
-              "endTimeUnixNano": "1791012832857971968",
+              "startTimeUnixNano": "1791061349478894080",
+              "endTimeUnixNano": "1791061352922853120",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}]}"
                   }
                 },
                 {
@@ -294,7 +294,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}]}"
                   }
                 },
                 {
@@ -318,7 +318,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":1,\"langgraph_node\":\"search\",\"langgraph_triggers\":[\"branch:to:search\"],\"langgraph_path\":[\"__pregel_pull\",\"search\"],\"langgraph_checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f\",\"checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 1, \"langgraph_node\": \"search\", \"langgraph_triggers\": [\"branch:to:search\"], \"langgraph_path\": [\"__pregel_pull\", \"search\"], \"langgraph_checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545\", \"checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545\"}"
                   }
                 },
                 {
@@ -334,18 +334,18 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "6976ae7fb7fc8e90",
-              "parentSpanId": "5926c6bcedd87dc2",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "ecb303041155e83e",
+              "parentSpanId": "87feccfee88f28d6",
               "name": "search",
               "kind": 1,
-              "startTimeUnixNano": "1791012830406498048",
-              "endTimeUnixNano": "1791012832858153216",
+              "startTimeUnixNano": "1791061349478753024",
+              "endTimeUnixNano": "1791061352922967808",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}]}"
                   }
                 },
                 {
@@ -357,7 +357,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}]}"
                   }
                 },
                 {
@@ -381,7 +381,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":1,\"langgraph_node\":\"search\",\"langgraph_triggers\":[\"branch:to:search\"],\"langgraph_path\":[\"__pregel_pull\",\"search\"],\"langgraph_checkpoint_ns\":\"search:c41a211e-f71e-79ef-4d79-d39203e3129f\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 1, \"langgraph_node\": \"search\", \"langgraph_triggers\": [\"branch:to:search\"], \"langgraph_path\": [\"__pregel_pull\", \"search\"], \"langgraph_checkpoint_ns\": \"search:fe96c9fa-704e-2e93-8e3d-a91748c5e545\"}"
                   }
                 },
                 {
@@ -395,20 +395,72 @@
                 "code": 1
               },
               "flags": 256
-            },
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "resource": {
+        "attributes": [
+          {
+            "key": "telemetry.sdk.language",
+            "value": {
+              "stringValue": "python"
+            }
+          },
+          {
+            "key": "telemetry.sdk.name",
+            "value": {
+              "stringValue": "opentelemetry"
+            }
+          },
+          {
+            "key": "telemetry.sdk.version",
+            "value": {
+              "stringValue": "1.45.0"
+            }
+          },
+          {
+            "key": "service.instance.id",
+            "value": {
+              "stringValue": "7008af59-8740-44f0-afdc-ff9d33f91deb"
+            }
+          },
+          {
+            "key": "telemetry.auto.version",
+            "value": {
+              "stringValue": "0.66b0"
+            }
+          },
+          {
+            "key": "service.name",
+            "value": {
+              "stringValue": "unknown_service:python3"
+            }
+          }
+        ]
+      },
+      "scopeSpans": [
+        {
+          "scope": {
+            "name": "openinference.instrumentation.langchain",
+            "version": "0.1.78"
+          },
+          "spans": [
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "835d6ca1079a36b7",
-              "parentSpanId": "f06939809a5142de",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "b2650845221be5b5",
+              "parentSpanId": "01a05fd6b3434433",
               "name": "ChatOpenAI",
               "kind": 1,
-              "startTimeUnixNano": "1791012832859083008",
-              "endTimeUnixNano": "1791012835106898176",
+              "startTimeUnixNano": "1791061352924045056",
+              "endTimeUnixNano": "1791061354661287168",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[[{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"SystemMessage\"],\"kwargs\":{\"content\":\"Write a concise answer from the facts above.\",\"type\":\"system\"}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"HumanMessage\"],\"kwargs\":{\"content\":\"What is an agent trace?\",\"type\":\"human\",\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"AIMessage\"],\"kwargs\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}},\"tool_calls\":[],\"invalid_tool_calls\":[]}}]]}"
+                    "stringValue": "{\"messages\": [[{\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"SystemMessage\"], \"kwargs\": {\"content\": \"Write a concise answer from the facts above.\", \"type\": \"system\"}}, {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"HumanMessage\"], \"kwargs\": {\"content\": \"What is an agent trace?\", \"type\": \"human\", \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"AIMessage\"], \"kwargs\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}, \"tool_calls\": [], \"invalid_tool_calls\": []}}]]}"
                   }
                 },
                 {
@@ -420,7 +472,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"generations\":[[{\"text\":\"An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning.\",\"generation_info\":{\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ChatGeneration\",\"message\":{\"lc\":1,\"type\":\"constructor\",\"id\":[\"langchain\",\"schema\",\"messages\",\"AIMessage\"],\"kwargs\":{\"content\":\"An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":124,\"prompt_tokens\":125,\"total_tokens\":249,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":62,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXh6pPDCqqY08c8to8pkmEfoP6m\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"id\":\"lc_run--01a100af-065b-7190-833c-c11b443bf075-0\",\"usage_metadata\":{\"input_tokens\":125,\"output_tokens\":124,\"total_tokens\":249,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":62}},\"tool_calls\":[],\"invalid_tool_calls\":[]}}}]],\"llm_output\":{\"token_usage\":{\"completion_tokens\":124,\"prompt_tokens\":125,\"total_tokens\":249,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":62,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXh6pPDCqqY08c8to8pkmEfoP6m\",\"service_tier\":\"default\"},\"run\":null,\"type\":\"LLMResult\"}"
+                    "stringValue": "{\"generations\": [[{\"text\": \"An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents.\", \"generation_info\": {\"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ChatGeneration\", \"message\": {\"lc\": 1, \"type\": \"constructor\", \"id\": [\"langchain\", \"schema\", \"messages\", \"AIMessage\"], \"kwargs\": {\"content\": \"An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 96, \"prompt_tokens\": 149, \"total_tokens\": 245, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 46, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1AHQQGzXMvkUbS86kQTl6mTAMf6\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"id\": \"lc_run--01a10393-61dc-79b2-a2fb-94ed44373aaf-0\", \"usage_metadata\": {\"input_tokens\": 149, \"output_tokens\": 96, \"total_tokens\": 245, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 46}}, \"tool_calls\": [], \"invalid_tool_calls\": []}}}]], \"llm_output\": {\"token_usage\": {\"completion_tokens\": 96, \"prompt_tokens\": 149, \"total_tokens\": 245, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 46, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1AHQQGzXMvkUbS86kQTl6mTAMf6\", \"service_tier\": \"default\"}, \"run\": null, \"type\": \"LLMResult\"}"
                   }
                 },
                 {
@@ -462,7 +514,7 @@
                 {
                   "key": "llm.input_messages.2.message.content",
                   "value": {
-                    "stringValue": "An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\n\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\n\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning."
+                    "stringValue": "An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\n\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\n\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system."
                   }
                 },
                 {
@@ -474,13 +526,13 @@
                 {
                   "key": "llm.output_messages.0.message.content",
                   "value": {
-                    "stringValue": "An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning."
+                    "stringValue": "An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents."
                   }
                 },
                 {
                   "key": "llm.invocation_parameters",
                   "value": {
-                    "stringValue": "{\"model\":\"openai/gpt-6-luna\",\"model_name\":\"openai/gpt-6-luna\",\"stream\":false,\"_type\":\"openai-chat\",\"stop\":null}"
+                    "stringValue": "{\"model\": \"openai/gpt-6-luna\", \"model_name\": \"openai/gpt-6-luna\", \"stream\": false, \"_type\": \"openai-chat\", \"stop\": null}"
                   }
                 },
                 {
@@ -504,19 +556,19 @@
                 {
                   "key": "llm.token_count.prompt",
                   "value": {
-                    "intValue": "125"
+                    "intValue": "149"
                   }
                 },
                 {
                   "key": "llm.token_count.completion",
                   "value": {
-                    "intValue": "124"
+                    "intValue": "96"
                   }
                 },
                 {
                   "key": "llm.token_count.total",
                   "value": {
-                    "intValue": "249"
+                    "intValue": "245"
                   }
                 },
                 {
@@ -528,7 +580,7 @@
                 {
                   "key": "llm.token_count.completion_details.reasoning",
                   "value": {
-                    "intValue": "62"
+                    "intValue": "46"
                   }
                 },
                 {
@@ -552,7 +604,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langchain_chat_model\",\"langgraph_step\":1,\"langgraph_node\":\"call_model\",\"langgraph_triggers\":[\"branch:to:call_model\"],\"langgraph_path\":[\"__pregel_pull\",\"call_model\"],\"langgraph_checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727|call_model:442a608e-daef-0597-707c-ce4c36787c47\",\"checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727\",\"ls_provider\":\"openai\",\"ls_model_name\":\"openai/gpt-6-luna\",\"ls_model_type\":\"chat\",\"ls_temperature\":null,\"lc_versions\":{\"langchain-core\":\"1.6.6\",\"langchain-openai\":\"1.6.7\"}}"
+                    "stringValue": "{\"ls_integration\": \"langchain_chat_model\", \"langgraph_step\": 1, \"langgraph_node\": \"call_model\", \"langgraph_triggers\": [\"branch:to:call_model\"], \"langgraph_path\": [\"__pregel_pull\", \"call_model\"], \"langgraph_checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f|call_model:eb6b2c4b-85cd-9fc6-69fd-ee7c39fb138b\", \"checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f\", \"ls_provider\": \"openai\", \"ls_model_name\": \"openai/gpt-6-luna\", \"ls_model_type\": \"chat\", \"ls_temperature\": null, \"lc_versions\": {\"langchain-core\": \"1.6.6\", \"langchain-openai\": \"1.6.7\"}}"
                   }
                 },
                 {
@@ -568,18 +620,18 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "f06939809a5142de",
-              "parentSpanId": "bd184870f763c317",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "01a05fd6b3434433",
+              "parentSpanId": "cc1c518bb4a917b2",
               "name": "call_model",
               "kind": 1,
-              "startTimeUnixNano": "1791012832858917888",
-              "endTimeUnixNano": "1791012835107453184",
+              "startTimeUnixNano": "1791061352923860992",
+              "endTimeUnixNano": "1791061354661565952",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}]}"
                   }
                 },
                 {
@@ -591,7 +643,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":124,\"prompt_tokens\":125,\"total_tokens\":249,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":62,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXh6pPDCqqY08c8to8pkmEfoP6m\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100af-065b-7190-833c-c11b443bf075-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":125,\"output_tokens\":124,\"total_tokens\":249,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":62}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 96, \"prompt_tokens\": 149, \"total_tokens\": 245, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 46, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1AHQQGzXMvkUbS86kQTl6mTAMf6\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-61dc-79b2-a2fb-94ed44373aaf-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 149, \"output_tokens\": 96, \"total_tokens\": 245, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 46}}}}]}"
                   }
                 },
                 {
@@ -615,7 +667,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":1,\"langgraph_node\":\"call_model\",\"langgraph_triggers\":[\"branch:to:call_model\"],\"langgraph_path\":[\"__pregel_pull\",\"call_model\"],\"langgraph_checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727|call_model:442a608e-daef-0597-707c-ce4c36787c47\",\"checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 1, \"langgraph_node\": \"call_model\", \"langgraph_triggers\": [\"branch:to:call_model\"], \"langgraph_path\": [\"__pregel_pull\", \"call_model\"], \"langgraph_checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f|call_model:eb6b2c4b-85cd-9fc6-69fd-ee7c39fb138b\", \"checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f\"}"
                   }
                 },
                 {
@@ -631,18 +683,18 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "bd184870f763c317",
-              "parentSpanId": "223d6d53ba0d9ccf",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "cc1c518bb4a917b2",
+              "parentSpanId": "1d5fb238ad612007",
               "name": "writer_agent",
               "kind": 1,
-              "startTimeUnixNano": "1791012832858571008",
-              "endTimeUnixNano": "1791012835107939072",
+              "startTimeUnixNano": "1791061352923495168",
+              "endTimeUnixNano": "1791061354661936896",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}]}"
                   }
                 },
                 {
@@ -654,7 +706,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":124,\"prompt_tokens\":125,\"total_tokens\":249,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":62,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXh6pPDCqqY08c8to8pkmEfoP6m\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100af-065b-7190-833c-c11b443bf075-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":125,\"output_tokens\":124,\"total_tokens\":249,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":62}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 96, \"prompt_tokens\": 149, \"total_tokens\": 245, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 46, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1AHQQGzXMvkUbS86kQTl6mTAMf6\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-61dc-79b2-a2fb-94ed44373aaf-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 149, \"output_tokens\": 96, \"total_tokens\": 245, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 46}}}}]}"
                   }
                 },
                 {
@@ -678,7 +730,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":2,\"langgraph_node\":\"write\",\"langgraph_triggers\":[\"branch:to:write\"],\"langgraph_path\":[\"__pregel_pull\",\"write\"],\"langgraph_checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727\",\"checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 2, \"langgraph_node\": \"write\", \"langgraph_triggers\": [\"branch:to:write\"], \"langgraph_path\": [\"__pregel_pull\", \"write\"], \"langgraph_checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f\", \"checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f\"}"
                   }
                 },
                 {
@@ -694,18 +746,18 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "223d6d53ba0d9ccf",
-              "parentSpanId": "5926c6bcedd87dc2",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "1d5fb238ad612007",
+              "parentSpanId": "87feccfee88f28d6",
               "name": "write",
               "kind": 1,
-              "startTimeUnixNano": "1791012832858390016",
-              "endTimeUnixNano": "1791012835108278016",
+              "startTimeUnixNano": "1791061352923243008",
+              "endTimeUnixNano": "1791061354662168064",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}]}"
                   }
                 },
                 {
@@ -717,7 +769,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":124,\"prompt_tokens\":125,\"total_tokens\":249,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":62,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXh6pPDCqqY08c8to8pkmEfoP6m\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100af-065b-7190-833c-c11b443bf075-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":125,\"output_tokens\":124,\"total_tokens\":249,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":62}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 96, \"prompt_tokens\": 149, \"total_tokens\": 245, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 46, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1AHQQGzXMvkUbS86kQTl6mTAMf6\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-61dc-79b2-a2fb-94ed44373aaf-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 149, \"output_tokens\": 96, \"total_tokens\": 245, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 46}}}}]}"
                   }
                 },
                 {
@@ -741,7 +793,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\",\"langgraph_step\":2,\"langgraph_node\":\"write\",\"langgraph_triggers\":[\"branch:to:write\"],\"langgraph_path\":[\"__pregel_pull\",\"write\"],\"langgraph_checkpoint_ns\":\"write:810699d0-d241-c77b-aa0e-a027ab212727\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\", \"langgraph_step\": 2, \"langgraph_node\": \"write\", \"langgraph_triggers\": [\"branch:to:write\"], \"langgraph_path\": [\"__pregel_pull\", \"write\"], \"langgraph_checkpoint_ns\": \"write:6adb767f-ed1c-ab7e-f61d-53c0f979ae7f\"}"
                   }
                 },
                 {
@@ -757,17 +809,17 @@
               "flags": 256
             },
             {
-              "traceId": "2790928deea2b5a1cc09cae41cdc7b9b",
-              "spanId": "5926c6bcedd87dc2",
+              "traceId": "bb011b339b47b44ab98494833b7a430c",
+              "spanId": "87feccfee88f28d6",
               "name": "research_agent",
               "kind": 1,
-              "startTimeUnixNano": "1791012830405728000",
-              "endTimeUnixNano": "1791012835108648960",
+              "startTimeUnixNano": "1791061349477843968",
+              "endTimeUnixNano": "1791061354662377216",
               "attributes": [
                 {
                   "key": "input.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is an agent trace?\"}]}"
+                    "stringValue": "{\"messages\": [{\"role\": \"user\", \"content\": \"What is an agent trace?\"}]}"
                   }
                 },
                 {
@@ -779,7 +831,7 @@
                 {
                   "key": "output.value",
                   "value": {
-                    "stringValue": "{\"messages\":[{\"type\":\"human\",\"data\":{\"content\":\"What is an agent trace?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"name\":null,\"id\":\"944b9b8c-2bfb-4dc5-8b76-08bc9c4c8028\"}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of the steps an AI agent took to handle a task. It may include the input it received, actions or tool calls it made, results it got back, and the final response.\\n\\nFor example: *user asks for the weather → agent calls a weather tool → tool returns the forecast → agent summarizes it.*\\n\\nTraces help people debug and evaluate an agent’s behavior. They don’t necessarily reveal the model’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":191,\"prompt_tokens\":25,\"total_tokens\":216,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":88,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXeuCY2EjB5O5kTBNjfaCrtAq8U\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100ae-fcc7-77c2-b3dd-8522b20f04e5-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":25,\"output_tokens\":191,\"total_tokens\":216,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":88}}}},{\"type\":\"ai\",\"data\":{\"content\":\"An **agent trace** is a record of how an AI agent handled a task—such as the steps it took, tools it used, and results it received. Traces help with debugging and evaluation, but don’t necessarily show the agent’s full internal reasoning.\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token_usage\":{\"completion_tokens\":124,\"prompt_tokens\":125,\"total_tokens\":249,\"completion_tokens_details\":{\"accepted_prediction_tokens\":0,\"audio_tokens\":0,\"reasoning_tokens\":62,\"rejected_prediction_tokens\":0,\"text_tokens\":null},\"prompt_tokens_details\":{\"audio_tokens\":0,\"cache_write_tokens\":0,\"cached_tokens\":0,\"image_tokens\":null,\"text_tokens\":null,\"cache_creation_tokens\":0}},\"model_provider\":\"openai\",\"model_name\":\"openai/gpt-6-luna\",\"system_fingerprint\":null,\"id\":\"chatcmpl-EUoXh6pPDCqqY08c8to8pkmEfoP6m\",\"service_tier\":\"default\",\"finish_reason\":\"stop\",\"logprobs\":null},\"type\":\"ai\",\"name\":null,\"id\":\"lc_run--01a100af-065b-7190-833c-c11b443bf075-0\",\"tool_calls\":[],\"invalid_tool_calls\":[],\"usage_metadata\":{\"input_tokens\":125,\"output_tokens\":124,\"total_tokens\":249,\"input_token_details\":{\"audio\":0,\"cache_read\":0,\"cache_creation\":0},\"output_token_details\":{\"audio\":0,\"reasoning\":62}}}}]}"
+                    "stringValue": "{\"messages\": [{\"type\": \"human\", \"data\": {\"content\": \"What is an agent trace?\", \"additional_kwargs\": {}, \"response_metadata\": {}, \"type\": \"human\", \"name\": null, \"id\": \"f6b15395-342f-44c5-bfd1-fcaa3cd8c878\"}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of what an AI agent did during a task. It commonly includes the user\u2019s request, the agent\u2019s actions or tool calls, the results it received, and its final response. Some traces also record timestamps, errors, and other execution details.\\n\\nFor example, a trace might show that an agent searched the web, opened two pages, extracted information, and then answered a question.\\n\\nTraces are useful for debugging, evaluating, and monitoring agents. They don\u2019t necessarily include the agent\u2019s private internal reasoning; what\u2019s recorded depends on the system.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 223, \"prompt_tokens\": 25, \"total_tokens\": 248, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 96, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1ADGQYQLXMTkiPH7xDAmDKTcU1J\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-5467-77d0-aad0-f90b45a1d6af-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 25, \"output_tokens\": 223, \"total_tokens\": 248, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 96}}}}, {\"type\": \"ai\", \"data\": {\"content\": \"An **agent trace** is a chronological record of an AI agent\u2019s task: its actions or tool calls, the results it receives, and its final response. Traces help people debug and evaluate agents.\", \"additional_kwargs\": {\"refusal\": null}, \"response_metadata\": {\"token_usage\": {\"completion_tokens\": 96, \"prompt_tokens\": 149, \"total_tokens\": 245, \"completion_tokens_details\": {\"accepted_prediction_tokens\": 0, \"audio_tokens\": 0, \"reasoning_tokens\": 46, \"rejected_prediction_tokens\": 0, \"text_tokens\": null}, \"prompt_tokens_details\": {\"audio_tokens\": 0, \"cache_write_tokens\": 0, \"cached_tokens\": 0, \"image_tokens\": null, \"text_tokens\": null, \"cache_creation_tokens\": 0}}, \"model_provider\": \"openai\", \"model_name\": \"openai/gpt-6-luna\", \"system_fingerprint\": null, \"id\": \"chatcmpl-EV1AHQQGzXMvkUbS86kQTl6mTAMf6\", \"service_tier\": \"default\", \"finish_reason\": \"stop\", \"logprobs\": null}, \"type\": \"ai\", \"name\": null, \"id\": \"lc_run--01a10393-61dc-79b2-a2fb-94ed44373aaf-0\", \"tool_calls\": [], \"invalid_tool_calls\": [], \"usage_metadata\": {\"input_tokens\": 149, \"output_tokens\": 96, \"total_tokens\": 245, \"input_token_details\": {\"audio\": 0, \"cache_read\": 0, \"cache_creation\": 0}, \"output_token_details\": {\"audio\": 0, \"reasoning\": 46}}}}]}"
                   }
                 },
                 {
@@ -797,7 +849,7 @@
                 {
                   "key": "metadata",
                   "value": {
-                    "stringValue": "{\"ls_integration\":\"langgraph\"}"
+                    "stringValue": "{\"ls_integration\": \"langgraph\"}"
                   }
                 },
                 {