From 5d513d8053ae1d4a7826e0b58f5e5f5214eb5785 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 19:42:38 -0700 Subject: [PATCH 1/6] fix(lens): align source setup with available worker images (#44476) * fix(lens): document working preview setup before coordinated releases * docs(lens): keep current setup guidance factual and preserve Helm options * fix(lens): support explicit worker images on Compose 2 --- deploy/lens/README.md | 74 +++++++++++++--------- deploy/lens/compose.yaml | 2 +- docker/docker-compose.tracing.yml | 3 + tests/e2e/migrations/lens_compose_smoke.sh | 13 ++++ 4 files changed, 60 insertions(+), 32 deletions(-) diff --git a/deploy/lens/README.md b/deploy/lens/README.md index 4971bfae671..6f8c7d3940e 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -2,62 +2,74 @@ Lens reviews recorded activity and saves evidence-linked findings in the LiteLLM dashboard under Observability, Lens (`/ui/lens/`) -## Install the release stack +## Install -Each stable, RC, and dev release containing Lens publishes the worker at the same version on GHCR and Docker Hub. Use the [LiteLLM releases page](https://github.com/BerriAI/litellm/releases) to select a version that includes the coordinated worker release +Build LiteLLM and its worker from the same source commit with the same release identity. The worker runs separately and connects to your gateway using a limited worker token -For a new local installation, install Docker with Compose, download the two release files, and create a private environment file. Replace `X.Y.Z` with the release version, without `v` (RCs use `X.Y.Z-rc.N`) +### New local installation + +Install Docker with Compose and Git. This builds LiteLLM and its worker from the same checkout and starts the existing local tracing stack: ```bash -mkdir litellm-lens -cd litellm-lens -LENS_RELEASE=X.Y.Z -curl -fSLo compose.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/stack.yaml" -curl -fSLo config.yaml "https://raw.githubusercontent.com/BerriAI/litellm/v${LENS_RELEASE}/deploy/lens/config.yaml" -umask 077 -printf 'LITELLM_VERSION=%s\nLITELLM_MASTER_KEY=sk-%s\nLITELLM_SALT_KEY=sk-%s\n' \ - "$LENS_RELEASE" "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" > .env -printf 'POSTGRES_PASSWORD=%s\nCLICKHOUSE_PASSWORD=%s\n' \ - "$(openssl rand -hex 32)" "$(openssl rand -hex 32)" >> .env -docker compose up -d +git clone https://github.com/BerriAI/litellm.git +cd litellm +export LITELLM_RELEASE_TAG="sha-$(git rev-parse HEAD)" +export LENS_WORKER_IMAGE="litellm-lens-worker:${LITELLM_RELEASE_TAG}" +export OPENAI_API_KEY='sk-...' +docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \ + -f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" . +docker compose -f docker/docker-compose.tracing.yml up -d --build ``` -Open `http://localhost:4000/ui/`, log in as `admin` with `LITELLM_MASTER_KEY` from `.env`, and add a model in the dashboard. In Lens, select **Connect worker**, choose that model and a monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?**, copy the worker token, and add `LENS_WORKER_TOKEN=` to `.env` +Open `http://localhost:4002/ui/` and sign in as `admin` with password `sk-1234`. Go to **Lens > Investigations > Connect worker**, choose a model and monthly budget, then **Get install command**. Expand **Using Docker Compose or Helm?** and copy the worker token. In the same terminal, run: ```bash -docker compose --profile lens up -d +export LITELLM_URL=http://litellm:4000 +export LENS_WORKER_TOKEN='' +docker compose -f docker/docker-compose.tracing.yml -f deploy/lens/compose.yaml up -d ``` -The stack starts LiteLLM, PostgreSQL, ClickHouse, and the worker from published images. The dashboard shows **Worker connected**. The worker has a limited token, no database credentials, and no provider keys. The stack exposes only the dashboard on localhost; use your normal ingress and managed databases for a public production deployment +The worker joins the gateway's Docker network, and the dashboard shows **Worker connected**. Save the token privately for restarts and upgrades -Keep `.env` private and preserve its salt key. Keep both named database volumes. To upgrade, wait for active investigations to finish, stop the worker, change only `LITELLM_VERSION`, then pull and recreate the stack: +This stack is for local evaluation: it binds to localhost and uses development database credentials. For a hosted deployment, keep your normal database, keys, networking, and deployment process. Build both images from one source revision with the same `LITELLM_RELEASE_TAG`, publish the worker to your registry, and set `LENS_WORKER_IMAGE` on LiteLLM to that image + +### Existing LiteLLM installation + +Keep your deployment and PostgreSQL database. A working gateway/worker pair can stay as it is until you upgrade both. For a gateway built from source, use its exact commit and `LITELLM_RELEASE_TAG`; a release version or the latest commit on `main` is not a substitute for that source identity + +The public development package is `ghcr.io/berriai/litellm-lens-worker-dev:sha-`. It publishes amd64 images on Lens-related changes, so an arbitrary source commit may have no image. Check the exact image exists before using it. If it is unavailable, your gateway uses a different release identity, or you need native arm64, build the worker from the gateway's checkout: ```bash -docker compose --profile lens stop lens-worker -# Update LITELLM_VERSION in .env to the new release -docker compose --profile lens pull -docker compose --profile lens up -d +export LITELLM_RELEASE_TAG='' +export LENS_WORKER_IMAGE='/litellm-lens-worker:' +docker build --build-arg LITELLM_RELEASE_TAG="$LITELLM_RELEASE_TAG" \ + -f deploy/lens/Dockerfile -t "$LENS_WORKER_IMAGE" . ``` -This preserves your investigations, findings, model credentials, and worker token. Never use `down -v` during an upgrade. If moving from an existing installation, keep its databases and add the standalone worker instead of creating an empty replacement stack +For a remote worker host, publish that image to a registry the host can pull from. Set the gateway's `LENS_WORKER_IMAGE` to the resulting image reference, restart the gateway using its normal deployment process, then copy its install command. Prefer the published image digest for hosted installations. Do not change the gateway's release identity just to accept another worker + +For Kubernetes or Render, run the standalone worker using `LITELLM_URL` and `LENS_WORKER_TOKEN` from setup. Keep existing databases and secrets. The worker needs no inbound port. ## Helm -The componentized `helm/litellm` chart includes an optional Lens worker. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values: +The componentized source chart at `helm/litellm` includes an optional Lens worker. Use the chart from the same checkout as your gateway and keep your component image overrides in your values. Configure PostgreSQL and ClickHouse as usual, install the chart, then obtain a limited worker token from Lens setup. Store it in a Kubernetes Secret and enable the worker in your values: ```yaml lensWorker: enabled: true + image: + repository: + digest: sha256: tokenSecret: name: litellm-lens-worker key: token ``` -Published release charts pin the worker's approved image digest. Source charts without a digest default to the chart's application version. The chart connects the worker to the backend service. Keep these values and the Secret when upgrading the chart so the gateway and worker upgrade together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.digest` (or `tag` for a source build), and `lensWorker.url`. A digest takes precedence over the tag. The dashboard uses the chart's worker image for standalone install commands too +Set the worker repository and digest explicitly to an image built from the gateway's source commit and release identity. The chart connects the worker to the backend service. Keep these values and the Secret when upgrading the chart and update the gateway and worker image overrides together. `lensWorker.replicaCount` controls simultaneous investigations. To use a private registry or external proxy, set `lensWorker.image.repository`, `lensWorker.image.digest` (or `tag` for a source build), and `lensWorker.url`. A digest takes precedence over the tag. The dashboard uses the chart's worker image for standalone install commands too ## Standalone worker -Upgrade your existing LiteLLM proxy to a release that includes Lens with PostgreSQL and agent tracing. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries: +Start with a source deployment that includes Lens, PostgreSQL, and agent tracing, and prepare its matching worker as described above. Configure one ClickHouse URL for trace writes, bounded reads, and Lens queries: ```yaml general_settings: @@ -74,13 +86,13 @@ Retention changes require a proxy restart. ClickHouse removes expired rows durin In **Lens > Investigations**, click **Connect worker**, choose an analysis model and monthly limit, then **Get install command**. Use **Advanced options** to select an existing virtual key or change the proxy URL if the server running Docker needs a different network address. Copy the command and run it on your server. The dashboard shows **Worker connected** when the container checks in -The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. No source checkout, environment file, or second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis +The command already contains the compatible worker image and one worker token. The selected virtual key stays on the proxy; its secret is never sent to the worker. Once the matching image is available on the worker host, no second LiteLLM deployment is needed. Keep the command private because it includes the token. The LiteLLM release provides the dashboard and APIs; the container only runs background analysis -The dashboard selects the worker image matching the running gateway release. Release images support Linux amd64 and arm64. CI also publishes `:sha-` development images; use those only with a gateway built from the same commit and release tag +The dashboard uses the gateway's `LENS_WORKER_IMAGE` override when set. Public `:sha-` development images must match both the gateway commit and release identity. Build from source for the worker host's native architecture After upgrading the gateway, update the worker image and redeploy it while keeping its proxy URL and token. Existing containers do not update automatically. If an investigation reports a worker compatibility error, update the image before retrying -For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and `LITELLM_VERSION` (without `v`) in a private environment file. To use another registry, set `LENS_WORKER_IMAGE` to the compatible image instead of setting a version: +For deployments managed with Compose, download `compose.yaml` and provide `LITELLM_URL`, `LENS_WORKER_TOKEN`, and an explicit `LENS_WORKER_IMAGE` in a private environment file: ```bash docker compose --env-file /path/to/lens.env -f compose.yaml up -d @@ -219,9 +231,9 @@ Upgrades using `--use_prisma_db_push` stop before schema changes if any legacy L ## Release compatibility -Released gateway and worker images carry `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release +Gateway and worker builds carry the same `LITELLM_RELEASE_TAG`. A worker announces its release and protocol before claiming an investigation. A mismatch returns HTTP 409 with the required image, leaving queued investigations untouched. During a rolling upgrade, workers wait for a gateway from their release -The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Worker-only Compose accepts `LITELLM_VERSION` (without `v`) or an explicit `LENS_WORKER_IMAGE`. Release workers are available as `ghcr.io/berriai/litellm-lens-worker:vX.Y.Z` and `docker.io/litellm/litellm-lens-worker:vX.Y.Z`, including matching RC/dev suffixes, on amd64 and arm64 +The dashboard reads its image from the running gateway. `LENS_WORKER_IMAGE` overrides the registry/image for private deployments. Set an explicit `LENS_WORKER_IMAGE` for worker-only Compose. Verify that the image exists and matches the gateway before deploying it For source development, use `make lens-dev`, which gives the proxy and source worker the same commit identity. For custom containers, build both from the same checkout with `--build-arg LITELLM_RELEASE_TAG=sha-$(git rev-parse HEAD)` and set the proxy's `LENS_WORKER_IMAGE` to the worker image you built. An unlabelled custom build refuses worker setup and claims instead of guessing from the Python package version. Normal package-index installations use their installed release version diff --git a/deploy/lens/compose.yaml b/deploy/lens/compose.yaml index 799f0a4fb1e..aa915fef663 100644 --- a/deploy/lens/compose.yaml +++ b/deploy/lens/compose.yaml @@ -1,6 +1,6 @@ services: lens-worker: - image: ${LENS_WORKER_IMAGE:-ghcr.io/berriai/litellm-lens-worker:v${LITELLM_VERSION:?Set LITELLM_VERSION to the gateway release, without the v prefix}} + image: ${LENS_WORKER_IMAGE:-${LITELLM_VERSION:+ghcr.io/berriai/litellm-lens-worker:v}${LITELLM_VERSION:-}} environment: LITELLM_URL: ${LITELLM_URL:?Set the URL reachable from this container} LENS_WORKER_TOKEN: ${LENS_WORKER_TOKEN:?Create a worker credential in the Lens UI} diff --git a/docker/docker-compose.tracing.yml b/docker/docker-compose.tracing.yml index 8f960d50872..c8d90fbc0ae 100644 --- a/docker/docker-compose.tracing.yml +++ b/docker/docker-compose.tracing.yml @@ -5,6 +5,8 @@ services: build: context: .. target: runtime + args: + LITELLM_RELEASE_TAG: ${LITELLM_RELEASE_TAG:-} command: ["--config", "/app/tracing-config.yaml", "--port", "4000"] environment: LITELLM_MASTER_KEY: sk-1234 @@ -15,6 +17,7 @@ services: CLICKHOUSE_URL: http://default:local-tracing@clickhouse:8123 CLICKHOUSE_DATABASE: litellm OPENAI_API_KEY: ${OPENAI_API_KEY:-} + LENS_WORKER_IMAGE: ${LENS_WORKER_IMAGE:-} volumes: - ./tracing-config.yaml:/app/tracing-config.yaml:ro ports: diff --git a/tests/e2e/migrations/lens_compose_smoke.sh b/tests/e2e/migrations/lens_compose_smoke.sh index 69a13b58d88..6a57926895b 100644 --- a/tests/e2e/migrations/lens_compose_smoke.sh +++ b/tests/e2e/migrations/lens_compose_smoke.sh @@ -1,6 +1,19 @@ #!/usr/bin/env bash set -euo pipefail +worker_image() { + env -u LENS_WORKER_IMAGE -u LITELLM_VERSION \ + LITELLM_URL=http://litellm:4000 LENS_WORKER_TOKEN=config-test "$@" \ + docker compose --env-file /dev/null -f deploy/lens/compose.yaml config --images +} +[[ "$(worker_image LENS_WORKER_IMAGE=registry.example/lens:source)" == registry.example/lens:source ]] +[[ "$(worker_image LITELLM_VERSION=1.2.3)" == ghcr.io/berriai/litellm-lens-worker:v1.2.3 ]] +[[ "$(worker_image LENS_WORKER_IMAGE=registry.example/lens:source LITELLM_VERSION=1.2.3)" == registry.example/lens:source ]] +if worker_image > /dev/null 2>&1; then + printf 'Worker Compose accepted neither an image nor a release version\n' >&2 + exit 1 +fi + qa_dir=$(mktemp -d) master_key="sk-$(openssl rand -hex 32)" compose=(docker compose -p lens-compose-ci --env-file "$qa_dir/env" -f deploy/lens/stack.yaml) From b61376a99ef24f5e8d9fc94683f95bf9de67da7a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:10:30 -0700 Subject: [PATCH 2/6] feat(sdk): add run_tool_loop and arun_tool_loop helpers (#44381) * feat(sdk): add run_tool_loop and arun_tool_loop helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(tests): allow-list bounded tool-loop recursion in recursive detector Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): harden run_tool_loop per review Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(deps): keep uv.lock at revision 3 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(tests): authorize typing-extensions PSF-2.0 license Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Krrish Dholakia Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/__init__.py | 1 + litellm/constants.py | 2 + litellm/tool_loop.py | 172 ++++++++++ pyproject.toml | 1 + tests/code_coverage_tests/liccheck.ini | 1 + tests/unit/test_tool_loop.py | 420 +++++++++++++++++++++++++ uv.lock | 2 + 7 files changed, 599 insertions(+) create mode 100644 litellm/tool_loop.py create mode 100644 tests/unit/test_tool_loop.py diff --git a/litellm/__init__.py b/litellm/__init__.py index b6428b51bfb..fea7a27a5fb 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1474,6 +1474,7 @@ from .rust_bridge import rust from .rag.main import * from .sandbox.main import * from .decisions.main import * +from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop from .search.main import * from .realtime_api.main import ( _arealtime, diff --git a/litellm/constants.py b/litellm/constants.py index 49514fc4d0e..d58fc8a6318 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2234,3 +2234,5 @@ HARNESS_SNAPSHOT_SKIP_DIRS: Final = frozenset( ".ruff_cache", } ) + +DEFAULT_TOOL_LOOP_MAX_ROUNDS: Final = 20 diff --git a/litellm/tool_loop.py b/litellm/tool_loop.py new file mode 100644 index 00000000000..64d6de51818 --- /dev/null +++ b/litellm/tool_loop.py @@ -0,0 +1,172 @@ +"""Client-side tool-calling loop helpers for litellm.completion.""" + +from collections.abc import Awaitable, Callable, Sequence +from typing import TypeAlias, cast + +from typing_extensions import TypedDict, Unpack + +from litellm.constants import DEFAULT_TOOL_LOOP_MAX_ROUNDS +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionAssistantMessage, + ChatCompletionAssistantToolCall, + ChatCompletionToolCallFunctionChunk, + ChatCompletionToolMessage, + ChatCompletionToolParam, +) +from litellm.types.utils import ChatCompletionMessageToolCall, Message, ModelResponse + +ToolExecutor: TypeAlias = Callable[[ChatCompletionMessageToolCall], ChatCompletionToolMessage] +AsyncToolExecutor: TypeAlias = Callable[[ChatCompletionMessageToolCall], Awaitable[ChatCompletionToolMessage]] + + +class _ToolLoopCompletionKwargs(TypedDict, total=False, extra_items=object): + """Extra keywords forwarded verbatim to ``litellm.completion``, which owns their contract.""" + + +class ToolLoopMaxRoundsExceeded(RuntimeError): + max_rounds: int + + def __init__(self, max_rounds: int) -> None: + self.max_rounds = max_rounds + super().__init__(f"model still requested tool calls on round {max_rounds} of {max_rounds}") + + +def _validate_tool_loop_args(max_rounds: int, completion_kwargs: _ToolLoopCompletionKwargs) -> None: + if max_rounds < 1: + raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") + if completion_kwargs.get("stream"): + raise ValueError("run_tool_loop requires whole responses; stream=True is not supported") + + +def _expect_model_response(response: object) -> ModelResponse: + if not isinstance(response, ModelResponse): + raise TypeError(f"run_tool_loop requires completion to return a ModelResponse, got {type(response).__name__}") + return response + + +def _assistant_tool_call(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionAssistantToolCall: + return ChatCompletionAssistantToolCall( + id=tool_call.id, + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=tool_call.function.name, arguments=tool_call.function.arguments + ), + ) + + +def _assistant_message( + message: Message, tool_calls: tuple[ChatCompletionMessageToolCall, ...] +) -> ChatCompletionAssistantMessage: + return cast( # cast-ok: dict literal with thinking_blocks and reasoning_items spread in only when set + "ChatCompletionAssistantMessage", + { + "role": "assistant", + "content": message.content, + "tool_calls": [_assistant_tool_call(tc) for tc in tool_calls], + **{ + key: value + for key, value in ( + ("thinking_blocks", getattr(message, "thinking_blocks", None)), + ("reasoning_items", getattr(message, "reasoning_items", None)), + ) + if value is not None + }, + }, + ) + + +def _function_tool_call(tool_call: object) -> ChatCompletionMessageToolCall: + if not isinstance(tool_call, ChatCompletionMessageToolCall): + raise TypeError( + f"run_tool_loop only executes function tool calls, got custom tool call {getattr(tool_call, 'id', None)}" + ) + return tool_call + + +def _function_tool_calls(message: Message) -> tuple[ChatCompletionMessageToolCall, ...]: + return tuple(_function_tool_call(tool_call) for tool_call in message.tool_calls or ()) + + +def run_tool_loop( + *, + model: str, + messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam], + execute_tool: ToolExecutor, + max_rounds: int = DEFAULT_TOOL_LOOP_MAX_ROUNDS, + **completion_kwargs: Unpack[_ToolLoopCompletionKwargs], # kwargs-ok: forwarded verbatim to litellm.completion +) -> str | None: + """Call completion, execute each requested tool, and repeat until the model answers. + + Returns the final assistant message content. Raises ToolLoopMaxRoundsExceeded when the + model is still requesting tools after max_rounds completions. + + Example: + execute = functools.partial(run_repo_tool, repository="litellm", revision="main") + answer = litellm.run_tool_loop( + model="anthropic/claude-sonnet-5-5", messages=messages, tools=tools, execute_tool=execute + ) + """ + import litellm + + _validate_tool_loop_args(max_rounds, completion_kwargs) + history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns + for round_number in range(1, max_rounds + 1): + message = ( + _expect_model_response( + litellm.completion(model=model, messages=list(history), tools=list(tools), **completion_kwargs) + ) + .choices[0] + .message + ) + if not message.tool_calls: + return message.content + tool_calls = _function_tool_calls(message) + if round_number == max_rounds: + break + tool_results = tuple(execute_tool(tool_call) for tool_call in tool_calls) + history = (*history, _assistant_message(message, tool_calls), *tool_results) + raise ToolLoopMaxRoundsExceeded(max_rounds) + + +async def arun_tool_loop( + *, + model: str, + messages: Sequence[AllMessageValues], + tools: Sequence[ChatCompletionToolParam], + execute_tool: AsyncToolExecutor, + max_rounds: int = DEFAULT_TOOL_LOOP_MAX_ROUNDS, + **completion_kwargs: Unpack[_ToolLoopCompletionKwargs], # kwargs-ok: forwarded verbatim to litellm.acompletion +) -> str | None: + """Async version of run_tool_loop, awaiting each execute_tool call in order. + + Returns the final assistant message content. Raises ToolLoopMaxRoundsExceeded when the + model is still requesting tools after max_rounds completions. + + Example: + execute = functools.partial(arun_repo_tool, repository="litellm", revision="main") + answer = await litellm.arun_tool_loop( + model="anthropic/claude-sonnet-5-5", messages=messages, tools=tools, execute_tool=execute + ) + """ + import litellm + + _validate_tool_loop_args(max_rounds, completion_kwargs) + history: tuple[AllMessageValues, ...] = tuple(messages) # rebind-ok: rounds append new turns + for round_number in range(1, max_rounds + 1): + message = ( + _expect_model_response( + await litellm.acompletion(model=model, messages=list(history), tools=list(tools), **completion_kwargs) + ) + .choices[0] + .message + ) + if not message.tool_calls: + return message.content + tool_calls = _function_tool_calls(message) + if round_number == max_rounds: + break + tool_results = tuple([await execute_tool(tool_call) for tool_call in tool_calls]) + history = (*history, _assistant_message(message, tool_calls), *tool_results) + raise ToolLoopMaxRoundsExceeded(max_rounds) diff --git a/pyproject.toml b/pyproject.toml index d765bccb552..f04e66a04eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.1,<2.0", + "typing-extensions>=4.13.0,<5.0", ] [project.urls] diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 70c49c5c256..a62af1b3725 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -177,3 +177,4 @@ hypothesis: >=6.165.10 # MPL 2.0 license pytest-rerunfailures: >=15.1 # MPL 2.0 license pytest-recording: >=0.13.4 # MIT license expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE +typing-extensions: >=4.13.0 # PSF-2.0 license - https://github.com/python/typing_extensions/blob/main/LICENSE diff --git a/tests/unit/test_tool_loop.py b/tests/unit/test_tool_loop.py new file mode 100644 index 00000000000..c2aee58bb5f --- /dev/null +++ b/tests/unit/test_tool_loop.py @@ -0,0 +1,420 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.tool_loop import ToolLoopMaxRoundsExceeded +from litellm.types.llms.openai import ChatCompletionToolMessage +from litellm.types.utils import ChatCompletionMessageToolCall + +OPENAI_CHAT_COMPLETIONS_URL: Final = "https://api.openai.com/v1/chat/completions" +WEATHER_TOOLS: Final = ( + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + }, +) + + +def _openai_response(content: str | None, tool_calls: list | None = None) -> dict: + return { + "id": "chatcmpl-tool-loop", + "object": "chat.completion", + "created": 1739462947, + "model": "gpt-5-mini", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls" if tool_calls else "stop", + "message": { + "role": "assistant", + "content": content, + "tool_calls": tool_calls, + }, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + +def _tool_call(call_id: str, name: str, arguments: dict) -> dict: + return { + "id": call_id, + "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments)}, + } + + +def _tool_result(tool_call: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + return ChatCompletionToolMessage(role="tool", content='{"temp": "72F"}', tool_call_id=tool_call.id or "") + + +def _request_bodies(respx_mock: respx.MockRouter) -> list[dict]: + return [json.loads(call.request.content) for call in respx_mock.calls] + + +def test_final_answer_without_tool_calls_returns_content(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response("done")) + ) + executor_called: Final = [] + + def executor(tc: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executor_called.append(tc) + return _tool_result(tc) + + answer: Final = litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=executor, + api_key="sk-test", + ) + + assert answer == "done" + assert executor_called == [] + assert route.call_count == 1 + + +def test_two_rounds_appends_assistant_and_tool_messages_in_order(respx_mock: respx.MockRouter) -> None: + tool_calls: Final = [ + _tool_call("call_1", "get_weather", {"city": "Paris"}), + _tool_call("call_2", "get_weather", {"city": "Tokyo"}), + ] + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response(200, json=_openai_response(None, tool_calls)), + httpx.Response(200, json=_openai_response("Paris 72F, Tokyo 60F")), + ] + ) + executed: Final = [] + + def executor(tc: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tc) + return _tool_result(tc) + + messages: Final = [{"role": "user", "content": "weather in Paris and Tokyo?"}] + messages_snapshot: Final = [dict(message) for message in messages] + + answer: Final = litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=messages, + tools=WEATHER_TOOLS, + execute_tool=executor, + api_key="sk-test", + ) + + assert answer == "Paris 72F, Tokyo 60F" + assert route.call_count == 2 + assert [tc.id for tc in executed] == ["call_1", "call_2"] + assert [tc.function.name for tc in executed] == ["get_weather", "get_weather"] + assert [tc.function.arguments for tc in executed] == [ + '{"city": "Paris"}', + '{"city": "Tokyo"}', + ] + + second_body: Final = _request_bodies(respx_mock)[1] + assert second_body["messages"] == [ + {"role": "user", "content": "weather in Paris and Tokyo?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }, + { + "id": "call_2", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Tokyo"}'}, + }, + ], + }, + {"role": "tool", "content": '{"temp": "72F"}', "tool_call_id": "call_1"}, + {"role": "tool", "content": '{"temp": "72F"}', "tool_call_id": "call_2"}, + ] + + assert len(messages) == len(messages_snapshot) + assert messages == messages_snapshot + + +def test_response_format_and_tools_forwarded_every_round(respx_mock: respx.MockRouter) -> None: + respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response(200, json=_openai_response(None, [_tool_call("call_1", "get_weather", {"city": "Paris"})])), + httpx.Response(200, json=_openai_response('{"summary": "sunny"}')), + ] + ) + response_format: Final = { + "type": "json_schema", + "json_schema": { + "name": "weather_report", + "schema": { + "type": "object", + "properties": {"summary": {"type": "string"}}, + "required": ["summary"], + }, + }, + } + + litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "weather?"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + response_format=response_format, + api_key="sk-test", + ) + + bodies: Final = _request_bodies(respx_mock) + assert len(bodies) == 2 + for body in bodies: + assert body["response_format"] == response_format + assert body["tools"] == list(WEATHER_TOOLS) + + +def test_max_rounds_exceeded_raises_without_executing_last_round(respx_mock: respx.MockRouter) -> None: + tool_call: Final = _tool_call("call_1", "get_weather", {"city": "Paris"}) + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response(None, [tool_call])) + ) + executed: Final = [] + + def executor(tc: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tc) + return _tool_result(tc) + + with pytest.raises(ToolLoopMaxRoundsExceeded) as exc_info: + litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "weather?"}], + tools=WEATHER_TOOLS, + execute_tool=executor, + max_rounds=2, + api_key="sk-test", + ) + + assert exc_info.value.max_rounds == 2 + assert route.call_count == 2 + assert [tc.id for tc in executed] == ["call_1"] + + +def test_max_rounds_below_one_rejected_before_any_request(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response("done")) + ) + + with pytest.raises(ValueError, match="max_rounds must be >= 1"): + litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + max_rounds=0, + api_key="sk-test", + ) + + assert route.call_count == 0 + + +def test_stream_rejected_before_any_request(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=_openai_response("done")) + ) + + with pytest.raises(ValueError, match="stream=True is not supported"): + litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=_tool_result, + stream=True, + api_key="sk-test", + ) + + assert route.call_count == 0 + + +def test_custom_tool_call_raises_type_error_without_executing(respx_mock: respx.MockRouter) -> None: + custom_response: Final = _openai_response( + None, + [{"id": "call_custom", "type": "custom", "custom": {"name": "apply_patch", "input": "*** patch"}}], + ) + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + return_value=httpx.Response(200, json=custom_response) + ) + executor_called: Final = [] + + def executor(tc: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executor_called.append(tc) + return _tool_result(tc) + + with pytest.raises(TypeError, match="custom tool call call_custom"): + litellm.run_tool_loop( + model="openai/gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + tools=WEATHER_TOOLS, + execute_tool=executor, + api_key="sk-test", + ) + + assert executor_called == [] + assert route.call_count == 1 + + +def _responses_payload(response_id: str, output: list) -> dict: + return { + "id": response_id, + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-5.5", + "output": output, + "parallel_tool_calls": True, + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + "error": None, + "incomplete_details": None, + "instructions": None, + "metadata": None, + "temperature": None, + "tool_choice": "auto", + "tools": [], + "top_p": None, + "max_output_tokens": None, + "previous_response_id": None, + "reasoning": None, + "truncation": None, + "user": None, + } + + +def test_responses_bridge_replays_reasoning_items_across_rounds(respx_mock: respx.MockRouter) -> None: + round_one: Final = _responses_payload( + "resp_1", + [ + {"type": "reasoning", "id": "rs_abc123", "summary": [], "encrypted_content": "enc_xyz"}, + { + "type": "function_call", + "id": "fc_1", + "call_id": "call_1", + "name": "get_weather", + "arguments": '{"city": "Paris"}', + "status": "completed", + }, + ], + ) + round_two: Final = _responses_payload( + "resp_2", + [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Paris is 72F", "annotations": []}], + } + ], + ) + route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[httpx.Response(200, json=round_one), httpx.Response(200, json=round_two)] + ) + executed: Final = [] + + def executor(tc: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tc) + return _tool_result(tc) + + answer: Final = litellm.run_tool_loop( + model="openai/responses/gpt-5.5", + messages=[{"role": "user", "content": "weather in Paris?"}], + tools=WEATHER_TOOLS, + execute_tool=executor, + api_key="sk-test", + ) + + assert answer == "Paris is 72F" + assert route.call_count == 2 + assert [tc.id for tc in executed] == ["fc_1"] + + second_input: Final = _request_bodies(respx_mock)[1]["input"] + item_types: Final = [item.get("type") for item in second_input] + reasoning_index: Final = next(i for i, item in enumerate(second_input) if item.get("type") == "reasoning") + function_call_index: Final = next( + i for i, item in enumerate(second_input) if item.get("type") == "function_call" + ) + reasoning_item: Final = second_input[reasoning_index] + assert reasoning_item["id"] == "rs_abc123" + assert reasoning_item["encrypted_content"] == "enc_xyz" + assert reasoning_index < function_call_index, f"reasoning item must precede function_call: {item_types}" + + +async def test_arun_tool_loop_two_rounds(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler()) + tool_calls: Final = [ + _tool_call("call_1", "get_weather", {"city": "Paris"}), + _tool_call("call_2", "get_weather", {"city": "Tokyo"}), + ] + route: Final = respx_mock.post(OPENAI_CHAT_COMPLETIONS_URL).mock( + side_effect=[ + httpx.Response(200, json=_openai_response(None, tool_calls)), + httpx.Response(200, json=_openai_response("Paris 72F, Tokyo 60F")), + ] + ) + executed: Final = [] + + async def executor(tc: ChatCompletionMessageToolCall) -> ChatCompletionToolMessage: + executed.append(tc) + return _tool_result(tc) + + messages: Final = [{"role": "user", "content": "weather in Paris and Tokyo?"}] + messages_snapshot: Final = [dict(message) for message in messages] + + answer: Final = await litellm.arun_tool_loop( + model="openai/gpt-5-mini", + messages=messages, + tools=WEATHER_TOOLS, + execute_tool=executor, + api_key="sk-test", + ) + + assert answer == "Paris 72F, Tokyo 60F" + assert route.call_count == 2 + assert [tc.id for tc in executed] == ["call_1", "call_2"] + + second_body: Final = _request_bodies(respx_mock)[1] + assert second_body["messages"] == [ + {"role": "user", "content": "weather in Paris and Tokyo?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }, + { + "id": "call_2", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Tokyo"}'}, + }, + ], + }, + {"role": "tool", "content": '{"temp": "72F"}', "tool_call_id": "call_1"}, + {"role": "tool", "content": '{"temp": "72F"}', "tool_call_id": "call_2"}, + ] + assert messages == messages_snapshot diff --git a/uv.lock b/uv.lock index 0d1b17e6a6d..a5438188d4a 100644 --- a/uv.lock +++ b/uv.lock @@ -4521,6 +4521,7 @@ dependencies = [ { name = "pyyaml" }, { name = "tiktoken" }, { name = "tokenizers" }, + { name = "typing-extensions" }, ] [package.optional-dependencies] @@ -4851,6 +4852,7 @@ requires-dist = [ { name = "tokenizers", specifier = ">=0.21.0,<1.0" }, { name = "tomlkit", marker = "extra == 'cli'", specifier = ">=0.13.3,<1.0" }, { name = "tomlkit", marker = "extra == 'proxy'", specifier = ">=0.13.3,<1.0" }, + { name = "typing-extensions", specifier = ">=4.13.0,<5.0" }, { name = "uvicorn", marker = "extra == 'proxy'", specifier = ">=0.33.0,<1.0" }, { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" }, { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" }, From 29363076619795084143b42205db04c6b3a77932 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Sat, 3 Oct 2026 20:16:41 -0700 Subject: [PATCH 3/6] feat(lens): guide setup through the first investigation (#44475) * feat(lens): guide setup through the first investigation * fix(lens): restore the onboarding reference visuals * fix(lens): compact onboarding and animate gateway flow * feat(lens): refine onboarding motion and linked examples * feat(lens): turn the LED swarm into organized dot groups * fix(lens): make the LED dot flow visibly animate * feat(lens): refine the swarm scale palette and motion * fix(lens): preserve investigations during activity refresh errors --- .../lens/LensNavigation.integration.test.tsx | 36 ++- .../lens/LensSetup.integration.test.tsx | 274 ++++++++++++++++++ .../lens/LensWorkspace.integration.test.tsx | 21 +- .../src/components/lens/LensWorkspace.tsx | 120 ++++++-- .../InvestigationsView.integration.test.tsx | 8 +- .../investigations/InvestigationsWelcome.tsx | 50 ++-- .../src/components/lens/route.tsx | 8 +- .../src/components/lens/setup/GatewayFlow.tsx | 148 ++++++++++ .../lens/setup/LensGettingStarted.tsx | 227 +++++++++++++++ .../lens/setup/LensIntroduction.module.css | 114 ++++++++ .../lens/setup/LensIntroduction.tsx | 190 ++++++++++++ .../components/lens/setup/LensSetupSteps.tsx | 229 +++++++++++++++ .../src/components/lens/setup/useLensSetup.ts | 59 ++++ .../view_logs/TraceView/TracingSetupCard.tsx | 62 ++-- 14 files changed, 1444 insertions(+), 102 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/GatewayFlow.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.module.css create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts diff --git a/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx index 1ac4b5e6aef..769a77500e8 100644 --- a/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensNavigation.integration.test.tsx @@ -1,7 +1,7 @@ import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import LensPage from "@/app/(dashboard)/lens/page"; const { auth } = vi.hoisted(() => ({ auth: vi.fn() })); @@ -16,32 +16,44 @@ vi.mock("./investigations/InvestigationsView", () => ({ })); describe("Lens navigation", () => { - beforeEach(() => auth.mockReturnValue({ accessToken: "test-token", userRole: "Admin", isViewOnly: false })); + beforeEach(() => { + testQueryClient.clear(); + auth.mockReturnValue({ accessToken: "test-token", userRole: "Admin", isViewOnly: false }); + vi.stubGlobal( + "fetch", + vi.fn(async (input) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") return Response.json({ data: [{}] }); + if (path === "/lens") return Response.json({ lenses: [], workers: [], tracing_enabled: true }); + return Response.json({ traces: true, requests: false, data: [] }); + }), + ); + }); it("opens traces by default and pauses polling while viewing investigations", async () => { const user = userEvent.setup(); const onUrlUpdate = vi.fn(); renderWithProviders(, { onUrlUpdate }); expect(screen.getByRole("tab", { name: "Traces" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); await user.click(screen.getByRole("tab", { name: "Investigations" })); - expect(screen.getByText("Manage investigations")).toBeVisible(); + expect(await screen.findByText("Manage investigations")).toBeVisible(); expect(screen.getByText("Trace polling paused")).not.toBeVisible(); expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("tab")).toBe("investigations"); await user.click(screen.getByRole("tab", { name: "Traces" })); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); }); - it("opens existing lens links on investigations", () => { + it("opens existing lens links on investigations", async () => { renderWithProviders(, { searchParams: "?lens=saved-lens" }); expect(screen.getByRole("tab", { name: "Investigations" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByText("Manage investigations")).toBeVisible(); + expect(await screen.findByText("Manage investigations")).toBeVisible(); }); - it("honors an explicit traces tab even when a saved investigation is in the URL", () => { + it("honors an explicit traces tab even when a saved investigation is in the URL", async () => { renderWithProviders(, { searchParams: "?tab=traces&lens=saved-lens" }); expect(screen.getByRole("tab", { name: "Traces" })).toHaveAttribute("aria-selected", "true"); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); }); it.each(["Internal User", "Internal Viewer", "Org Admin"])( @@ -50,7 +62,7 @@ describe("Lens navigation", () => { auth.mockReturnValue({ accessToken: "test-token", userRole, isViewOnly: false }); const user = userEvent.setup(); renderWithProviders(); - expect(screen.getByText("Trace polling active")).toBeVisible(); + expect(await screen.findByText("Trace polling active")).toBeVisible(); await user.click(screen.getByRole("tab", { name: "Investigations" })); expect(screen.getByText(/Investigations require proxy administrator access/)).toBeVisible(); expect(screen.queryByText("Manage investigations")).not.toBeInTheDocument(); @@ -60,10 +72,10 @@ describe("Lens navigation", () => { it.each([ { userRole: "Admin Viewer", isViewOnly: false }, { userRole: "Admin", isViewOnly: true }, - ])("preserves read-only investigation access for $userRole with isViewOnly=$isViewOnly", (session) => { + ])("preserves read-only investigation access for $userRole with isViewOnly=$isViewOnly", async (session) => { auth.mockReturnValue({ accessToken: "test-token", ...session }); renderWithProviders(, { searchParams: "?tab=investigations" }); - expect(screen.getByText("Read-only investigations")).toBeVisible(); + expect(await screen.findByText("Read-only investigations")).toBeVisible(); expect(screen.queryByText("Manage investigations")).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx new file mode 100644 index 00000000000..036b02f04c4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx @@ -0,0 +1,274 @@ +import { act, fireEvent, screen, within, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { LensWorkspace } from "./LensWorkspace"; +import { createLensDemoData } from "./demo/createLensDemo"; +import type { LensList } from "./model/types"; + +const network = vi.fn(); +const list = vi.fn<() => Promise>(); +const data = createLensDemoData(); +const worker = () => ({ + id: "setup-worker", + analysis_key_id: "a".repeat(64), + revoked: false, + last_seen: new Date().toISOString(), + scope: data.lenses[0].scope, +}); + +function serve({ enabled = false, traces = false, requests = false, connected = false } = {}) { + list.mockResolvedValue({ lenses: [], workers: connected ? [worker()] : [], tracing_enabled: enabled }); + network.mockImplementation(async (input, init) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") + return enabled + ? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] }) + : Response.json({ detail: "Tracing is not enabled" }, { status: 501 }); + if (path === "/lens/activity/available") return Response.json({ traces, requests }); + if (path === "/lens" && init?.method === "POST") { + const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...JSON.parse(String(init.body)) } }; + list.mockResolvedValue({ lenses: [saved], workers: [worker()], tracing_enabled: true }); + return Response.json(saved); + } + if (path === "/lens") return Response.json(await list()); + if (path === "/key/generate") return Response.json({ token_id: worker().analysis_key_id }); + if (path === "/lens/workers/register") { + list.mockResolvedValue({ lenses: [], workers: [worker()], tracing_enabled: true }); + return Response.json({ worker: worker(), token: "test-worker-token", image: "test-worker-image" }); + } + if (path === "/models") return Response.json({ data: [{ id: "analysis" }] }); + if (path === "/model_group/info") + return Response.json({ data: [{ model_group: "analysis", providers: ["OpenAI"], mode: "chat" }] }); + if (path === "/key/info") return Response.json({ info: { models: ["analysis"], max_budget: 100 } }); + if (path === "/lens/agents") return Response.json(["support_agent"]); + if (path === "/lens/preview/sample") return Response.json({ eligible: 1, selected: 1, executions: [] }); + if (path.endsWith("/runs")) return Response.json(data.lenses[0].jobs); + return Response.json({ data: [] }); + }); +} + +beforeEach(() => { + testQueryClient.clear(); + network.mockReset(); + list.mockReset(); + vi.stubGlobal("fetch", network); + Element.prototype.scrollIntoView = vi.fn(); + serve(); +}); + +describe("Lens setup journey", () => { + it.each(["/lens", "/lens/activity/available"])( + "keeps recorded traces visible while %s is pending", + async (pendingPath) => { + serve({ enabled: true, traces: true }); + const normal = network.getMockImplementation()!; + network.mockImplementation((input, init) => + new URL(String(input), "http://localhost").pathname === pendingPath + ? new Promise(() => {}) + : normal(input, init), + ); + renderWithProviders(); + expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); + }, + ); + + it.each(["/v1/traces", "/lens/activity/available"])( + "opens a saved investigation while %s is pending", + async (pendingPath) => { + serve(); + list.mockResolvedValue({ lenses: data.lenses, workers: [worker()], tracing_enabled: false }); + const normal = network.getMockImplementation()!; + network.mockImplementation((input, init) => + new URL(String(input), "http://localhost").pathname === pendingPath + ? new Promise(() => {}) + : normal(input, init), + ); + renderWithProviders(, { + searchParams: `?lens=${data.lenses[0].id}`, + }); + expect(await screen.findByRole("heading", { name: data.lenses[0].settings.name })).toBeVisible(); + }, + ); + + it("shares the introduction across tabs and stays in setup after the first trace", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { onUrlUpdate }); + expect(await screen.findByRole("heading", { name: "Before you start" })).toBeVisible(); + await user.click(screen.getByRole("tab", { name: "Investigations" })); + expect(screen.getByRole("heading", { name: "Before you start" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Set up Lens" })); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("setup")).toBe("lens")); + serve({ enabled: true }); + await user.click(screen.getByRole("button", { name: "Check setup" })); + expect(await screen.findByText("Trace storage is connected")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Continue to your agent" })); + expect(screen.getByRole("button", { name: "Check for traces" })).toBeVisible(); + serve({ enabled: true, traces: true }); + await user.click(screen.getByRole("button", { name: "Check for traces" })); + expect(await screen.findByText(/Your first trace is ready/)).toBeVisible(); + expect(screen.queryByRole("table", { name: "Agent runs" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Continue to worker" })); + const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); + await user.click(connection.getByRole("combobox", { name: "Analysis model" })); + await user.click(await screen.findByRole("option", { name: "analysis" })); + await user.click(connection.getByRole("button", { name: "Get install command" })); + const connected = within(await screen.findByRole("dialog", { name: "Worker connected" })); + await user.click(connected.getByRole("button", { name: "New investigation" })); + expect(await screen.findByRole("dialog", { name: "Which activity should we investigate?" })).toBeVisible(); + expect(screen.getByRole("heading", { name: "Get Lens running", hidden: true })).toBeInTheDocument(); + }); + + it("resumes setup after a reload and leaves only when the user chooses traces", async () => { + serve({ enabled: true, traces: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?tab=investigations&setup=lens", + }); + expect(await screen.findByRole("button", { name: "Connect worker" })).toBeVisible(); + await user.click(screen.getByRole("tab", { name: "Traces" })); + expect(screen.getByRole("heading", { name: "Get Lens running" })).toBeVisible(); + await user.click(screen.getByRole("button", { name: "View traces" })); + expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); + await user.click(screen.getByRole("tab", { name: "Investigations" })); + expect(await screen.findByRole("heading", { name: "Run your first investigation" })).toBeVisible(); + expect(screen.queryByRole("link", { name: "Set up traces" })).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Get Lens running" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Set up Lens" })); + expect(await screen.findByRole("heading", { name: "Get Lens running" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeVisible(); + }); + + it("allows request-only investigations without forcing agent instrumentation", async () => { + serve({ requests: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?tab=investigations", + }); + expect(await screen.findByText("Request logs received")).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeEnabled(); + expect(screen.queryByRole("heading", { name: "Before you start" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Set up Lens" })); + expect(await screen.findByRole("heading", { name: "Before you start" })).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeEnabled(); + await user.click(screen.getByRole("button", { name: /Enable tracing on the gateway/ })); + expect(screen.getByRole("button", { name: "Continue with request logs" })).toBeEnabled(); + await user.click(screen.getByRole("button", { name: /Send your first trace/ })); + await user.click(screen.getByRole("button", { name: "Continue with request logs" })); + const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); + await user.click(connection.getByRole("combobox", { name: "Analysis model" })); + await user.click(await screen.findByRole("option", { name: "analysis" })); + await user.click(connection.getByRole("button", { name: "Get install command" })); + const connected = within(await screen.findByRole("dialog", { name: "Worker connected" })); + await user.click(connected.getByRole("button", { name: "New investigation" })); + expect(await screen.findByRole("dialog", { name: "Which activity should we investigate?" })).toBeVisible(); + }); + + it("keeps setup recoverable when checking for a first trace fails", async () => { + serve({ enabled: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?setup=lens", + }); + await screen.findByRole("button", { name: "Check for traces" }); + const normal = network.getMockImplementation()!; + network.mockImplementation(async (input, init) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces") return Response.json({ detail: "Trace storage unavailable" }, { status: 503 }); + return normal(input, init); + }); + await user.click(screen.getByRole("button", { name: "Check for traces" })); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not check setup"); + expect(screen.queryByRole("button", { name: "Continue to worker" })).not.toBeInTheDocument(); + serve({ enabled: true, traces: true }); + await user.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByRole("button", { name: "Continue to worker" })).toBeEnabled(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + }); + + it("keeps request-only users in investigations when an activity refresh fails", async () => { + serve({ requests: true, connected: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?tab=investigations", + }); + expect(await screen.findByText("Request logs received")).toBeVisible(); + const normal = network.getMockImplementation()!; + network.mockImplementation((input, init) => + new URL(String(input), "http://localhost").pathname === "/lens/activity/available" + ? Promise.resolve(Response.json({ detail: "Activity unavailable" }, { status: 503 })) + : normal(input, init), + ); + await act(() => testQueryClient.refetchQueries()); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not check recorded activity"); + expect(screen.queryByRole("heading", { name: "Get Lens running" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "New investigation" })).toBeDisabled(); + network.mockImplementation(normal); + await user.click(screen.getByRole("button", { name: "Retry" })); + expect(await screen.findByText("Request logs received")).toBeVisible(); + expect(screen.getByRole("button", { name: "New investigation" })).toBeEnabled(); + }); + + it("keeps administrator-only setup unavailable to trace viewers", async () => { + serve({ enabled: true, traces: true }); + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: "?setup=lens", + }); + expect(await screen.findByText(/A gateway administrator can connect a worker/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeDisabled(); + await user.click(screen.getByRole("tab", { name: "Investigations" })); + expect(screen.getByRole("button", { name: "Connect worker" })).toBeDisabled(); + expect(network.mock.calls.some(([input]) => new URL(String(input), "http://localhost").pathname === "/lens")).toBe( + false, + ); + }); + + it.each(["traces", "requests with trace errors", "requests with pending traces", "traces with activity errors"])( + "finishes guided setup with %s and opens the saved investigation", + async (scenario) => { + const source = scenario.startsWith("requests") ? "requests" : "traces"; + const activity = { enabled: true, traces: source === "traces", requests: source === "requests", connected: true }; + serve(activity); + const normal = network.getMockImplementation()!; + const failingPath = scenario === "requests with trace errors" ? "/v1/traces" : "/lens/activity/available"; + network.mockImplementation((input, init) => { + const path = new URL(String(input), "http://localhost").pathname; + if (path === "/v1/traces" && scenario === "requests with pending traces") + return new Promise(() => {}); + if (scenario.endsWith("errors") && path === failingPath) + return Promise.resolve(Response.json({ detail: "Activity unavailable" }, { status: 503 })); + return normal(input, init); + }); + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { + searchParams: "?setup=lens", + onUrlUpdate, + }); + await user.click(await screen.findByRole("button", { name: "New investigation" })); + const dialog = within(screen.getByRole("dialog")); + fireEvent.change(dialog.getByRole("textbox", { name: "Investigation name" }), { + target: { value: "My first review" }, + }); + await user.click(dialog.getByRole("button", { name: "Continue" })); + await user.click(dialog.getByRole("button", { name: "Continue" })); + await waitFor(() => expect(dialog.getByRole("button", { name: "Run and monitor" })).toBeEnabled()); + await user.click(dialog.getByRole("button", { name: "Run and monitor" })); + expect(await screen.findByRole("heading", { name: "My first review" })).toBeVisible(); + expect(screen.queryByRole("heading", { name: "Get Lens running" })).not.toBeInTheDocument(); + expect( + within(screen.getByRole("tablist", { name: "Lens" })).getByRole("tab", { name: "Investigations" }), + ).toHaveAttribute("aria-selected", "true"); + await waitFor(() => expect(onUrlUpdate.mock.lastCall?.[0].searchParams.get("setup")).toBeNull()); + const create = network.mock.calls.find( + ([input, init]) => new URL(String(input), "http://localhost").pathname === "/lens" && init?.method === "POST", + ); + expect(create).toBeDefined(); + expect(JSON.parse(String(create?.[1]?.body))).toEqual( + expect.objectContaining({ name: "My first review", source }), + ); + }, + ); +}); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index 999d2f11980..41ce6ed77c6 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -30,8 +30,9 @@ describe("Lens interactive demo", () => { renderWithProviders(, { onUrlUpdate, }); - expect(await screen.findByText("Enable tracing")).toBeVisible(); - await user.click(screen.getByRole("button", { name: "Preview sample" })); + expect(await screen.findByRole("heading", { name: "The gateway that helps your agents improve" })).toBeVisible(); + expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Explore with sample data" })); expect(await screen.findByText("Where is order #1042?")).toBeVisible(); expect(screen.getByRole("switch", { name: "Demo data" })).toBeChecked(); await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true")); @@ -84,7 +85,7 @@ describe("Lens interactive demo", () => { await expectUrl(onUrlUpdate, (url) => expect(url.get("view")).toBe("conversation")); expect(network).not.toHaveBeenCalled(); await user.click(screen.getByRole("switch", { name: "Demo data" })); - expect(await screen.findByText("Enable tracing")).toBeVisible(); + expect(await screen.findByRole("heading", { name: "The gateway that helps your agents improve" })).toBeVisible(); await expectUrl(onUrlUpdate, (url) => expect([...url.keys()]).toEqual([])); }); @@ -93,7 +94,7 @@ describe("Lens interactive demo", () => { const onUrlUpdate = vi.fn(); network.mockResolvedValue(Response.json({ detail: "Tracing is not enabled" }, { status: 501 })); renderWithProviders(, { - searchParams: "?tab=traces&trace=live-trace&span=live-span&lens=live-lens", + searchParams: "?tab=traces&setup=lens&trace=live-trace&span=live-span&lens=live-lens", onUrlUpdate, }); await user.click(await screen.findByRole("switch", { name: "Demo data" })); @@ -129,8 +130,7 @@ describe("Lens interactive demo", () => { searchParams: "?tab=investigations", onUrlUpdate, }); - await screen.findByRole("button", { name: "Preview sample" }); - await user.click(screen.getByRole("button", { name: "Preview sample" })); + await user.click(await screen.findByRole("button", { name: "Explore with sample data" })); network.mockClear(); await user.click(await screen.findByRole("row", { name: /Repeated lookups leave customers without an answer/ })); const finding = screen.getByRole("dialog"); @@ -155,7 +155,7 @@ describe("Lens interactive demo", () => { await expectUrl(onUrlUpdate, (url) => expect(url.get("demo")).toBe("true")); await expectUrl(onUrlUpdate, (url) => expect(url.has("span")).toBe(false)); await user.click(screen.getByRole("switch", { name: "Demo data" })); - expect(await screen.findByRole("heading", { name: "Find what needs attention" })).toBeVisible(); + expect(await screen.findByRole("heading", { name: "The gateway that helps your agents improve" })).toBeVisible(); }); it("has no demo entry for existing investigations, populated traces, or connecting another agent", async () => { @@ -180,7 +180,7 @@ describe("Lens interactive demo", () => { expect(screen.queryByRole("button", { name: "Set up tracing" })).not.toBeInTheDocument(); }); - it("shows the header preview only for the active tab that still needs setup", async () => { + it("offers sample data in the main panel only for the active tab that needs setup", async () => { const user = userEvent.setup(); const saved = createLensDemoData().lenses[0]; network.mockImplementation(async (input) => { @@ -191,13 +191,14 @@ describe("Lens interactive demo", () => { return Response.json({ data: [], traces: false, requests: false }); }); renderWithProviders(); - expect(await screen.findByRole("button", { name: "Preview sample" })).toBeVisible(); + expect(await screen.findByRole("button", { name: "Explore with sample data" })).toBeVisible(); const tabs = within(screen.getByRole("tablist", { name: "Lens" })); await user.click(tabs.getByRole("tab", { name: "Investigations" })); expect(await screen.findByRole("row", { name: new RegExp(saved.settings.name) })).toBeVisible(); expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Explore with sample data" })).not.toBeInTheDocument(); await user.click(tabs.getByRole("tab", { name: "Traces" })); - await user.click(await screen.findByRole("button", { name: "Preview sample" })); + await user.click(await screen.findByRole("button", { name: "Explore with sample data" })); expect(await screen.findByRole("table", { name: "Agent runs" })).toBeVisible(); expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx index bac37e55641..63bef720211 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.tsx @@ -3,7 +3,7 @@ import { useId, useState } from "react"; import { useQuery } from "@tanstack/react-query"; import { Tabs as TabsPrimitive } from "@base-ui/react/tabs"; -import { Activity, Aperture, ArrowUpRight, ScanSearch } from "lucide-react"; +import { Activity, Aperture, ArrowUpRight, Loader2, ScanSearch } from "lucide-react"; import AgentTracesPage from "@/components/view_logs/TraceView/AgentTracesPage"; import { Switch } from "@/components/ui/switch"; import { Tabs, TabsContent } from "@/components/ui/tabs"; @@ -17,6 +17,9 @@ import { useLensApi } from "./services"; import { investigationActivity, type InvestigationActivity } from "./model/status"; import { cn } from "@/lib/cva.config"; import { LENS_TABS, useLensRoute, type LensTab } from "./route"; +import { Button } from "@/components/ui/button"; +import { LensGettingStarted } from "./setup/LensGettingStarted"; +import { useLensSetup, type LensSetupState } from "./setup/useLensSetup"; type WorkspaceProps = { accessToken: string; userRole: string; readOnly: boolean }; @@ -144,15 +147,51 @@ const PANEL = "flex min-h-0 flex-1 flex-col overflow-y-auto animate-in fade-in-0 duration-300 motion-reduce:animate-none"; function LensContent({ accessToken, userRole, readOnly }: WorkspaceProps) { - const { tab, lensId, demo, setTab, setDemo } = useLensRoute(); + const { tab, lensId, demo, settingUp, setTab, setLensId, setDemo, setSetup } = useLensRoute(); const [previewTarget, setPreviewTarget] = useState(null); const activeTab = tab ?? (lensId ? "investigations" : "traces"); const canInvestigate = isProxyAdminTierRole(userRole); + const setupState = useLensSetup(accessToken, !demo, canInvestigate, settingUp); + const setupLocation = { tab: activeTab, canInvestigate, selected: !!lensId, requested: settingUp }; + const showSetup = !demo && needsSetup(setupState, setupLocation); + const showSetupButton = !demo && !showSetup && isProxyAdminRole(userRole); + const startSetup = () => { + if (!settingUp) setSetup(true); + }; + const showTraces = () => { + setSetup(false); + setTab(setupState.tracesReady ? "traces" : "investigations"); + }; + const showCreated = (id: string) => { + setSetup(false); + setLensId(id); + setTab("investigations"); + }; const activity = useInvestigationActivity(accessToken, canInvestigate); const preview = (view: LensTab) => ({ target: previewTarget, open: !demo && activeTab === view ? () => setDemo(true) : undefined, }); + const setupContent = setupState.loading ? ( +

+

+ ) : ( + + setDemo(true)} + /> + + ); return (
setTab(value as LensTab)} className="min-h-0 flex-1 gap-0"> @@ -177,38 +216,65 @@ function LensContent({ accessToken, userRole, readOnly }: WorkspaceProps) {
+ {showSetupButton && ( + + )}
- - - - - - - - {canInvestigate ? ( - - ) : ( -

- Investigations require proxy administrator access. You can still view your traces. -

- )} - - + {showSetup ? ( + setupContent + ) : ( + <> + + + + + + + + {canInvestigate ? ( + + ) : ( +

+ Investigations require proxy administrator access. You can still view your traces. +

+ )} + + + + )}
); } + +function needsSetup( + state: LensSetupState, + { + tab, + canInvestigate, + selected, + requested, + }: { tab: LensTab; canInvestigate: boolean; selected: boolean; requested: boolean }, +) { + if (requested) return true; + if (!state.missingTraces) return false; + if (tab === "traces") return true; + const hasActivity = state.hasInvestigations || state.hasRequests || selected; + return canInvestigate && !hasActivity; +} diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx index 05bebc9b923..26445880070 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx @@ -309,12 +309,10 @@ it("guides a first-time administrator into worker connection and lens setup", as }); const user = userEvent.setup(); renderWithProviders(withPreview(, vi.fn())); - const guide = within(await screen.findByRole("region", { name: "Find what needs attention" })); + const guide = within(await screen.findByRole("region", { name: "Run your first investigation" })); expect(apiClient.get).toHaveBeenCalledWith("/lens/activity/available", { accessToken: "test" }); - expect(await guide.findByRole("link", { name: "View traces" })).toHaveAttribute( - "href", - expect.stringMatching(/^\/ui\/lens\/?\?tab=traces$/), - ); + expect(guide.getByText("Traces received")).toBeVisible(); + expect(guide.queryByRole("link", { name: "View traces" })).not.toBeInTheDocument(); expect(await screen.findByRole("button", { name: "Preview sample" })).toBeVisible(); await user.click(guide.getByRole("button", { name: "Connect worker" })); const connection = within(await screen.findByRole("dialog", { name: "Connect a worker" })); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx index 6e39e2e2f3e..6b0b20ead4b 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx @@ -34,36 +34,50 @@ export function InvestigationsWelcome({ const canConnect = activityReady && !readOnly; const traceStatus = activityStatus(tracesReady, requestsReady, checking); const waitingForWorker = activityReady && !connected; - const firstStepTitle = requestsReady && !tracesReady ? "Recorded activity" : "Set up traces"; + const stepOffset = Number(!activityReady); + const introduction = activityReady + ? { + title: "Run your first investigation", + description: "Your recorded activity is ready. Connect a worker and choose what Lens should look for.", + } + : { + title: "Find what needs attention", + description: "Check how your agents behave. Get findings you can trace back to what happened.", + }; const traceButtonClass = buttonVariants({ - variant: activityReady ? "ghost" : "default", + variant: "default", className: "col-start-2 w-fit sm:col-start-auto", }); return (
{showPreview && }

- Find what needs attention + {introduction.title}

-

- Check how your agents behave. Get findings you can trace back to what happened. -

+

{introduction.description}

+ {activityReady && ( +

+ {traceStatus} +

+ )}
    -
  1. - -
    -

    {firstStepTitle}

    -

    {traceStatus}

    -
    - - {tracesReady ? "View traces" : "Set up traces"} - -
    2. + {!activityReady && ( +
  2. + +
    +

    Set up traces

    +

    {traceStatus}

    +
    + + Set up traces + +
    3. + )}
  3. - +

    Connect a worker

    @@ -83,7 +97,7 @@ export function InvestigationsWelcome({ data-state={workerReady ? "active" : "inactive"} className="grid grid-cols-[28px_minmax(0,1fr)] items-center gap-x-4 gap-y-3 py-5 text-muted-foreground data-[state=active]:text-foreground sm:grid-cols-[28px_minmax(0,1fr)_auto]" > - +

    Run an investigation

    diff --git a/ui/litellm-dashboard/src/components/lens/route.tsx b/ui/litellm-dashboard/src/components/lens/route.tsx index 9d07973fd09..abc5cf2c0ad 100644 --- a/ui/litellm-dashboard/src/components/lens/route.tsx +++ b/ui/litellm-dashboard/src/components/lens/route.tsx @@ -18,6 +18,7 @@ const LENS_PARSERS = { tab: parseAsStringLiteral(lensTabs), lens: parseAsString, demo: parseAsBoolean.withDefault(false), + setup: parseAsStringLiteral(["lens"]), }; const ISSUE_PARSERS = { issue: parseAsString }; @@ -56,14 +57,16 @@ export interface LensRoute { readonly tab: LensTab | null; readonly lensId: string | null; readonly demo: boolean; + readonly settingUp: boolean; setTab(tab: LensTab): void; setLensId(lensId: string | null): void; setDemo(demo: boolean): void; + setSetup(settingUp: boolean): void; } /** Lens navigation lives in the URL, sample session included, so any view is a shareable link. */ export function useLensRoute(): LensRoute { - const [{ tab, lens, demo }, setParams] = useQueryStates(SESSION_PARSERS, { history: "push" }); + const [{ tab, lens, demo, setup }, setParams] = useQueryStates(SESSION_PARSERS, { history: "push" }); const setTab = useCallback((next: LensTab) => void setParams({ tab: next }), [setParams]); const setLensId = useCallback( (next: string | null) => void setParams({ ...CLEARED_RESULTS, lens: next }), @@ -73,7 +76,8 @@ export function useLensRoute(): LensRoute { (next: boolean) => void setParams(next ? { ...CLEARED_SESSION, demo: true } : CLEARED_SESSION), [setParams], ); - return { tab, lensId: lens, demo, setTab, setLensId, setDemo }; + const setSetup = useCallback((next: boolean) => void setParams({ setup: next ? "lens" : null }), [setParams]); + return { tab, lensId: lens, demo, settingUp: setup === "lens", setTab, setLensId, setDemo, setSetup }; } export function useIssueRoute() { diff --git a/ui/litellm-dashboard/src/components/lens/setup/GatewayFlow.tsx b/ui/litellm-dashboard/src/components/lens/setup/GatewayFlow.tsx new file mode 100644 index 00000000000..bc61078a18d --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/GatewayFlow.tsx @@ -0,0 +1,148 @@ +import { useId } from "react"; +import styles from "./LensIntroduction.module.css"; + +const dotColors = [ + "fill-sky-400 dark:fill-sky-300", + "fill-indigo-400 dark:fill-indigo-300", + "fill-violet-400 dark:fill-violet-300", +]; + +const swarmRows = Array.from({ length: 11 }, (_, row) => ({ + y: 4 + row * 10, + duration: `${[4.8, 5.2, 5.6, 5][row % 4]}s`, + delay: `${-row * 0.17}s`, + dots: Array.from({ length: 80 }, (_, column) => { + const seed = (column % 24) + row * 31; + const variation = (seed * 73 + seed * seed * 19) % 101; + return { + x: column * 10 - 235, + color: dotColors[variation % dotColors.length], + opacity: variation < 15 ? 0 : 0.5 + variation * 0.005, + }; + }), +})); + +const organizedColumns = Array.from({ length: 58 }, (_, column) => column - 8); + +export function GatewayFlow() { + const id = useId(); + return ( +
    +
    +
    +

    Agent swarms

    +

    Every run, every recorded step

    +
    +
    +

    LiteLLM gateway

    +

    One place, your infrastructure

    +
    +
    +

    Lens

    +

    Findings to improve your agents

    +
    +
    + +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx b/ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx new file mode 100644 index 00000000000..01b62c864da --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx @@ -0,0 +1,227 @@ +"use client"; + +import { useRef, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { Check, ShieldCheck } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import type { TraceSummary } from "@/components/view_logs/TraceView/traceTypes"; +import { lensQueries } from "../api/queries"; +import { useSaveLens } from "../api/mutations"; +import { useLensApi } from "../services"; +import { TraceSheet } from "../investigations/TraceSheet"; +import { useAnalysisKeyInfo } from "./worker/AnalysisKeyDetails"; +import { WorkerDialog } from "./worker/WorkerDialog"; +import { InvestigationSetupDialog } from "./InvestigationSetupDialog"; +import { LensIntroduction } from "./LensIntroduction"; +import type { LensSetupState } from "./useLensSetup"; +import type { Settings } from "../model/types"; +import { initialSetupStep, LensSetupSteps } from "./LensSetupSteps"; + +export function LensGettingStarted({ + accessToken, + state, + readOnly, + canInvestigate, + canMintTracingKey, + onStart, + onExit, + onCreated, + onDemo, +}: { + accessToken: string; + state: LensSetupState; + readOnly: boolean; + canInvestigate: boolean; + canMintTracingKey: boolean; + onStart: () => void; + onExit: () => void; + onCreated: (id: string) => void; + onDemo?: () => void; +}) { + const setupRef = useRef(null); + const [step, setStep] = useState(() => initialSetupStep(state)); + const [workerOpen, setWorkerOpen] = useState(false); + const [investigationOpen, setInvestigationOpen] = useState(false); + const [trace, setTrace] = useState(null); + const selectStep = (index: number) => { + onStart(); + setStep(index); + setupRef.current?.querySelector(`[aria-controls="lens-setup-step-${index}"]`)?.focus(); + }; + const canLeave = state.tracesReady || state.requestsReady || state.hasInvestigations; + const start = () => { + onStart(); + setupRef.current?.scrollIntoView({ block: "start" }); + setupRef.current?.focus({ preventScroll: true }); + }; + const connectWorker = () => { + selectStep(2); + setWorkerOpen(true); + }; + const createInvestigation = () => { + selectStep(3); + setInvestigationOpen(true); + }; + + return ( +
    + +
    +
    +
    +
    +

    + Get Lens running +

    +

    + Each step checks your connection, so you’ll see when it’s working. +

    +
    + {canLeave && ( + + )} +
    + + {state.error && ( +
    +

    Could not check setup. {state.error}

    + +
    + )} + {(readOnly || !canInvestigate) && ( +

    + A gateway administrator can connect a worker and run investigations. +

    + )} +
    + +
    + {workerOpen && ( + setWorkerOpen(false)} + onChanged={state.refresh} + onReady={ + state.ready + ? () => { + setWorkerOpen(false); + createInvestigation(); + } + : undefined + } + /> + )} + {investigationOpen && ( + setInvestigationOpen(false)} + onCreated={onCreated} + /> + )} + {trace && ( + setTrace(null)} + /> + )} +
    + ); +} + +function FirstInvestigation({ + accessToken, + state, + onClose, + onCreated, +}: { + accessToken: string; + state: LensSetupState; + onClose: () => void; + onCreated: (id: string) => void; +}) { + const api = useLensApi(accessToken); + const saveLens = useSaveLens(accessToken); + const models = useQuery({ ...lensQueries.models(api) }); + const modelDetails = useQuery({ ...lensQueries.modelDetails(api) }); + const workers = state.workers.filter((worker) => !worker.revoked); + const analysisAccess = useAnalysisKeyInfo( + accessToken, + workers.length === 1 ? workers[0].analysis_key_id ?? undefined : undefined, + ); + const defaultModel = analysisAccess.data?.models.length === 1 ? analysisAccess.data.models[0] : undefined; + const save = async (settings: Settings) => { + if (!state.ready) + throw new Error("Wait for recorded activity and a connected worker before starting an investigation"); + const saved = await saveLens.mutateAsync({ settings }); + state.refresh(); + onCreated(saved.id); + }; + + return ( + model.id) ?? []} + modelDetails={modelDetails.data?.data ?? []} + modelsLoading={models.isLoading} + modelsError={models.error?.message} + onClose={onClose} + onSave={save} + /> + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.module.css b/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.module.css new file mode 100644 index 00000000000..b9f5fc11d88 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.module.css @@ -0,0 +1,114 @@ +.swarmRow { + animation: swarm-flow 5.2s linear infinite; +} + +.organizedDots { + animation: organized-flow 1.6s linear infinite; +} + +.gatewayGlow { + opacity: 0.7; + transition: opacity 300ms ease; +} + +.flowGraphic:hover .gatewayGlow { + opacity: 1; +} + +.sampleCard { + transition: + border-color 200ms ease, + box-shadow 200ms ease; +} + +.sampleCard:hover, +.sampleCard:focus-within { + border-color: color-mix(in oklab, var(--color-indigo-500) 22%, var(--border)); + box-shadow: 0 4px 20px -16px color-mix(in oklab, var(--color-indigo-500) 40%, transparent); +} + +.examples { + --evidence-tint: color-mix(in oklab, var(--color-rose-400) 6%, transparent); + --evidence-outline: color-mix(in oklab, var(--color-rose-400) 32%, transparent); +} + +.evidenceLink { + cursor: pointer; + border-radius: 4px; + outline-offset: 4px; + transition: background-color 200ms ease; +} + +.evidenceLink:focus-visible { + outline: 2px solid var(--ring); +} + +.evidenceLink:hover { + background: var(--evidence-tint); +} + +.evidenceOutput, +.evidenceFinding, +.evidenceBar { + transition: + background-color 200ms ease, + box-shadow 200ms ease; +} + +.evidenceFinding { + border-radius: 6px; +} + +.examples:is( + [data-evidence-active="true"], + :has([data-evidence-link]:hover), + :has([data-evidence-link]:focus-visible) + ) { + .evidenceOutput { + box-shadow: inset 0 0 0 1px var(--evidence-outline); + } + + .evidenceFinding { + background: var(--evidence-tint); + box-shadow: 0 0 0 7px var(--evidence-tint); + } + + .evidenceBar { + box-shadow: 0 0 10px 2px var(--evidence-outline); + } +} + +@keyframes swarm-flow { + from { + transform: translateX(0); + } + to { + transform: translateX(240px); + } +} + +@keyframes organized-flow { + from { + transform: translateX(0); + } + to { + transform: translateX(80px); + } +} + +@media (prefers-reduced-motion: reduce) { + .swarmRow, + .organizedDots { + animation: none; + opacity: 0.75; + } + + .gatewayGlow, + .sampleCard, + .evidenceLink, + .evidenceOutput, + .evidenceFinding, + .evidenceBar { + transition: none; + } +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx b/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx new file mode 100644 index 00000000000..c4820a95517 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensIntroduction.tsx @@ -0,0 +1,190 @@ +import { useState } from "react"; +import { ArrowRight, ArrowUpRight, ChevronDown, ChevronRight } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import styles from "./LensIntroduction.module.css"; +import { GatewayFlow } from "./GatewayFlow"; + +const traceSteps = [ + { name: "research_agent", start: 0, duration: 14.2, failed: false }, + { name: "plan", start: 0, duration: 1.7, failed: false }, + { name: "search_docs", start: 1.7, duration: 3.1, failed: false }, + { name: "run_benchmark", start: 4.8, duration: 1.3, failed: true }, + { name: "search_docs", start: 6.1, duration: 3.6, failed: false }, + { name: "answer", start: 9.7, duration: 4.5, failed: false }, +]; + +const sampleAnswer = "The column store is 40% faster than the row store for your workload."; + +type EvidenceProps = { highlighted: boolean; onHighlight: () => void }; + +function TraceExample({ highlighted, onHighlight }: EvidenceProps) { + return ( +
    +
    +

    + Tracing +

    +

    Sample trace · 14.2s

    +
    +
      + {traceSteps.map((item, index) => { + const linked = item.failed || item.name === "answer"; + const rowClass = + "grid w-full grid-cols-[minmax(0,1fr)_minmax(48px,1fr)_2.5rem] items-center gap-2 text-left sm:gap-3"; + const content = ( + <> + 0 ? "pl-2" : ""}`}> + {index === 0 ? ( + +
    +
    +
    +

    run_benchmark · output

    +

    Error: benchmark runner unavailable (503)

    +
    +
    +

    answer · output

    +

    “{sampleAnswer}”

    +
    +
    +
    + ); +} + +function FindingExamples({ highlighted, onHighlight }: EvidenceProps) { + return ( +
    +
    +

    + Lens findings +

    +

    500 sample runs reviewed

    +
    +
    +
    +

    + +

    +
    +

    Reports a speedup even though the benchmark failed.

    +
    + “…40% faster than the row store for your workload.” +
    +

    + Next step: Report the failed benchmark instead of + estimating. +

    +

    38 linked runs · High priority

    +
    +
    +
    +

    +

    +
    +

    After an order lookup times out, the agent promises to check and ends the run.

    +

    12 linked runs · Medium priority

    +
    +
    +
    +
    + ); +} + +export function LensIntroduction({ onStart, onDemo }: { onStart: () => void; onDemo?: () => void }) { + const [highlighted, setHighlighted] = useState(false); + const toggleEvidence = () => setHighlighted((current) => !current); + return ( +
    +

    + The gateway that helps your agents improve +

    +

    + Turn recorded agent runs into findings linked to the exact steps, so you know what happened and what to change. +

    +
    + {onDemo && ( + + )} + + + Docs +
    + +
    + + +
    +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx b/ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx new file mode 100644 index 00000000000..cdf95ef25ef --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx @@ -0,0 +1,229 @@ +import { ArrowRight, Check, ChevronDown } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { TracingSetupFields } from "@/components/view_logs/TraceView/TracingSetupCard"; +import type { TraceSummary } from "@/components/view_logs/TraceView/traceTypes"; +import type { LensSetupState } from "./useLensSetup"; +import { cn } from "@/lib/cva.config"; + +type StepProps = { + accessToken: string; + state: LensSetupState; + readOnly: boolean; + canInvestigate: boolean; + canMintTracingKey: boolean; + onStep: (step: number) => void; + onTrace: (trace: TraceSummary) => void; + onConnect: () => void; + onCreate: () => void; +}; + +export function initialSetupStep(state: LensSetupState) { + if (state.tracesReady || state.requestsReady) return state.connected ? 3 : 2; + return state.tracingEnabled ? 1 : 0; +} + +function StorageStep({ state, onStep, ...props }: StepProps) { + if (!state.tracingEnabled) + return ( + <> + + + + ); + return ( +
    +

    + Trace storage is connected +

    + + +
    + ); +} + +function continuationLabel(state: LensSetupState) { + if (state.connected) return "Continue to investigation"; + return state.tracesReady ? "Continue to worker" : "Continue with request logs"; +} + +function ActivityContinuation({ + state, + onConnect, + onCreate, + readOnly, + canInvestigate, +}: Pick) { + if (!state.tracesReady && !state.requestsReady) return null; + return ( +
    + + {state.requestsReady && !state.tracesReady && ( +

    + Request logs are already available. You can investigate them now and add agent traces later. +

    + )} +
    + ); +} + +function AgentStep({ state, ...props }: StepProps) { + if (!state.tracingEnabled) + return ( + <> +

    Connect trace storage in step 1 before sending a trace.

    + + + ); + return ( + <> + + {state.tracesReady && ( +

    + Your first trace is ready. Continue setup so Lens can investigate your agent’s behavior. +

    + )} + + + ); +} + +function WorkerStep({ state, onConnect, onCreate, readOnly, canInvestigate }: StepProps) { + const activityReady = state.tracesReady || state.requestsReady; + return ( +
    +

    + {state.connected + ? "Worker connected. You’re ready to create an investigation." + : "The worker reviews recorded activity using a model on your gateway. You choose its analysis model and spending limit."} +

    + {!activityReady && ( +

    Record activity in step 2 before connecting a worker.

    + )} + +
    + ); +} + +function InvestigationStep({ state, onCreate, readOnly, canInvestigate }: StepProps) { + return ( +
    +

    + Choose the activity to review and describe how your agent should behave. Lens will show findings with evidence + and suggested changes. +

    + {!state.ready && ( +

    + Recorded activity and a connected worker are required before you can run an investigation. +

    + )} + +
    + ); +} + +export function LensSetupSteps({ step, ...props }: StepProps & { step: number }) { + const items = [ + { + title: "Enable tracing on the gateway", + description: "Connect ClickHouse and restart the gateway.", + complete: props.state.tracingEnabled, + content: , + }, + { + title: "Send your first trace", + description: "Capture your agent’s inputs, outputs, and tool calls.", + complete: props.state.tracesReady, + content: , + }, + { + title: "Connect a worker", + description: "Choose a model and run the worker on your infrastructure.", + complete: props.state.connected, + content: , + }, + { + title: "Run your first investigation", + description: "Describe the expected behavior and review a sample of activity.", + complete: props.state.hasInvestigations, + content: , + }, + ]; + return ( +
      + {items.map((item, index) => ( +
    1. +

      + +

      + +
      2. + ))} +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts b/ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts new file mode 100644 index 00000000000..12903e8a15c --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts @@ -0,0 +1,59 @@ +import { useQuery } from "@tanstack/react-query"; +import { useNow } from "@/hooks/useNow"; +import { isTracingNotEnabled, useTraceAvailability } from "@/components/view_logs/TraceView/useAgentTraces"; +import { lensQueries } from "../api/queries"; +import { workerConnected } from "../model/status"; +import { useLensApi } from "../services"; + +function traceSetupState(traces: ReturnType, configured: boolean) { + const disabled = isTracingNotEnabled(traces.error); + return { + tracesReady: traces.data === true && !traces.error, + tracingEnabled: !disabled && (traces.isSuccess || configured), + missingTraces: disabled || traces.data === false, + error: disabled ? null : traces.error, + }; +} + +export function useLensSetup(accessToken: string, enabled: boolean, canInvestigate: boolean, settingUp: boolean) { + const api = useLensApi(accessToken); + const now = useNow(2000); + const traces = useTraceAvailability(accessToken, enabled); + const list = useQuery({ ...lensQueries.list(api, settingUp), enabled: enabled && canInvestigate }); + const activity = useQuery(lensQueries.activity(api, enabled && canInvestigate && list.isSuccess)); + const data = list.data ?? { lenses: [], workers: [], tracing_enabled: false }; + const traceState = traceSetupState(traces, data.tracing_enabled); + const hasRequests = activity.data?.requests === true; + const requestsReady = hasRequests && !activity.error; + const connected = data.workers.some((worker) => workerConnected(worker, now)); + const hasInvestigations = data.lenses.length > 0; + const activityReady = traceState.tracesReady || requestsReady; + const activityError = activityReady ? null : traceState.error || activity.error; + const error = list.error || activityError; + const loadingActivity = !activityReady && list.isSuccess && activity.isPending; + const loadingInvestigations = canInvestigate && (list.isPending || loadingActivity); + const refresh = () => { + void traces.refetch(); + if (canInvestigate) { + void list.refetch(); + void activity.refetch(); + } + }; + return { + tracingEnabled: traceState.tracingEnabled, + tracesReady: traceState.tracesReady, + requestsReady, + hasRequests, + connected, + hasInvestigations, + missingTraces: traceState.missingTraces, + loading: (!activityReady && traces.isPending) || loadingInvestigations, + checking: traces.isFetching || list.isFetching || activity.isFetching, + ready: activityReady && connected && !error, + workers: data.workers, + error: error?.message, + refresh, + }; +} + +export type LensSetupState = ReturnType; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx index e00870e71eb..26904ef49ff 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -636,16 +636,7 @@ function ConnectAgent({ ); } -export function TracingSetupCard({ - detail, - accessToken, - onOpenTrace, - connected = false, - onCheck, - checking = false, - readOnly = false, - canMintTracingKey = false, -}: { +type TracingSetupProps = { detail: string | null; accessToken: string; onOpenTrace: (trace: TraceSummary) => void; @@ -654,19 +645,47 @@ export function TracingSetupCard({ checking?: boolean; readOnly?: boolean; canMintTracingKey?: boolean; -}) { +}; + +export function TracingSetupFields({ + detail, + accessToken, + onOpenTrace, + connected = false, + onCheck, + checking = false, + readOnly = false, + canMintTracingKey = false, +}: TracingSetupProps) { const [checked, setChecked] = useState(false); - const enabled = detail === null; const check = () => { setChecked(true); onCheck?.(); }; + return detail === null ? ( + + ) : ( + + ); +} + +export function TracingSetupCard(props: TracingSetupProps) { + const enabled = props.detail === null; return (
    - {!connected && } + {!props.connected && }
    -

    {setupTitle(enabled, connected)}

    +

    {setupTitle(enabled, props.connected ?? false)}

    {enabled ? ( @@ -689,20 +708,7 @@ export function TracingSetupCard({ ? "Send your agent’s runs to LiteLLM to see its inputs, outputs, and tool calls." : "Tracing needs ClickHouse and a small update to your LiteLLM proxy configuration."}

    - {enabled ? ( - - ) : ( - - )} +
    ); } From cf22deb96a88e8435caa2123c03d525f6fc17341 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:55:01 -0700 Subject: [PATCH 4/6] test(ci): fix six CircleCI test regressions on main (#44429) * test(ci): fix four CircleCI regressions on main Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ci): stop reloading auth_checks in unit tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(constants): cover CLI JWT expiry env parsing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(gateway): restore proxy lifespan after importing gateway.main in launch tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/ui/fixtures/migratedPages.ts | 2 +- .../_support/bedrock_runtime_peer.py | 2 + ..._bedrock_runtime_chat_completions_chaos.py | 2 + .../test_mcp_servers.py | 20 ++++- tests/unit/gateway/test_launch.py | 21 +++++- ...st_auth_checks_object_access_and_lookup.py | 32 +------- .../test_user_api_key_auth_request_flow.py | 32 +++----- tests/unit/proxy/test_custom_proxy.py | 73 +++++++++---------- tests/unit/test_constants.py | 33 +++++++++ 9 files changed, 121 insertions(+), 96 deletions(-) diff --git a/tests/e2e/ui/fixtures/migratedPages.ts b/tests/e2e/ui/fixtures/migratedPages.ts index bce09b49e10..d44dfc625aa 100644 --- a/tests/e2e/ui/fixtures/migratedPages.ts +++ b/tests/e2e/ui/fixtures/migratedPages.ts @@ -99,7 +99,7 @@ export const MIGRATED_E2E_PAGES: Readonly> = { group: "Settings", content: { role: "heading", name: "UI Theme Customization" }, }, - logs: { segment: "logs", linkName: "Logs", content: { role: "heading", name: "Request Logs" } }, + logs: { segment: "logs", linkName: "Logs", content: { role: "tab", name: "Request Logs" } }, "admin-panel": { segment: "admin-panel", linkName: "Admin Settings", diff --git a/tests/integration/_support/bedrock_runtime_peer.py b/tests/integration/_support/bedrock_runtime_peer.py index 3a547260590..64115d9eaa0 100644 --- a/tests/integration/_support/bedrock_runtime_peer.py +++ b/tests/integration/_support/bedrock_runtime_peer.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import re import threading diff --git a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py index 5f59fa883ce..820d033135f 100644 --- a/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py +++ b/tests/integration/providers/test_bedrock_runtime_chat_completions_chaos.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import base64 import binascii diff --git a/tests/store_model_in_db_tests/test_mcp_servers.py b/tests/store_model_in_db_tests/test_mcp_servers.py index 94e14798c54..5c1a996b276 100644 --- a/tests/store_model_in_db_tests/test_mcp_servers.py +++ b/tests/store_model_in_db_tests/test_mcp_servers.py @@ -1,6 +1,6 @@ import sys from datetime import datetime -from typing import List, Optional +from typing import Final, List, Optional import pytest from litellm._uuid import uuid import os @@ -391,6 +391,7 @@ async def test_create_mcp_server_invalid_alias(): @_SKIP_NO_MCP @pytest.mark.asyncio async def test_edit_mcp_server_redacts_credentials(): + mock_get_server: Final = mock.AsyncMock() with ( mock.patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.MCP_AVAILABLE", @@ -399,6 +400,10 @@ async def test_edit_mcp_server_redacts_credentials(): mock.patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.get_prisma_client_or_throw" ) as mock_get_prisma, + mock.patch( + "litellm.proxy.management_endpoints.mcp_management_endpoints.get_mcp_server", + new=mock_get_server, + ), mock.patch( "litellm.proxy.management_endpoints.mcp_management_endpoints.update_mcp_server", new_callable=mock.AsyncMock, @@ -422,6 +427,18 @@ async def test_edit_mcp_server_redacts_credentials(): mock_manager.reload_servers_from_database = mock.AsyncMock() server_id = str(uuid.uuid4()) + stored_server: Final = LiteLLM_MCPServerTable( + server_id=server_id, + alias="Updated Server", + url="https://updated.example.com/mcp", + transport=MCPTransport.http, + created_at=datetime.now(), + updated_at=datetime.now(), + credentials={"auth_value": "secret"}, + teams=[], + ) + mock_get_server.return_value = stored_server + updated_server = LiteLLM_MCPServerTable( server_id=server_id, alias="Updated Server", @@ -458,6 +475,7 @@ async def test_edit_mcp_server_redacts_credentials(): mock_update.assert_awaited_once() mock_manager.update_server.assert_called_once_with(updated_server) mock_manager.reload_servers_from_database.assert_awaited_once() + mock_get_server.assert_awaited_once_with(mock_prisma, server_id) def test_validate_mcp_server_name_direct(): diff --git a/tests/unit/gateway/test_launch.py b/tests/unit/gateway/test_launch.py index a783ce6ac7e..f76aea030fe 100644 --- a/tests/unit/gateway/test_launch.py +++ b/tests/unit/gateway/test_launch.py @@ -1,3 +1,4 @@ +import importlib import os import socket import sys @@ -12,10 +13,10 @@ import pytest from uvicorn.importer import import_from_string from uvicorn.main import main as uvicorn_main -import gateway.main from gateway.launch import GATEWAY_APP, main, pool_database_url, uvicorn_argv from litellm.proxy.db.db_url_settings import DatabaseURLSettings from litellm.proxy.db.pgbouncer import PGBOUNCER_POOLED_ENV_VAR, PgBouncerError, PgBouncerSettings +from litellm.proxy.proxy_server import app as proxy_app DB_ENV: Final = { "DATABASE_HOST": "db.internal", @@ -107,8 +108,22 @@ class TestUvicornArgv: argv: Final = uvicorn_argv(("--timeout-keep-alive", "30"), {"KEEPALIVE_TIMEOUT": "75"}) assert _uvicorn_params(argv)["timeout_keep_alive"] == 30 - def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self): - assert import_from_string(cast(str, _uvicorn_params(uvicorn_argv((), {}))["app"])) is gateway.main.app + def test_the_app_uvicorn_is_told_to_serve_is_the_trimmed_gateway(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(proxy_app.router, "lifespan_context", proxy_app.router.lifespan_context) + for key in ( + "DATABASE_URL", + "DIRECT_URL", + "DATABASE_URL_READ_REPLICA", + "DATABASE_HOST", + "DATABASE_HOST_READ_REPLICA", + "DATABASE_PASSWORD", + "IAM_TOKEN_DB_AUTH", + "AZURE_POSTGRESQL_AUTH", + ): + monkeypatch.delenv(key, raising=False) + + served: Final = import_from_string(cast(str, _uvicorn_params(uvicorn_argv((), {}))["app"])) + assert served is importlib.import_module("gateway.main").app class TestPoolDatabaseUrl: diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 6c8b6571991..059cff0c385 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -109,25 +109,6 @@ def set_salt_key(monkeypatch): monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234") -@pytest.fixture(autouse=True) -def reset_constants_module(): - """Reset constants module to ensure clean state before each test""" - import importlib - - from litellm import constants - from litellm.proxy.auth import auth_checks - - # Reload modules before test - importlib.reload(constants) - importlib.reload(auth_checks) - - yield - - # Reload modules after test to clean up - importlib.reload(constants) - importlib.reload(auth_checks) - - @pytest.fixture def valid_sso_user_defined_values(): return LiteLLM_UserTable( @@ -875,19 +856,10 @@ def test_get_cli_jwt_auth_token_default_expiration(valid_sso_user_defined_values def test_get_cli_jwt_auth_token_custom_expiration(valid_sso_user_defined_values, monkeypatch): - """Test generating CLI JWT token with custom expiration via environment variable""" - import importlib - - from litellm import constants + """Test generating a CLI JWT token with custom expiration via the configured constant""" from litellm.proxy.auth import auth_checks - # Set custom expiration to 48 hours - monkeypatch.setenv("LITELLM_CLI_JWT_EXPIRATION_HOURS", "48") - - # Reload the constants module to pick up the new env var - importlib.reload(constants) - # Also reload auth_checks to pick up the new constant value - importlib.reload(auth_checks) + monkeypatch.setattr(auth_checks, "CLI_JWT_EXPIRATION_HOURS", 48) token = auth_checks.ExperimentalUIJWTToken.get_cli_jwt_auth_token(valid_sso_user_defined_values) diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index fc8bc289735..8b33202b483 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -7321,15 +7321,10 @@ async def test_expired_cli_session_token_is_rejected(monkeypatch): on the shared validation path, not only for DB-backed keys.""" monkeypatch.delenv("EXPERIMENTAL_UI_LOGIN", raising=False) monkeypatch.setenv("LITELLM_SALT_KEY", "sk-salt-cli-test") - monkeypatch.setenv("LITELLM_CLI_JWT_EXPIRATION_HOURS", "-1") - import importlib - - from litellm import constants from litellm.proxy.auth import auth_checks - importlib.reload(constants) - importlib.reload(auth_checks) + monkeypatch.setattr(auth_checks, "CLI_JWT_EXPIRATION_HOURS", -1) user_info = LiteLLM_UserTable( user_id="cli-admin", @@ -7346,22 +7341,17 @@ async def test_expired_cli_session_token_is_rejected(monkeypatch): mock_request.headers = {"authorization": f"Bearer {cli_token}"} mock_request.query_params = {} - try: - with ( - patch("litellm.proxy.proxy_server.master_key", "sk-master"), - patch("litellm.proxy.proxy_server.prisma_client", None), - ): - with pytest.raises(ProxyException) as exc_info: - await user_api_key_auth( - request=mock_request, - api_key=f"Bearer {cli_token}", - ) + with ( + patch("litellm.proxy.proxy_server.master_key", "sk-master"), + patch("litellm.proxy.proxy_server.prisma_client", None), + ): + with pytest.raises(ProxyException) as exc_info: + await user_api_key_auth( + request=mock_request, + api_key=f"Bearer {cli_token}", + ) - assert exc_info.value.type == ProxyErrorTypes.expired_key - finally: - monkeypatch.delenv("LITELLM_CLI_JWT_EXPIRATION_HOURS", raising=False) - importlib.reload(constants) - importlib.reload(auth_checks) + assert exc_info.value.type == ProxyErrorTypes.expired_key @pytest.mark.asyncio diff --git a/tests/unit/proxy/test_custom_proxy.py b/tests/unit/proxy/test_custom_proxy.py index b646a4e80e7..a08ceccd4f3 100644 --- a/tests/unit/proxy/test_custom_proxy.py +++ b/tests/unit/proxy/test_custom_proxy.py @@ -1,52 +1,45 @@ import os +from typing import Final import uvicorn from dotenv import load_dotenv -from fastapi import FastAPI, Request +from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse - -load_dotenv() - -# Set the SERVER_ROOT_PATH environment variable to match the custom mount path -os.environ["SERVER_ROOT_PATH"] = "/my-custom-path" - -from litellm.proxy.proxy_server import app as litellm_app -from litellm.proxy.proxy_server import proxy_startup_event - -# Create main FastAPI app -app = FastAPI(title="Custom LiteLLM Server", lifespan=proxy_startup_event) - -# Add CORS middleware -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -custom_path = "/my-custom-path" - -# Mount LiteLLM app at /litellm -app.mount(custom_path, litellm_app) -# Default route at / -@app.get("/") -async def root(): - return { - "message": "Welcome to the API Gateway", - "litellm_endpoint": f"{custom_path}", - } +def build_app() -> FastAPI: + load_dotenv() + os.environ["SERVER_ROOT_PATH"] = "/my-custom-path" + from litellm.proxy.proxy_server import app as litellm_app + from litellm.proxy.proxy_server import proxy_startup_event -# Health check endpoint -@app.get("/health") -async def health_check(): - return {"status": "healthy"} + app: Final = FastAPI(title="Custom LiteLLM Server", lifespan=proxy_startup_event) + custom_path: Final = "/my-custom-path" + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + app.mount(custom_path, litellm_app) + + @app.get("/") + async def root() -> dict[str, str]: + return { + "message": "Welcome to the API Gateway", + "litellm_endpoint": custom_path, + } + + @app.get("/health") + async def health_check() -> dict[str, str]: + return {"status": "healthy"} + + return app if __name__ == "__main__": - # Run the server on port 8000 - uvicorn.run(app, host="0.0.0.0", port=4000, log_level="info") + uvicorn.run(build_app(), host="0.0.0.0", port=4000, log_level="info") diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 12e473f68a4..d5981b906a3 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -68,3 +68,36 @@ def _build_constant_env_var_map() -> dict[str, str]: env_var_map[constant_name] = env_var_name return env_var_map + + +@pytest.mark.parametrize( + ("cli_value", "litellm_cli_value", "expected"), + [ + ("48", None, 48), + (None, "48", 48), + (None, None, 24), + ("48", "72", 48), + ], + ids=("canonical-only", "alias-only", "default", "canonical-wins"), +) +def test_cli_jwt_expiration_hours_from_environment( + monkeypatch: pytest.MonkeyPatch, + cli_value: str | None, + litellm_cli_value: str | None, + expected: int, +) -> None: + monkeypatch.delenv("CLI_JWT_EXPIRATION_HOURS", raising=False) + monkeypatch.delenv("LITELLM_CLI_JWT_EXPIRATION_HOURS", raising=False) + + try: + if cli_value is not None: + monkeypatch.setenv("CLI_JWT_EXPIRATION_HOURS", cli_value) + if litellm_cli_value is not None: + monkeypatch.setenv("LITELLM_CLI_JWT_EXPIRATION_HOURS", litellm_cli_value) + + importlib.reload(litellm.constants) + assert litellm.constants.CLI_JWT_EXPIRATION_HOURS == expected + finally: + monkeypatch.delenv("CLI_JWT_EXPIRATION_HOURS", raising=False) + monkeypatch.delenv("LITELLM_CLI_JWT_EXPIRATION_HOURS", raising=False) + importlib.reload(litellm.constants) From 984b4134be04fc407aabaf10363ce56d0701aa94 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:45:16 -0700 Subject: [PATCH 5/6] fix(proxy-extras): log v1 migration failures at ERROR so LITELLM_LOG=ERROR shows them (#44202) * fix(proxy-extras): log v1 migration failures at ERROR so LITELLM_LOG=ERROR shows them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy-extras): reuse litellm secret redaction and mask configured DB passwords exactly Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy-extras): name the password alternation in _redact_credentials Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(proxy-extras): wrap v1 migration ERROR lines at 120 columns Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy-extras): integration cells for v1 migration ERROR logging and password redaction Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy-extras): integration cells for component DB env vars, JSON logs, migration Job and v2 resolver Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy-extras): give every v1 migration integration cell the 900s timeout Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy-extras): bind the recovery forwarder before migrating so the retry cannot race it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy-extras): drop the slow P3005 integration cell and bound migration subprocesses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy-extras): keep command repr and tolerate non-sequence cmd in migration error logs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy-extras): double the subprocess boundary in the cmd=None retry test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: jesus Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng --- .../litellm_proxy_extras/utils.py | 81 ++- .../test_v1_migration_error_logging.py | 558 ++++++++++++++++++ .../test_litellm_proxy_extras_utils.py | 414 ++++++++++++- 3 files changed, 1048 insertions(+), 5 deletions(-) create mode 100644 tests/integration/database/test_v1_migration_error_logging.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 245244250ee..1fd292b8137 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -1,3 +1,4 @@ +import functools import glob import os import random @@ -10,7 +11,8 @@ import time from collections.abc import Callable from dataclasses import dataclass, replace from pathlib import Path -from typing import TYPE_CHECKING, Final, Optional +from typing import TYPE_CHECKING, Final, Optional, Union +from urllib.parse import unquote, urlsplit from litellm_proxy_extras import prisma_toolchain from litellm_proxy_extras._logging import logger @@ -202,6 +204,66 @@ def _max_migration_timestamp(names) -> int: return max(_migration_timestamp(n) for n in names) +_REDACTED: Final = "REDACTED" +_PASSWORD_QUERY_KEYS: Final = frozenset(("password", "sslpassword")) + + +@functools.cache +def _secret_shape_redactor() -> Callable[[str], str]: + try: + from litellm._logging import redact_secrets + except ImportError: + return lambda text: text + return redact_secrets + + +def _url_passwords(url: str) -> frozenset[str]: + try: + parts: Final = urlsplit(url) + except ValueError: + return frozenset() + query_pairs: Final = tuple(pair.partition("=") for pair in parts.query.split("&")) + raw_query_passwords: Final = tuple( + value for key, separator, value in query_pairs if separator and key.lower() in _PASSWORD_QUERY_KEYS + ) + raw_passwords: Final = ((parts.password,) if parts.password else ()) + raw_query_passwords + return frozenset(password for password in raw_passwords + tuple(map(unquote, raw_passwords)) if password) + + +def _configured_database_passwords() -> frozenset[str]: + database_url: Final = os.getenv("DATABASE_URL") + direct_url: Final = os.getenv("DIRECT_URL") + database_passwords: Final = _url_passwords(database_url) if database_url else frozenset() + direct_passwords: Final = _url_passwords(direct_url) if direct_url else frozenset() + return database_passwords | direct_passwords + + +def _redact_credentials(text: str) -> str: + """Mask configured database passwords before passing the text to LiteLLM redaction.""" + passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True) + alternation: Final = "|".join(re.escape(password) for password in passwords) + password_pattern: Final = ( + re.compile(rf"(?P:|password=)(?:{alternation})(?=@|&|$|[\s'\"\]),])", re.IGNORECASE) + if passwords + else None + ) + result: Final = password_pattern.sub(rf"\g{_REDACTED}", text) if password_pattern is not None else text + return _secret_shape_redactor()(result) + + +def _redacted_command(command: object) -> Union[str, tuple[str, ...], list[str]]: + if isinstance(command, tuple): + return tuple(_redact_credentials(str(argument)) for argument in command) + if isinstance(command, list): + return [_redact_credentials(str(argument)) for argument in command] + return _redact_credentials(str(command)) + + +def _redact_command_error(error: subprocess.CalledProcessError) -> str: + redacted_command: Final = _redacted_command(error.cmd) + return str(subprocess.CalledProcessError(error.returncode, redacted_command)) + + def _get_prisma_command() -> str: """Get the Prisma command to use, bypassing Python wrapper in offline mode.""" if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")): @@ -315,7 +377,8 @@ class ProxyExtrasDBManager: return False except subprocess.CalledProcessError as e: logger.warning( - f"Error creating baseline migration: {e}, {e.stderr}, {e.stdout}" + f"Error creating baseline migration: {_redact_command_error(e)}, " + f"{_redact_credentials(str(e.stderr))}, {_redact_credentials(str(e.stdout))}" ) raise e @@ -1572,6 +1635,11 @@ class ProxyExtrasDBManager: f"Error: {stderr}" ) raise + else: + logger.error( + "prisma migrate deploy failed with an error the resolver does not handle: " + f"{_redact_credentials(stderr)}" + ) else: if ProxyExtrasDBManager.spend_logs_is_partitioned(): raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR) @@ -1586,7 +1654,7 @@ class ProxyExtrasDBManager: ) return True except subprocess.TimeoutExpired: - logger.warning( + logger.error( "Attempt %s timed out. Raise %s if this database needs longer to apply its schema.", attempt + 1, PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR if use_migrate else PRISMA_COMMAND_TIMEOUT_ENV_VAR, @@ -1599,7 +1667,12 @@ class ProxyExtrasDBManager: if attempts_left > 0 else "" ) - logger.info(f"The process failed to execute. Details: {e}.{retry_msg}") + stderr_detail: Final = ( + f" stderr: {_redact_credentials(str(e.stderr))}" if e.stderr else "" + ) + logger.error( + f"The process failed to execute. Details: {_redact_command_error(e)}.{stderr_detail}{retry_msg}" + ) time.sleep(random.randrange(5, 15)) finally: os.chdir(original_dir) diff --git a/tests/integration/database/test_v1_migration_error_logging.py b/tests/integration/database/test_v1_migration_error_logging.py new file mode 100644 index 00000000000..c3a3d8c43d6 --- /dev/null +++ b/tests/integration/database/test_v1_migration_error_logging.py @@ -0,0 +1,558 @@ +from __future__ import annotations + +import json +import os +import re +import signal +import socket +import socketserver +import subprocess +import sys +import threading +import uuid +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager, suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, cast +from urllib.parse import quote, urlsplit, urlunsplit + +import psycopg +import pytest +from psycopg import sql +from psycopg.types.json import Jsonb + +from tests.integration._support.client import eventually +from tests.integration._support.process import _free_port as free_port + +REPO_ROOT: Final = Path(__file__).resolve().parents[3] +MIGRATIONS_DIR: Final = REPO_ROOT / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations" +MIGRATION_NAME: Final = "20260921190000_agent_identity" +pytestmark: Final = pytest.mark.timeout(300) +PASSWORD: Final = "wr ong'pw9" +FRAGMENTS: Final = ("wr ong", "wr+ong", "ong'pw9", "wr%20ong", "ong%27pw9", "pw9") +SHIPPED_MIGRATIONS: Final = tuple( + sorted(path.name for path in MIGRATIONS_DIR.iterdir() if path.is_dir() and path.name != "0_init") +) +LOG_PREFIX: Final = r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - [^\n]+ - " +LOG_RECORD_START: Final = rf"{LOG_PREFIX}(?:DEBUG|INFO|WARNING|ERROR|CRITICAL) - " +ERROR_RECORD: Final = re.compile(rf"(?ms)^({LOG_PREFIX}ERROR - .*?)(?={LOG_RECORD_START}|\Z)") +RETRY_COUNT: Final = re.compile(r"Retrying\.\.\. \((\d+) attempts left\)") +FRAGMENT_PATTERN: Final = re.compile( + "|".join(re.escape(fragment) for fragment in sorted(FRAGMENTS, key=len, reverse=True)) +) + + +@dataclass(frozen=True, slots=True) +class MigrationResult: + returncode: int + output: str + + +def _migration_log_path(test_name: str, tmp_path: Path) -> Path: + log_directory: Final = ( + Path(os.environ["INTEGRATION_RESULTS_DIR"]) if "INTEGRATION_RESULTS_DIR" in os.environ else tmp_path + ) + log_path: Final = log_directory / f"v1-migration-{test_name}-{uuid.uuid4().hex[:8]}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + return log_path + + +def _database_url( + admin_url: str, + role: str, + password: str, + database: str, + *, + encode_password: bool = True, +) -> str: + parsed: Final = urlsplit(admin_url) + authority: Final = parsed.netloc.rsplit("@", 1)[-1] + encoded_password: Final = quote(password, safe="") if encode_password else password + netloc: Final = f"{quote(role, safe='')}:{encoded_password}@{authority}" + return urlunsplit(parsed._replace(netloc=netloc, path=f"/{database}")) + + +def _replace_port(database_url: str, port: int, hostname: str | None = None) -> str: + parsed: Final = urlsplit(database_url) + target_host: Final = hostname or parsed.hostname + assert target_host is not None + host: Final = f"[{target_host}]" if ":" in target_host else target_host + userinfo: Final = parsed.netloc.rsplit("@", 1)[0] + return urlunsplit(parsed._replace(netloc=f"{userinfo}@{host}:{port}")) + + +def _unreachable_database_url(database_url: str) -> str: + parsed: Final = urlsplit(database_url) + url: Final = _database_url( + database_url, + parsed.username or "", + PASSWORD, + parsed.path.lstrip("/"), + encode_password=False, + ) + return _replace_port(url, free_port()) + + +@contextmanager +def owned_database(password: str) -> Iterator[str]: + admin_url: Final = os.environ["DATABASE_URL"] + role: Final = f"v1_migration_{uuid.uuid4().hex}" + database: Final = f"v1_migration_{uuid.uuid4().hex}" + database_url: Final = _database_url(admin_url, role, password, database) + try: + with psycopg.connect(admin_url, autocommit=True) as admin: + admin.execute( + sql.SQL("CREATE ROLE {} WITH LOGIN PASSWORD {}").format(sql.Identifier(role), sql.Literal(password)) + ) + admin.execute(sql.SQL("CREATE DATABASE {} OWNER {}").format(sql.Identifier(database), sql.Identifier(role))) + yield database_url + finally: + with psycopg.connect(admin_url, autocommit=True) as admin: + admin.execute(sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(sql.Identifier(database))) + admin.execute(sql.SQL("DROP ROLE IF EXISTS {}").format(sql.Identifier(role))) + + +def _migration_environment(database_url: str | None, extra_env: Mapping[str, str]) -> dict[str, str]: + excluded_variables: Final = ( + ("DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + if database_url is not None + else ("DATABASE_URL", "DIRECT_URL", "USE_V2_MIGRATION_RESOLVER") + ) + inherited_environment: Final = {key: value for key, value in os.environ.items() if key not in excluded_variables} + database_environment: Final = {"DATABASE_URL": database_url} if database_url is not None else {} + return { + **inherited_environment, + **database_environment, + "LITELLM_LOG": "ERROR", + **extra_env, + } + + +def _migration_invocation( + database_url: str | None, + tmp_path: Path, + extra_env: Mapping[str, str], + resolver: Literal["legacy", "v2"] = "legacy", +) -> tuple[tuple[str, ...], dict[str, str]]: + config_path: Final = tmp_path / "config.yaml" + config_path.write_text( + "model_list:\n - model_name: integration-fake\n litellm_params:\n model: openai/integration-fake\n" + ) + resolver_flag: Final = "--use_legacy_migration_resolver" if resolver == "legacy" else "--use_v2_migration_resolver" + command: Final = ( + sys.executable, + "-I", + "-m", + "litellm.proxy.proxy_cli", + "--config", + str(config_path), + resolver_flag, + "--skip_server_startup", + ) + environment: Final = _migration_environment(database_url, extra_env) + return command, environment + + +def run_v1_migrations( + database_url: str | None, + tmp_path: Path, + extra_env: Mapping[str, str], + test_name: str, + resolver: Literal["legacy", "v2"] = "legacy", +) -> MigrationResult: + command, environment = _migration_invocation(database_url, tmp_path, extra_env, resolver) + output_path: Final = _migration_log_path(test_name, tmp_path) + with output_path.open("w") as output_file: + completed: Final = subprocess.run( + command, + cwd=REPO_ROOT, + env=environment, + stdout=output_file, + stderr=subprocess.STDOUT, + text=True, + timeout=240, + ) + output: Final = output_path.read_text() + return MigrationResult(completed.returncode, output) + + +def error_lines(output: str) -> tuple[str, ...]: + return tuple(match.group(1) for match in ERROR_RECORD.finditer(output)) + + +def _json_error_records(output: str) -> tuple[dict[str, object], ...]: + records: Final = tuple(_parse_json_record(line, output) for line in output.splitlines() if line.startswith("{")) + return tuple(record for record in records if record.get("level") == "ERROR") + + +def _parse_json_record(line: str, output: str) -> dict[str, object]: + try: + record: Final = json.loads(line) + except json.JSONDecodeError: + pytest.fail(_safe_output(output)) + assert isinstance(record, dict), _safe_output(output) + return cast(dict[str, object], record) + + +def _retry_count_texts(lines: tuple[str, ...]) -> tuple[str, ...]: + return tuple(value for value in (_retry_count_text(line) for line in lines) if value is not None) + + +def _retry_count_text(line: str) -> str | None: + match: Final = RETRY_COUNT.search(line) + return match.group(1) if match is not None else None + + +def _safe_output(output: str) -> str: + return FRAGMENT_PATTERN.sub("[REDACTED]", output) + + +def _assert_no_password_fragments(output: str) -> None: + assert [fragment for fragment in FRAGMENTS if fragment in output] == [], _safe_output(output) + + +def _applied_migrations(database_url: str) -> tuple[str, ...]: + with psycopg.connect(database_url) as connection: + rows: Final = connection.execute( + 'SELECT migration_name FROM "_prisma_migrations" ' + "WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL ORDER BY migration_name" + ).fetchall() + return tuple(str(row[0]) for row in rows) + + +def _duplicate_migrations(database_url: str) -> tuple[tuple[str, int], ...]: + with psycopg.connect(database_url) as connection: + rows: Final = connection.execute( + 'SELECT migration_name, COUNT(*) FROM "_prisma_migrations" ' + "GROUP BY migration_name HAVING COUNT(*) > 1 ORDER BY migration_name" + ).fetchall() + return tuple((str(row[0]), int(row[1])) for row in rows) + + +def _retry_p3018_migration(database_url: str) -> None: + agent_ids: Final = (uuid.uuid4().hex, uuid.uuid4().hex) + with psycopg.connect(database_url) as connection: + connection.execute('DROP INDEX "LiteLLM_AgentIdentity_provider_tenant_id_client_id_key"') + for agent_id in agent_ids: + connection.execute( + 'INSERT INTO "LiteLLM_AgentsTable" ' + '("agent_id", "agent_name", "agent_card_params", "created_by", "updated_by") ' + "VALUES (%s, %s, %s, %s, %s)", + (agent_id, f"audit-agent-{agent_id}", Jsonb({}), "integration", "integration"), + ) + connection.execute( + 'INSERT INTO "LiteLLM_AgentIdentity" ' + '("agent_id", "provider", "issuer", "tenant_id", "client_id", "revision") ' + "VALUES (%s, %s, %s, %s, %s, %s)", + (agent_id, "entra", f"https://audit.invalid/{agent_id}", "tenant-1", "client-1", uuid.uuid4().hex), + ) + connection.execute('DELETE FROM "_prisma_migrations" WHERE migration_name = %s', (MIGRATION_NAME,)) + + +def _relay(source: socket.socket, destination: socket.socket) -> None: + try: + while data := source.recv(65536): + destination.sendall(data) + except (BrokenPipeError, ConnectionAbortedError, ConnectionResetError): + return + + +class _PostgresForwardingServer(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + request_queue_size = 64 + target: tuple[str, int] + + def __init__(self, port: int, target: tuple[str, int]) -> None: + self.target = target + super().__init__(("127.0.0.1", port), _PostgresForwardingHandler) + + +class _PostgresForwardingHandler(socketserver.BaseRequestHandler): + def handle(self) -> None: + server: Final = cast(_PostgresForwardingServer, self.server) + with socket.create_connection(server.target, timeout=10) as upstream: + reply: Final = threading.Thread(target=_relay, args=(upstream, self.request), daemon=True) + reply.start() + try: + _relay(self.request, upstream) + finally: + with suppress(OSError): + self.request.shutdown(socket.SHUT_WR) + with suppress(OSError): + upstream.shutdown(socket.SHUT_WR) + reply.join(timeout=10) + + +@contextmanager +def _gated_postgres_forwarder(port: int, target: tuple[str, int]) -> Iterator[Callable[[], None]]: + server: Final = _PostgresForwardingServer(port, target) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + try: + yield thread.start + finally: + if thread.is_alive(): + server.shutdown() + server.server_close() + if thread.ident is not None: + thread.join(timeout=10) + + +def _stop_process(process: subprocess.Popen[str]) -> None: + if process.poll() is None: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=30) + + +def _migration_log_has_p1001_or_process_exited(output_path: Path, process: subprocess.Popen[str]) -> tuple[bool, bool]: + p1001_logged: Final = any("P1001" in line for line in error_lines(output_path.read_text())) + process_exited: Final = process.poll() is not None + return p1001_logged, process_exited + + +def test_unreachable_database_emits_four_p1001_errors_without_password_fragments( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + unreachable_url: Final = _unreachable_database_url(database_url) + completed: Final = run_v1_migrations(unreachable_url, tmp_path, {}, request.node.name) + output: Final = completed.output + errors: Final = error_lines(output) + p1001_errors: Final = tuple(line for line in errors if "P1001" in line) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 4, _safe_output(output) + assert len(p1001_errors) == 4, _safe_output(output) + assert _retry_count_texts(errors) == (), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_wrong_password_emits_four_p1000_errors_without_password_fragments( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(f"correct-{uuid.uuid4().hex}") as database_url: + parsed: Final = urlsplit(database_url) + wrong_url: Final = _database_url( + database_url, + parsed.username or "", + PASSWORD, + parsed.path.lstrip("/"), + ) + completed: Final = run_v1_migrations(wrong_url, tmp_path, {}, request.node.name) + output: Final = completed.output + errors: Final = error_lines(output) + p1000_errors: Final = tuple(line for line in errors if "P1000" in line) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 4, _safe_output(output) + assert len(p1000_errors) == 4, _safe_output(output) + assert _retry_count_texts(errors) == (), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_duplicate_agent_identity_logs_the_p3018_migration_error( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + setup: Final = run_v1_migrations(database_url, tmp_path, {}, request.node.name) + setup_output: Final = setup.output + assert setup.returncode == 0, _safe_output(setup_output) + _retry_p3018_migration(database_url) + completed: Final = run_v1_migrations(database_url, tmp_path, {}, request.node.name) + output: Final = completed.output + errors: Final = error_lines(output) + p3018_errors: Final = tuple(line for line in errors if "P3018" in line) + expected_markers: Final = ((True, True), (True, True)) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 2, _safe_output(output) + assert tuple((MIGRATION_NAME in line, "P3018" in line) for line in p3018_errors) == expected_markers, ( + _safe_output(output) + ) + assert _retry_count_texts(errors) == ("3", "1"), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_clean_database_applies_exactly_the_shipped_migrations(tmp_path: Path, request: pytest.FixtureRequest) -> None: + with owned_database(PASSWORD) as database_url: + completed: Final = run_v1_migrations(database_url, tmp_path, {}, request.node.name) + output: Final = completed.output + assert completed.returncode == 0, _safe_output(output) + assert error_lines(output) == (), _safe_output(output) + assert _applied_migrations(database_url) == SHIPPED_MIGRATIONS, _safe_output(output) + _assert_no_password_fragments(output) + + +def test_unreachable_database_recovers_after_postgres_forwarder_starts( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + admin_url: Final = os.environ["DATABASE_URL"] + admin: Final = urlsplit(admin_url) + hostname: Final = admin.hostname + port: Final = admin.port + assert hostname is not None and port is not None + target_host: Final = "127.0.0.1" if hostname == "localhost" else hostname + forwarding_port: Final = free_port() + forwarded_url: Final = _replace_port(database_url, forwarding_port, "127.0.0.1") + command, environment = _migration_invocation(forwarded_url, tmp_path, {}) + output_path: Final = _migration_log_path(request.node.name, tmp_path) + with _gated_postgres_forwarder(forwarding_port, (target_host, port)) as open_forwarder: + with output_path.open("w") as output_file: + process: Final = subprocess.Popen( + command, + cwd=REPO_ROOT, + env=environment, + stdout=output_file, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + try: + observation: Final = eventually( + lambda: _migration_log_has_p1001_or_process_exited(output_path, process), + lambda state: state[0] or state[1], + seconds=240, + ) + assert observation[0], _safe_output(output_path.read_text()) + open_forwarder() + completed_returncode: Final = process.wait(timeout=240) + finally: + _stop_process(process) + output: Final = output_path.read_text() + errors: Final = error_lines(output) + p1001_errors: Final = tuple(line for line in errors if "P1001" in line) + assert completed_returncode == 0, _safe_output(output) + assert len(p1001_errors) == 1, _safe_output(output) + assert _retry_count_texts(errors) == (), _safe_output(output) + assert _duplicate_migrations(database_url) == (), _safe_output(output) + assert _applied_migrations(database_url) == SHIPPED_MIGRATIONS, _safe_output(output) + _assert_no_password_fragments(output) + + +def test_unreachable_database_keeps_password_masked_when_shape_redaction_is_disabled( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + unreachable_url: Final = _unreachable_database_url(database_url) + completed: Final = run_v1_migrations( + unreachable_url, + tmp_path, + {"LITELLM_DISABLE_REDACT_SECRETS": "true"}, + request.node.name, + ) + output: Final = completed.output + errors: Final = error_lines(output) + p1001_errors: Final = tuple(line for line in errors if "P1001" in line) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 4, _safe_output(output) + assert len(p1001_errors) == 4, _safe_output(output) + assert _retry_count_texts(errors) == (), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_component_database_env_vars_with_wrong_password_emit_four_p1000_errors_without_password_fragments( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + correct_password: Final = f"correct-{uuid.uuid4().hex}" + with owned_database(correct_password) as database_url: + parsed: Final = urlsplit(database_url) + host: Final = parsed.hostname + port: Final = parsed.port + username: Final = parsed.username + assert host is not None and port is not None and username is not None + extra_env: Final = { + "DATABASE_HOST": f"{host}:{port}", + "DATABASE_USERNAME": username, + "DATABASE_PASSWORD": PASSWORD, + "DATABASE_NAME": parsed.path.lstrip("/"), + } + completed: Final = run_v1_migrations(None, tmp_path, extra_env, request.node.name) + output: Final = completed.output + errors: Final = error_lines(output) + p1000_errors: Final = tuple(line for line in errors if "P1000" in line) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 4, _safe_output(output) + assert len(p1000_errors) == 4, _safe_output(output) + assert _retry_count_texts(errors) == (), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_json_logs_emit_four_valid_json_p1001_error_records_without_password_fragments( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + unreachable_url: Final = _unreachable_database_url(database_url) + completed: Final = run_v1_migrations(unreachable_url, tmp_path, {"JSON_LOGS": "true"}, request.node.name) + output: Final = completed.output + errors: Final = _json_error_records(output) + messages: Final = tuple(record.get("message") for record in errors) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 4, _safe_output(output) + assert tuple(isinstance(message, str) and "P1001" in message for message in messages) == ( + True, + True, + True, + True, + ), _safe_output(output) + assert error_lines(output) == (), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_migration_job_entrypoint_emits_four_p1001_errors_without_password_fragments( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + unreachable_url: Final = _unreachable_database_url(database_url) + command: Final = (sys.executable, "-I", "-m", "litellm.proxy.prisma_migration") + environment: Final = _migration_environment( + unreachable_url, + {"USE_V2_MIGRATION_RESOLVER": "false"}, + ) + output_path: Final = _migration_log_path(request.node.name, tmp_path) + with output_path.open("w") as output_file: + completed: Final = subprocess.run( + command, + cwd=REPO_ROOT, + env=environment, + stdout=output_file, + stderr=subprocess.STDOUT, + text=True, + timeout=240, + ) + output: Final = output_path.read_text() + errors: Final = error_lines(output) + p1001_errors: Final = tuple(line for line in errors if "P1001" in line) + assert completed.returncode == 1, _safe_output(output) + assert len(errors) == 4, _safe_output(output) + assert len(p1001_errors) == 4, _safe_output(output) + _assert_no_password_fragments(output) + + +def test_v2_resolver_unreachable_database_exits_2_and_names_p1001( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + unreachable_url: Final = _unreachable_database_url(database_url) + completed: Final = run_v1_migrations(unreachable_url, tmp_path, {}, request.node.name, resolver="v2") + output: Final = completed.output + assert completed.returncode == 2, _safe_output(output) + assert "P1001" in output, _safe_output(output) + assert error_lines(output) == (), _safe_output(output) + _assert_no_password_fragments(output) + + +def test_v2_resolver_clean_database_applies_exactly_the_shipped_migrations( + tmp_path: Path, request: pytest.FixtureRequest +) -> None: + with owned_database(PASSWORD) as database_url: + completed: Final = run_v1_migrations(database_url, tmp_path, {}, request.node.name, resolver="v2") + output: Final = completed.output + assert completed.returncode == 0, _safe_output(output) + assert error_lines(output) == (), _safe_output(output) + assert _applied_migrations(database_url) == SHIPPED_MIGRATIONS, _safe_output(output) + _assert_no_password_fragments(output) diff --git a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py index ea4a25283a1..70c6ec44790 100644 --- a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py +++ b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py @@ -1,11 +1,13 @@ import glob +import logging import os import re +import subprocess import sys import threading from dataclasses import dataclass from pathlib import Path -from typing import Final +from typing import Final, NoReturn, Optional import pytest @@ -19,6 +21,8 @@ sys.path.insert( from litellm_proxy_extras.utils import ( PARTITIONED_SPEND_LOGS_PUSH_ERROR, ProxyExtrasDBManager, + _redact_command_error, + _redact_credentials, filter_partitioned_spend_logs_diff, ) @@ -1412,3 +1416,411 @@ class TestMigrationJobOwnedDrift: assert 'PRIMARY KEY ("request_id")' not in filtered assert "LiteLLM_SpendLogs_legacy" not in filtered assert 'ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN "updated_by" TEXT;' in filtered + + +_P3018_UNCLASSIFIED_STDERR: Final = ( + "Error: P3018\n\n" + "A migration failed to apply. New migrations cannot be applied before the error is " + "recovered from.\n\n" + "Migration name: 20260921190000_agent_identity\n\n" + "Database error code: 23505\n\n" + "Database error:\n" + 'ERROR: could not create unique index "agent_identity_key"\n' + "DETAIL: Key (agent_id)=(agent-1) is duplicated.\n" +) + + +_FAKE_PRISMA_PID: Final = 424242 + + +class TestV1MigrationFailuresLogAtError: + @staticmethod + def _run_v1_migrations( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + *, + deploy_stderr: Optional[str] = None, + deploy_timeout: bool = False, + diff_stderr: Optional[str] = None, + database_url: Optional[str] = None, + ) -> tuple[bool, list[list[str]], list[int]]: + import litellm_proxy_extras.utils as utils_module + + calls: Final[list[list[str]]] = [] + killed_pids: Final[list[int]] = [] + + class _FakePrismaPopen: + def __init__( + self, + argv: tuple[str, ...], + *, + env: Optional[dict[str, str]] = None, + stdout: object = None, + stderr: object = None, + text: object = None, + start_new_session: object = None, + ) -> None: + self.args: Final = argv + self.argv: Final = argv + self.pid: Final = _FAKE_PRISMA_PID + self.returncode: Optional[int] = None + calls.append(list(argv)) + + def __enter__(self) -> "_FakePrismaPopen": + return self + + def __exit__(self, *args: object) -> None: + return None + + def _subcommand(self) -> tuple[str, str]: + known: Final = ( + ("migrate", "deploy"), + ("migrate", "diff"), + ("migrate", "resolve"), + ("db", "execute"), + ) + for index in range(len(self.argv) - 1): + pair: Final = tuple(self.argv[index : index + 2]) + if pair in known: + return pair + return ("", "") + + def communicate(self, timeout: Optional[float] = None) -> tuple[str, str]: + subcommand: Final = self._subcommand() + if subcommand == ("migrate", "deploy"): + if deploy_timeout: + raise subprocess.TimeoutExpired(self.argv, timeout) + if deploy_stderr is not None: + self.returncode = 1 + return "", deploy_stderr + self.returncode = 0 + return "No pending migrations to apply", "" + if subcommand == ("migrate", "diff") and diff_stderr is not None: + self.returncode = 1 + return "", diff_stderr + self.returncode = 0 + return "", "" + + migration_dir: Final = tmp_path / "migration_dir" + migration_dir.mkdir() + if database_url is None: + monkeypatch.delenv("DATABASE_URL", raising=False) + else: + monkeypatch.setenv("DATABASE_URL", database_url) + monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(migration_dir)) + monkeypatch.setattr( + utils_module.prisma_toolchain.subprocess, "Popen", _FakePrismaPopen + ) + monkeypatch.setattr( + utils_module.prisma_toolchain.os, "killpg", lambda pid, sig: killed_pids.append(pid) + ) + monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None) + + succeeded: Final = ProxyExtrasDBManager._run_migrations(use_migrate=True, use_v2_resolver=False) + return succeeded, calls, killed_pids + + @staticmethod + def _deploy_call_count(calls: list[list[str]]) -> int: + return sum(1 for call in calls if tuple(call[-2:]) == ("migrate", "deploy")) + + @staticmethod + def _error_messages(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelno >= logging.ERROR and record.name.startswith("litellm_proxy_extras") + ] + + def test_an_unrecognized_prisma_error_logs_its_stderr_at_error( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + stderr: Final = "Error: P1001: Can't reach database server at db:5432" + with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"): + succeeded, calls, _ = self._run_v1_migrations( + monkeypatch, tmp_path, deploy_stderr=stderr + ) + + assert succeeded is False + assert self._deploy_call_count(calls) == 4 + assert any(stderr in message for message in self._error_messages(caplog)) + + def test_an_unclassified_p3018_logs_its_stderr_and_retry_failure_at_error( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"): + succeeded, calls, _ = self._run_v1_migrations( + monkeypatch, tmp_path, deploy_stderr=_P3018_UNCLASSIFIED_STDERR + ) + + assert succeeded is False + assert self._deploy_call_count(calls) == 4 + messages: Final = self._error_messages(caplog) + assert any( + "20260921190000_agent_identity" in message and "is duplicated" in message + for message in messages + ) + assert any( + "The process failed to execute" in message and "Retrying... (3 attempts left)" in message + for message in messages + ) + + def test_called_process_error_with_no_command_retries_all_v1_attempts( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + import litellm_proxy_extras.utils as utils_module + + migration_dir: Final = tmp_path / "migration_dir" + migration_dir.mkdir() + monkeypatch.setenv("LITELLM_MIGRATION_DIR", str(migration_dir)) + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.delenv("DIRECT_URL", raising=False) + monkeypatch.setenv("PRISMA_OFFLINE_MODE", "true") + monkeypatch.setenv("PRISMA_CLI_PATH", sys.executable) + monkeypatch.setattr(utils_module.time, "sleep", lambda seconds: None) + calls: Final[list[None]] = [] + + class _FakePrismaPopen: + def __init__( + self, + argv: tuple[str, ...], + *, + env: Optional[dict[str, str]] = None, + stdout: object = None, + stderr: object = None, + text: object = None, + start_new_session: object = None, + ) -> None: + self.args: Final = None + self.returncode: Final = 1 + calls.append(None) + + def __enter__(self) -> "_FakePrismaPopen": + return self + + def __exit__(self, *args: object) -> None: + return None + + def communicate(self, timeout: Optional[float] = None) -> tuple[str, str]: + return "", "Error: P3018 unclassified" + + monkeypatch.setattr( + utils_module.prisma_toolchain.subprocess, "Popen", _FakePrismaPopen + ) + + try: + succeeded: Final = ProxyExtrasDBManager._run_migrations( + use_migrate=True, use_v2_resolver=False + ) + except TypeError as error: + pytest.fail( + f"_run_migrations raised TypeError after {len(calls)} Popen calls: {error}", + pytrace=False, + ) + + assert succeeded is False + assert len(calls) == 4 + + def test_a_timeout_logs_at_error_naming_the_migrate_deploy_timeout_env_var( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + from litellm_proxy_extras.prisma_toolchain import PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR + + with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"): + succeeded, calls, killed_pids = self._run_v1_migrations( + monkeypatch, tmp_path, deploy_timeout=True + ) + + assert succeeded is False + assert self._deploy_call_count(calls) == 4 + assert killed_pids == [_FAKE_PRISMA_PID] * 4 + assert any( + "timed out" in message and PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR in message + for message in self._error_messages(caplog) + ) + + def test_a_recovered_baseline_logs_nothing_at_error( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.ERROR, logger="litellm_proxy_extras"): + succeeded, calls, _ = self._run_v1_migrations( + monkeypatch, + tmp_path, + deploy_stderr=_P3005_STDERR, + database_url="postgresql://user:pass@db:5432/litellm", + ) + + assert succeeded is True + assert tuple(calls[0][-2:]) == ("migrate", "deploy") + assert self._error_messages(caplog) == [] + + def test_a_failed_baseline_recovery_logs_its_stderr_at_error( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + database_url: Final = "postgresql://llmproxy:s3cr3t 'p\"w@db:5432/litellm" + monkeypatch.delenv("DIRECT_URL", raising=False) + with caplog.at_level(logging.DEBUG, logger="litellm_proxy_extras"): + succeeded, calls, _ = self._run_v1_migrations( + monkeypatch, + tmp_path, + deploy_stderr=_P3005_STDERR, + diff_stderr=f"baseline diff failed: XYZ-7731 for {database_url}", + database_url=database_url, + ) + + assert succeeded is False + assert self._deploy_call_count(calls) == 4 + assert [ + record.getMessage() + for record in caplog.records + if "s3cr3t" in record.getMessage() or 'p"w' in record.getMessage() + ] == [] + messages: Final = self._error_messages(caplog) + assert any("postgresql://REDACTED@db:5432/litellm" in message for message in messages) + assert any("XYZ-7731" in message for message in messages) + + +@pytest.mark.parametrize( + "database_url,direct_url,text,expected", + ( + ( + "postgresql://u:pa ss@db:5432/litellm", + None, + 'Error: P1000: Authentication failed against database server at "postgresql://u:pa ss@db:5432/litellm"', + 'Error: P1000: Authentication failed against database server at "postgresql://REDACTED@db:5432/litellm"', + ), + ( + "postgresql://u:pa'ss@db:5432/litellm", + None, + "postgresql://u:pa'ss@db:5432/litellm", + "postgresql://REDACTED@db:5432/litellm", + ), + ( + 'postgresql://u:pa"ss@db:5432/litellm', + None, + 'postgresql://u:pa"ss@db:5432/litellm', + "postgresql://REDACTED@db:5432/litellm", + ), + ( + "postgresql://u:p@ss@db:5432/litellm", + None, + "postgresql://u:p@ss@db:5432/litellm", + "postgresql://REDACTED@db:5432/litellm", + ), + ( + "postgresql://u:p%20ss@db:5432/litellm", + None, + "postgresql://u:p ss@db:5432/litellm", + "postgresql://REDACTED@db:5432/litellm", + ), + ( + "postgresql://db/litellm?password=a b&sslmode=require", + None, + "postgresql://db/litellm?password=a b&sslmode=require", + "postgresql://db/litellm?REDACTED&sslmode=require", + ), + ( + "postgresql://db/litellm?sslpassword=zq'7x", + None, + "postgresql://db/litellm?sslpassword=zq'7x", + "postgresql://db/litellm?REDACTED", + ), + ( + None, + "postgresql://u:pa ss@db:5432/litellm", + "postgresql://u:pa ss@db:5432/litellm", + "postgresql://REDACTED@db:5432/litellm", + ), + ( + None, + None, + "postgresql://u:pw@db/x", + "postgresql://REDACTED@db/x", + ), + ( + "postgresql://u:p@db:5432/litellm", + None, + "Error: P1001: Can't reach database server at db:5432", + "Error: P1001: Can't reach database server at db:5432", + ), + ( + "postgresql://u:p@db:5432/litellm", + None, + "Error:P1001: Can't reach database server at db:5432", + "Error:P1001: Can't reach database server at db:5432", + ), + ( + None, + None, + "plain text with no URL", + "plain text with no URL", + ), + ), +) +def test_redact_credentials_masks_passwords_in_embedded_urls( + database_url: str | None, + direct_url: str | None, + text: str, + expected: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + if database_url is None: + monkeypatch.delenv("DATABASE_URL", raising=False) + else: + monkeypatch.setenv("DATABASE_URL", database_url) + if direct_url is None: + monkeypatch.delenv("DIRECT_URL", raising=False) + else: + monkeypatch.setenv("DIRECT_URL", direct_url) + assert _redact_credentials(text) == expected + + +@pytest.mark.parametrize("password", ("zq'7x", 'zq"7x', "zq'\"7x", "zq 7x", "zq@7x")) +def test_redact_command_error_masks_url_arguments(password: str, monkeypatch: pytest.MonkeyPatch) -> None: + database_url: Final = f"postgresql://u:{password}@db:5432/litellm" + monkeypatch.setenv("DATABASE_URL", database_url) + monkeypatch.delenv("DIRECT_URL", raising=False) + error: Final = subprocess.CalledProcessError(1, ["prisma", "migrate", "diff", "--to-url", database_url]) + + message: Final = _redact_command_error(error) + + assert "zq" not in message + assert "7x" not in message + assert "postgresql://REDACTED@db:5432/litellm" in message + assert "returned non-zero exit status 1" in message + + +@pytest.mark.parametrize( + "command", + ( + None, + Path("/usr/bin/prisma"), + 7, + ("prisma", "migrate", "deploy"), + ["prisma", "migrate", "deploy"], + "prisma migrate deploy", + ), +) +def test_redact_command_error_preserves_unredacted_command_format( + command: object, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("DATABASE_URL", raising=False) + monkeypatch.delenv("DIRECT_URL", raising=False) + error: Final = subprocess.CalledProcessError(1, command) + + assert _redact_command_error(error) == str(error) + + +def test_redact_command_error_masks_password_in_tuple_url_argument( + monkeypatch: pytest.MonkeyPatch, +) -> None: + password: Final = "zq 7x" + database_url: Final = f"postgresql://u:{password}@db:5432/litellm" + monkeypatch.setenv("DATABASE_URL", database_url) + monkeypatch.delenv("DIRECT_URL", raising=False) + error: Final = subprocess.CalledProcessError(1, ("prisma", "migrate", "deploy", "--to-url", database_url)) + + message: Final = _redact_command_error(error) + + assert message.startswith("Command '('") + assert password not in message + assert "postgresql://REDACTED@db:5432/litellm" in message From 9a5e828310efac7cc6646d4725d38cec1ea61f21 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 07:20:14 +0000 Subject: [PATCH 6/6] feat(ui): inline Lens settings tab and investigation editor (#44479) * feat(ui): move worker status into the Lens notch and New investigation into the list toolbar Co-Authored-By: Claude Fable 5.1 * feat(ui): make the Lens notch entry a settings gear that houses the worker section Co-Authored-By: Claude Fable 5.1 * feat(ui): replace the Lens worker modal with an inline Settings tab Co-Authored-By: Claude Fable 5.1 * feat(ui): section the Lens settings tab with tracing status and worker cards Co-Authored-By: Claude Fable 5.1 * fix(ui): let Lens settings sections span the full card width Co-Authored-By: Claude Fable 5.1 * feat(ui): replace the investigation setup modal with an inline side-by-side editor New, edit, and duplicate now take over the Investigations tab body: matching activity on the left, every setting on the right, with no wizard steps or modal Co-Authored-By: Claude Fable 5.1 * feat(ui): step the inline investigation setup vertically with traces alongside Setup now sits on the left as three progressive steps (activity, criteria, run) that collapse to a summary once done and reopen on click. Matching activity stays on the right for every step. The editor gets a back control and the Investigations notch shows a New, Editing, or Duplicate badge while the editor is open Co-Authored-By: Claude Fable 5.1 * feat(ui): page the matching activity preview with useInfiniteQuery as it scrolls Replace the Previous/Next offset buttons with the same infinite query and near-tail prefetch the traces list uses, so the preview keeps loaded runs and its title while the next page arrives Co-Authored-By: Claude Fable 5.1 * refactor(ui): make the investigation step field map exhaustive over the form schema Co-Authored-By: Claude Fable 5.1 * fix(ui): always show the Settings tab label in the Lens mode switch Co-Authored-By: Claude Fable 5.1 * refactor(ui): collapse Lens worker cards into compact status rows Co-Authored-By: Claude Fable 5.1 * fix(ui): keep the Lens Settings tab icon-only in every state Co-Authored-By: Claude Fable 5.1 * wip * fix(ui): tick the Lens worker health dot so an expired heartbeat goes stale Co-Authored-By: Claude Fable 5.1 * refactor(ui): regroup Lens settings, model and api layers Co-Authored-By: Claude Fable 5.1 * refactor(ui): dedupe Lens formatting helpers Co-Authored-By: Claude Fable 5.1 * fix(ui): poll Lens once, drive the interval from data, settle mutations before invalidating Co-Authored-By: Claude Fable 5.1 * refactor(ui): let Lens leaves fetch their own data Co-Authored-By: Claude Fable 5.1 * refactor(ui): bind drawer and trace shortcuts through react-hotkeys-hook One useShortcut hook replaces the three hand-rolled keydown listeners in SidePanel, the trace step tree and the log drawer, with a layer option deciding which keys a pane claims from the panel around it. The span tree footer now renders ShortcutHints from what is actually bound instead of hand-typed kbd text. * refactor(ui): extract Inspector from SidePanel Inspector.Root owns the open item, J/K stepping, Escape and full screen; Inspector.Row marks a list entry with aria-selected and data-state and toggles it on click or Enter/Space; Inspector.Panel is the resizable side panel with the exit animation and click-outside rules. The runs table and section compose these parts directly, so RunDrawer and the SidePanel prop bag go away. * refactor(ui): model the Lens worker screen as a tagged union and slot in its ready action workerScreen() decides between list, form and install from the worker rows, the registration result and the edit target, so WorkerSettings switches on one value and each card owns its own copy. The post-install CTA is now a ReactNode slot that LensWorkspace fills instead of an onReady callback threaded through LensSettings and WorkerSettings. Clipboard copy state lives in WorkerInstall as mutations, and the styled settings leaves export Props types, set data-slot and accept native element props. Co-Authored-By: Claude Fable 5.1 * refactor(ui): compose the Lens setup stepper from SetupStep children Each step's heading, summary and fields now live together in one SetupStep instead of four parallel structures keyed by index, and the last-step spacing comes from CSS rather than a passed index. The mode prop is now required since InvestigationsView always passes it. Co-Authored-By: Claude Fable 5.1 * fix(ui): keep the Lens Settings panel mounted so a pending worker install survives tab switches The settings TabsContent unmounted WorkerSettings whenever another tab was active, dropping the one-time worker token shown during install. The panel now uses keepMounted, and the workspace test registers a worker, switches tabs and back, then follows the connected worker into the first investigation through the slotted CTA. Co-Authored-By: Claude Fable 5.1 * test(ui): cover the Lens worker install waiting-to-connected transition Co-Authored-By: Claude Fable 5.1 * refactor(ui): give the Lens activity preview a grouped contract and a structural debounce useMatchingActivity now owns its return types (scope options, preview status, page and optional manual selection) instead of borrowing them from the components it feeds, and the preview takes those groups plus the section attributes. The clear-selection action moves into the preview footer, RunList becomes a RunRow leaf, and ScopeFields drops the unused nameField and id props now that MetadataFilters calls useId itself. The preview scope settles through a hashKey-based useDebouncedValue instead of JSON round-tripping into state, and the loading title follows isPlaceholderData since the query keeps previous data. RunFields and AnalysisModelField take the analysis models and the model gate as two objects instead of seven flat props. Co-Authored-By: Claude Fable 5.1 * refactor(ui): select the Lens investigations screen with a pure tagged union investigationScreen maps the list query and the route to one of loading, failed, welcome, list, detail, setup or missing, so the view can switch instead of juggling mutually exclusive booleans. The status model gains activeJob, carries connected inside Readiness and folds the activity probe into one ActivityCheck value for the welcome page Co-Authored-By: Claude Fable 5.1 * test(ui): cover the Lens preview footer clear action and the preview debounce Co-Authored-By: Claude Fable 5.1 * refactor(ui): let Lens investigation leaves own their URL slice and express intent InvestigationsView renders the screen union and owns every write through useInvestigationActions, so leaves receive on* handlers instead of the API writer. useInvestigationResults becomes useRunSnapshot; FindingsTab, HistoryTab, RunPicker and RequestEvidenceSheet read their own nuqs slice and run their own queries. The finding sheet becomes an Inspector side panel (FindingDetails) keyed per finding, with the trace and request evidence sheets grouped in EvidenceSheets. WatchAllBanner owns its mutation, the welcome page takes the readiness and activity values, the progress sampler records on the wall clock outside render, and run history invalidates when the list reports a scheduler-started job Co-Authored-By: Claude Fable 5.1 * test(ui): cover Lens investigation intents, pause, cancel, history refresh and the finding panel Co-Authored-By: Claude Fable 5.1 * fix(ui): keep the inspector open behind sheet overlays and mount HotkeysProvider from a client wrapper * feat(ui): open Lens runs and evidence in the inspector panel A quote's original trace step or logged request now stacks inside the finding panel behind a back link, keeping the finding and its feedback draft mounted. The detail Runs tab and the setup activity preview open runs in the same panel with J/K stepping, so the TraceSheet and RequestEvidenceSheet modals are gone. Picking a different finding or run clears any stacked evidence from the URL. * feat(ui): open Lens investigations in the inspector panel beside the list The investigations list stays on screen and a row opens its investigation in the side panel, so J/K walk investigations and their open findings in display order and the selected row carries the same highlight as runs. The panel body is the former detail page; findings and runs opened inside it nest their own inspector, which claims the keys from the one around it while open. Opening an investigation and peeking at a finding now replace each other in the URL. * fix(ui): run the Lens notch border along the tab pill and flag only a disconnected worker Co-Authored-By: Claude Fable 5.1 * fix(ui): show the shortcut hints in every inspector panel Co-Authored-By: Claude Fable 5.1 * refactor(ui): extract the Lens dot field into composable DotFieldRoot and DotFieldCanvas Move the dot grid model and canvas painter out of TracesTimeline into components/lens/dotField so other Lens surfaces can reuse it. The root owns layout and context; overlays compose as children. agoLabel moves to lens/model/format and the unused columnTop helper is dropped. Co-Authored-By: Claude Fable 5.1 * refactor(ui): drop the manual refresh button from the runs time controls Live polls and range changes refetch, so the button only cleared the zoom, which Escape already does Co-Authored-By: Claude Opus 5.5 * refactor(ui): extract the Lens run search into a composable SearchBox primitive Move the query parser, glob matcher and autocomplete out of runSearch into components/lens/search, generic over a QueryLanguage (field specs plus what free text searches). SearchBox.Root owns the ProseMirror state, menu and keyboard; SearchBox.Input and SearchBox.Suggestions compose under it. Clause highlighting becomes a ProseMirror plugin built from the language. RunSearch now only declares the run fields and composes the parts, so the investigations tab can define its own language and reuse the same box. Co-Authored-By: Claude Fable 5.1 * fix(ui): label the runs range by preset while Live and pin it once paused Co-Authored-By: Claude Opus 5.5 * refactor(ui): split the Lens search language from where its data lives QueryLanguage is now pure vocabulary (keys, groups, icons). Reading fields off loaded items moves to a ClientIndex consumed by a separate evaluator, and value suggestions come from an injectable ValueSource, so a server-backed runs list can plug in a facet lookup while the investigations tab keeps filtering in memory. The parsed query serializes to a typed SearchQuery (text terms plus eq/neq/glob/nglob filters) that the client evaluator consumes today and a server can consume later. The suggestion menu shows a loading row while a source is still answering. Co-Authored-By: Claude Fable 5.1 * style(ui): format the Lens SearchBox and its test with the project prettier config Co-Authored-By: Claude Fable 5.1 * feat(ui): mirror the Lens run filters as trace SQL with a copyable curl in the search footer The suggestions footer gains a slot, and SearchBox.ApiHint fills it with the API equivalent of the typed query: a dialect chip, a one-line preview and a Copy as curl button. The runs box translates each filter to a predicate over the agent_traces_by_key rollup, bounded to the range the list shows, and copies a POST to /v1/traces/query. Any other list can plug its own translate into the same part. Co-Authored-By: Claude Fable 5.1 * feat(ui): show the Lens introduction as a first-visit dialog with a typed don't-show-again The guided setup no longer replaces the Lens tabs. It opens in a dialog on the first visit of a session or from ?setup=lens, with a close and a "Don't show this again" checkbox in its top-right corner. The header "Set up Lens" button is gone. Dismissal state lives in a new schema-validated web storage helper (src/lib/storage.ts) that reads through useSyncExternalStore, so server renders see the fallback and other tabs stay in sync; only Lens uses it for now. Co-Authored-By: Claude Fable 5.1 * refactor(ui): keep only Copy as curl in the Lens run search footer Drop the SQL chip and predicate preview; SearchBox.ApiHint becomes SearchBox.CopyCommand, which takes the command for the current query. Co-Authored-By: Claude Fable 5.1 * feat(ui): align the Lens investigations list with the traces list Use the shared query SearchBox with investigation fields (name, agent, status, schedule), match the traces toolbar, and drop the count footer and inner padding. Co-Authored-By: Claude Opus 5.5 * feat(ui): let Lens settings bring back the introduction after don't show again Co-Authored-By: Claude Opus 5.5 * refactor(ui): share one InspectorTable between the traces list and the Lens investigations tree Compose TanStack Table, react-virtual and the shadcn table cells into InspectorTable parts (Root, Grid, Header, Body, Row, Indent). Investigations get findings as real sub-rows with TanStack expansion instead of a hand-rolled flattener. Co-Authored-By: Claude Opus 5.5 * refactor(ui): break Lens import cycles and move shared pieces out of lens Search and the dot field go to components/shared, run search and the preview button go to view_logs where they are consumed. Lens api, services and demo live under data/, all URL state in route.ts, storage keys in storage.ts, and the session frame styles become cva variants. Co-Authored-By: Claude Opus 5.5 * refactor(ui): one Lens readiness source and one onboarding flow Readiness is computed once in model/readiness and read through useLensReadiness, replacing useLensSetup, status.readiness and the welcome screen's own checks. The Investigations welcome now renders the same onboarding steps as the introduction dialog, with permissions and actions coming from an OnboardingProvider instead of props passed down four levels. StepIndicator and StateMessage are shared lens components, and the step panels are labelled accordion regions. Co-Authored-By: Claude Opus 5.5 * refactor(ui): read the Lens token from services and use semantic status colors Lens services carry the access token they were built for, so trace evidence, readiness and onboarding read it from context instead of a prop threaded through six components. List and history invalidation lives in one data hook. Status colors use the success, warning and destructive tokens, and template-literal class names go through cn. Co-Authored-By: Claude Opus 5.5 * refactor(ui): split Lens demo fixtures from the fake demo APIs Co-Authored-By: Claude Opus 5.5 * feat(ui): show Lens check history as a dot timeline Co-Authored-By: Claude Opus 5.5 * chores * fix(ui): clear stale Lens evidence on run change and keep read-only users off Settings Also names inline option objects to bring local/no-large-inline-object-arg back under budget. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Yujong Lee Co-authored-by: Claude Fable 5.1 --- ui/litellm-dashboard/package-lock.json | 14 + ui/litellm-dashboard/package.json | 1 + ui/litellm-dashboard/src/app/layout.tsx | 5 +- .../src/components/lens/HeaderActions.tsx | 13 - .../src/components/lens/LensModeSwitch.tsx | 101 ++++ ...test.tsx => LensPage.integration.test.tsx} | 4 + .../components/lens/LensServicesProvider.tsx | 13 - .../lens/LensSetup.integration.test.tsx | 266 +++++---- .../lens/LensWorkspace.integration.test.tsx | 267 ++++++++- .../src/components/lens/LensWorkspace.tsx | 388 ++++++-------- .../src/components/lens/api/mutations.ts | 93 ---- .../src/components/lens/data/LensServices.tsx | 40 ++ .../{ => data}/demo/createLensDemo.test.ts | 5 +- .../lens/data/demo/createLensDemo.ts | 76 +++ .../demo/fixtures.ts} | 78 +-- .../lens/{ => data}/demo/lensDemoLongTrace.ts | 0 .../lens/{ => data}/demo/scenarios.ts | 0 .../src/components/lens/data/mutations.ts | 56 ++ .../components/lens/{api => data}/queries.ts | 63 ++- .../components/lens/{api => data}/service.ts | 14 +- .../components/lens/hooks/useLensReadiness.ts | 58 ++ .../lens/hooks/useWorkerConnected.ts | 12 + .../lens/investigations/Evidence.tsx | 131 +++++ .../FindingDetails.integration.test.tsx | 135 +++++ .../lens/investigations/FindingDetails.tsx | 200 +++++++ .../FindingSheet.integration.test.tsx | 51 -- .../lens/investigations/FindingSheet.tsx | 164 ------ .../lens/investigations/InvestigationList.tsx | 491 +++++++++-------- .../InvestigationNavigation.tsx | 54 -- .../investigations/InvestigationProgress.tsx | 29 +- ...tionStatus.tsx => InvestigationStates.tsx} | 51 +- .../InvestigationsView.integration.test.tsx | 369 ++++++++++--- .../investigations/InvestigationsView.tsx | 505 +++++++----------- .../investigations/InvestigationsWelcome.tsx | 168 ------ .../lens/investigations/IssueBrief.tsx | 2 +- .../investigations/RequestEvidenceSheet.tsx | 69 --- .../lens/investigations/TraceSheet.test.tsx | 20 - .../lens/investigations/TraceSheet.tsx | 64 --- .../lens/investigations/WatchAllBanner.tsx | 31 +- .../investigations/detail/CriteriaTab.tsx | 26 +- .../investigations/detail/FindingsTab.tsx | 189 ++++--- .../lens/investigations/detail/HistoryTab.tsx | 58 +- .../detail/HistoryTimeline.test.ts | 98 ++++ .../investigations/detail/HistoryTimeline.tsx | 92 ++++ .../detail/InvestigationActions.test.tsx | 101 ++++ .../detail/InvestigationActions.tsx | 51 +- .../detail/InvestigationDetail.tsx | 172 ++---- .../detail/InvestigationSummary.tsx | 2 +- .../lens/investigations/detail/JobMeta.tsx | 23 + .../investigations/detail/RunNowDialog.tsx | 49 +- .../lens/investigations/detail/RunPicker.tsx | 41 +- .../lens/investigations/detail/RunsTab.tsx | 198 ++++--- .../investigations/detail/useRunHistory.ts | 20 + .../investigations/investigationQuery.test.ts | 76 +++ .../lens/investigations/investigationQuery.ts | 30 ++ .../investigationScreen.test.ts | 103 ++++ .../investigations/investigationScreen.ts | 51 ++ .../investigations/useInvestigationActions.ts | 26 + .../investigations/useInvestigationResults.ts | 107 ---- .../lens/investigations/useProgressSamples.ts | 32 ++ .../lens/investigations/useRunSnapshot.ts | 38 ++ .../src/components/lens/model/findings.ts | 9 +- .../src/components/lens/model/format.test.ts | 18 +- .../src/components/lens/model/format.ts | 12 +- .../src/components/lens/model/inbox.ts | 7 +- .../src/components/lens/model/progress.ts | 2 +- .../components/lens/model/readiness.test.ts | 53 ++ .../src/components/lens/model/readiness.ts | 47 ++ .../runRequest.test.ts} | 2 +- .../src/components/lens/model/runRequest.ts | 30 ++ .../src/components/lens/model/status.test.ts | 48 +- .../src/components/lens/model/status.ts | 30 +- .../src/components/lens/model/types.ts | 14 + .../lens/{setup => model}/watches.ts | 2 +- .../{setup => onboarding}/GatewayFlow.tsx | 5 +- .../lens/onboarding/LensGettingStarted.tsx | 81 +++ .../lens/onboarding/LensIntroDialog.tsx | 80 +++ .../LensIntroduction.module.css | 0 .../LensIntroduction.tsx | 25 +- .../lens/onboarding/OnboardingContext.tsx | 24 + .../lens/onboarding/OnboardingSetup.tsx | 52 ++ .../lens/onboarding/OnboardingSteps.tsx | 251 +++++++++ .../components/lens/{route.tsx => route.ts} | 88 ++- .../lens/runSearch/RunSearch.test.tsx | 140 ----- .../components/lens/runSearch/RunSearch.tsx | 257 --------- .../lens/runSearch/runQuery.test.ts | 96 ---- .../src/components/lens/runSearch/runQuery.ts | 111 ---- .../lens/runSearch/suggestions.test.ts | 75 --- .../components/lens/runSearch/suggestions.ts | 91 ---- .../src/components/lens/services.ts | 20 - .../components/lens/settings/LensSettings.tsx | 84 +++ .../lens/settings/SettingsSection.tsx | 33 ++ .../worker/AnalysisAccessFields.tsx | 8 +- .../worker/AnalysisKeyDetails.tsx | 57 +- .../AnalysisKeyPicker.integration.test.tsx | 7 +- .../worker/AnalysisKeyPicker.tsx | 10 +- .../{setup => settings}/worker/WorkerForm.tsx | 8 +- .../lens/settings/worker/WorkerInstall.tsx | 104 ++++ .../lens/settings/worker/WorkerList.tsx | 65 +++ .../WorkerSettings.integration.test.tsx} | 81 ++- .../lens/settings/worker/WorkerSettings.tsx | 168 ++++++ .../settings/worker/useAnalysisKeyInfo.ts | 10 + .../lens/settings/worker/usePrepareWorker.ts | 66 +++ .../worker/workerCommand.ts | 0 .../worker/workerSchema.test.ts | 0 .../worker/workerSchema.ts | 0 .../lens/settings/worker/workerScreen.test.ts | 45 ++ .../lens/settings/worker/workerScreen.ts | 21 + ...> InvestigationSetup.integration.test.tsx} | 370 +++++++------ .../lens/setup/InvestigationSetup.tsx | 235 ++++++++ .../lens/setup/InvestigationSetupDialog.tsx | 205 ------- .../lens/setup/LensGettingStarted.tsx | 227 -------- .../components/lens/setup/LensSetupSteps.tsx | 229 -------- .../lens/setup/MatchingActivity.tsx | 166 ------ .../lens/setup/MatchingActivityPreview.tsx | 191 ++++--- .../lens/setup/MonitoringDialog.tsx | 2 +- .../components/lens/setup/SetupSteps.test.tsx | 55 ++ .../src/components/lens/setup/SetupSteps.tsx | 79 +++ .../src/components/lens/setup/WatchPicker.tsx | 2 +- .../lens/setup/fields/AnalysisModelField.tsx | 38 +- .../ExpectationsFields.tsx} | 2 +- .../lens/setup/fields/MetadataFilters.tsx | 21 +- .../RunStep.tsx => fields/RunFields.tsx} | 37 +- .../lens/setup/fields/ScopeFields.tsx | 29 +- .../lens/setup/fields/analysisModels.ts | 22 +- .../lens/setup/fields/useAnalysisModels.ts | 33 ++ .../lens/setup/investigationSchema.test-d.ts | 23 + .../lens/setup/investigationSchema.test.ts | 13 +- .../lens/setup/investigationSchema.ts | 53 +- .../components/lens/setup/steps/ScopeStep.tsx | 38 -- .../lens/setup/useDebouncedValue.test.tsx | 38 ++ .../lens/setup/useDebouncedValue.ts | 36 ++ .../src/components/lens/setup/useLensSetup.ts | 59 -- .../lens/setup/useMatchingActivity.ts | 192 +++++++ .../lens/setup/worker/WorkerDialog.tsx | 196 ------- .../lens/setup/worker/WorkerInstall.tsx | 104 ---- .../lens/setup/worker/WorkerList.tsx | 62 --- .../src/components/lens/storage.ts | 6 + .../src/components/lens/ui/StateMessage.tsx | 39 ++ .../src/components/lens/ui/StepIndicator.tsx | 37 ++ .../src/components/lens/ui/frame.ts | 30 ++ .../src/components/shared/Inspector.test.tsx | 321 +++++++++++ .../src/components/shared/Inspector.tsx | 349 ++++++++++++ .../components/shared/InspectorTable.test.tsx | 106 ++++ .../src/components/shared/InspectorTable.tsx | 179 +++++++ .../components/shared/ShortcutHints.test.tsx | 46 ++ .../src/components/shared/ShortcutHints.tsx | 94 ++++ .../src/components/shared/SidePanel.tsx | 239 --------- .../src/components/shared/StatusDot.tsx | 1 + .../shared/__fixtures__/BoundShortcut.tsx | 19 + .../components/shared/dotField/DotField.tsx | 152 ++++++ .../components/shared/dotField/dots.test.ts | 68 +++ .../src/components/shared/dotField/dots.ts | 57 ++ .../shared/search/SearchBox.test.tsx | 250 +++++++++ .../components/shared/search/SearchBox.tsx | 389 ++++++++++++++ .../shared/search/__fixtures__/notes.ts | 47 ++ .../components/shared/search/evaluate.test.ts | 72 +++ .../src/components/shared/search/evaluate.ts | 42 ++ .../components/shared/search/language.test.ts | 41 ++ .../src/components/shared/search/language.ts | 89 +++ .../shared/search/searchQuery.test.ts | 30 ++ .../components/shared/search/searchQuery.ts | 35 ++ .../shared/search/suggestions.test.ts | 106 ++++ .../components/shared/search/suggestions.ts | 132 +++++ .../components/shared/search/valueSource.ts | 21 + .../components/shared/useShortcut.test.tsx | 76 +++ .../src/components/shared/useShortcut.ts | 51 ++ .../view_logs/LogDetailsDrawer/constants.ts | 7 - .../useKeyboardNavigation.test.tsx | 8 +- .../LogDetailsDrawer/useKeyboardNavigation.ts | 70 +-- .../view_logs/TraceView/AgentTracesPage.tsx | 15 +- .../AgentTracesSection.integration.test.tsx | 83 +-- .../TraceView/AgentTracesSection.tsx | 110 ++-- .../TraceView/AgentTracesTable.test.tsx | 55 +- .../view_logs/TraceView/AgentTracesTable.tsx | 275 +++++----- .../TraceView}/LensPreviewButton.tsx | 0 .../view_logs/TraceView/RunDrawer.test.tsx | 284 ---------- .../view_logs/TraceView/RunDrawer.tsx | 66 --- .../view_logs/TraceView/SpanTree.tsx | 33 -- .../view_logs/TraceView/TimeRangeControls.tsx | 101 ++-- .../view_logs/TraceView/TraceDrawer.test.tsx | 14 +- .../view_logs/TraceView/TraceDrawer.tsx | 65 +-- .../TraceView/TracesTimeline.test.ts | 12 +- .../view_logs/TraceView/TracesTimeline.tsx | 145 +---- .../TracingSetupCard.integration.test.tsx | 2 +- .../view_logs/TraceView/TracingSetupCard.tsx | 2 +- .../view_logs/TraceView/lensField.test.ts | 103 ---- .../view_logs/TraceView/lensField.ts | 61 --- .../TraceView/runSearch/RunSearch.test.tsx | 47 ++ .../TraceView/runSearch/RunSearch.tsx | 41 ++ .../TraceView}/runSearch/RunsToolbar.tsx | 11 +- .../TraceView}/runSearch/__fixtures__/runs.ts | 2 +- .../TraceView/runSearch/runQuery.test.ts | 47 ++ .../view_logs/TraceView/runSearch/runQuery.ts | 36 ++ .../TraceView/runSearch/runSql.test.ts | 92 ++++ .../view_logs/TraceView/runSearch/runSql.ts | 81 +++ .../components/view_logs/letterShortcut.ts | 9 - .../src/contexts/HotkeysProvider.tsx | 7 + ui/litellm-dashboard/src/lib/storage.test.tsx | 59 ++ ui/litellm-dashboard/src/lib/storage.ts | 95 ++++ .../tests/lens-test-utils.tsx | 39 ++ ui/litellm-dashboard/tests/setupTests.ts | 9 +- ui/litellm-dashboard/tests/test-utils.tsx | 5 +- 203 files changed, 9750 insertions(+), 6343 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/lens/HeaderActions.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx rename ui/litellm-dashboard/src/components/lens/{LensNavigation.integration.test.tsx => LensPage.integration.test.tsx} (96%) delete mode 100644 ui/litellm-dashboard/src/components/lens/LensServicesProvider.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/api/mutations.ts create mode 100644 ui/litellm-dashboard/src/components/lens/data/LensServices.tsx rename ui/litellm-dashboard/src/components/lens/{ => data}/demo/createLensDemo.test.ts (95%) create mode 100644 ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts rename ui/litellm-dashboard/src/components/lens/{demo/createLensDemo.ts => data/demo/fixtures.ts} (77%) rename ui/litellm-dashboard/src/components/lens/{ => data}/demo/lensDemoLongTrace.ts (100%) rename ui/litellm-dashboard/src/components/lens/{ => data}/demo/scenarios.ts (100%) create mode 100644 ui/litellm-dashboard/src/components/lens/data/mutations.ts rename ui/litellm-dashboard/src/components/lens/{api => data}/queries.ts (70%) rename ui/litellm-dashboard/src/components/lens/{api => data}/service.ts (96%) create mode 100644 ui/litellm-dashboard/src/components/lens/hooks/useLensReadiness.ts create mode 100644 ui/litellm-dashboard/src/components/lens/hooks/useWorkerConnected.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/Evidence.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FindingDetails.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FindingSheet.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/FindingSheet.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/InvestigationNavigation.tsx rename ui/litellm-dashboard/src/components/lens/investigations/{InvestigationStatus.tsx => InvestigationStates.tsx} (69%) delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/InvestigationsWelcome.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/RequestEvidenceSheet.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/TraceSheet.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/TraceSheet.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/detail/HistoryTimeline.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/detail/HistoryTimeline.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/detail/InvestigationActions.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/detail/JobMeta.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/detail/useRunHistory.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/investigationQuery.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/investigationQuery.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/investigationScreen.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/investigationScreen.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/useInvestigationActions.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/investigations/useInvestigationResults.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/useProgressSamples.ts create mode 100644 ui/litellm-dashboard/src/components/lens/investigations/useRunSnapshot.ts create mode 100644 ui/litellm-dashboard/src/components/lens/model/readiness.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/model/readiness.ts rename ui/litellm-dashboard/src/components/lens/{investigations/detail/RunNowDialog.test.ts => model/runRequest.test.ts} (97%) create mode 100644 ui/litellm-dashboard/src/components/lens/model/runRequest.ts rename ui/litellm-dashboard/src/components/lens/{setup => model}/watches.ts (98%) rename ui/litellm-dashboard/src/components/lens/{setup => onboarding}/GatewayFlow.tsx (96%) create mode 100644 ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/onboarding/LensIntroDialog.tsx rename ui/litellm-dashboard/src/components/lens/{setup => onboarding}/LensIntroduction.module.css (100%) rename ui/litellm-dashboard/src/components/lens/{setup => onboarding}/LensIntroduction.tsx (87%) create mode 100644 ui/litellm-dashboard/src/components/lens/onboarding/OnboardingContext.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/onboarding/OnboardingSetup.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/onboarding/OnboardingSteps.tsx rename ui/litellm-dashboard/src/components/lens/{route.tsx => route.ts} (53%) delete mode 100644 ui/litellm-dashboard/src/components/lens/runSearch/RunSearch.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/runSearch/RunSearch.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/runSearch/runQuery.test.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/runSearch/runQuery.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/runSearch/suggestions.test.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/runSearch/suggestions.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/services.ts create mode 100644 ui/litellm-dashboard/src/components/lens/settings/LensSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/settings/SettingsSection.tsx rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/AnalysisAccessFields.tsx (89%) rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/AnalysisKeyDetails.tsx (61%) rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/AnalysisKeyPicker.integration.test.tsx (91%) rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/AnalysisKeyPicker.tsx (91%) rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/WorkerForm.tsx (86%) create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/WorkerInstall.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/WorkerList.tsx rename ui/litellm-dashboard/src/components/lens/{setup/worker/WorkerDialog.integration.test.tsx => settings/worker/WorkerSettings.integration.test.tsx} (65%) create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/WorkerSettings.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/useAnalysisKeyInfo.ts create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/usePrepareWorker.ts rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/workerCommand.ts (100%) rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/workerSchema.test.ts (100%) rename ui/litellm-dashboard/src/components/lens/{setup => settings}/worker/workerSchema.ts (100%) create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/workerScreen.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/settings/worker/workerScreen.ts rename ui/litellm-dashboard/src/components/lens/setup/{InvestigationSetupDialog.integration.test.tsx => InvestigationSetup.integration.test.tsx} (64%) create mode 100644 ui/litellm-dashboard/src/components/lens/setup/InvestigationSetup.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/InvestigationSetupDialog.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensGettingStarted.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/LensSetupSteps.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/MatchingActivity.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/SetupSteps.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/SetupSteps.tsx rename ui/litellm-dashboard/src/components/lens/setup/{steps/ExpectationsStep.tsx => fields/ExpectationsFields.tsx} (98%) rename ui/litellm-dashboard/src/components/lens/setup/{steps/RunStep.tsx => fields/RunFields.tsx} (80%) create mode 100644 ui/litellm-dashboard/src/components/lens/setup/fields/useAnalysisModels.ts create mode 100644 ui/litellm-dashboard/src/components/lens/setup/investigationSchema.test-d.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/steps/ScopeStep.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/useDebouncedValue.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/setup/useDebouncedValue.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/useLensSetup.ts create mode 100644 ui/litellm-dashboard/src/components/lens/setup/useMatchingActivity.ts delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/worker/WorkerDialog.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/worker/WorkerInstall.tsx delete mode 100644 ui/litellm-dashboard/src/components/lens/setup/worker/WorkerList.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/storage.ts create mode 100644 ui/litellm-dashboard/src/components/lens/ui/StateMessage.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/ui/StepIndicator.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/ui/frame.ts create mode 100644 ui/litellm-dashboard/src/components/shared/Inspector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/Inspector.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/InspectorTable.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/InspectorTable.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/ShortcutHints.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/ShortcutHints.tsx delete mode 100644 ui/litellm-dashboard/src/components/shared/SidePanel.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/__fixtures__/BoundShortcut.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/dotField/DotField.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/dotField/dots.test.ts create mode 100644 ui/litellm-dashboard/src/components/shared/dotField/dots.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/SearchBox.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/search/SearchBox.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/search/__fixtures__/notes.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/evaluate.test.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/evaluate.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/language.test.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/language.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/searchQuery.test.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/searchQuery.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/suggestions.test.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/suggestions.ts create mode 100644 ui/litellm-dashboard/src/components/shared/search/valueSource.ts create mode 100644 ui/litellm-dashboard/src/components/shared/useShortcut.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/useShortcut.ts rename ui/litellm-dashboard/src/components/{lens => view_logs/TraceView}/LensPreviewButton.tsx (100%) delete mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/RunDrawer.test.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/RunDrawer.tsx delete mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/lensField.test.ts delete mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/lensField.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/runSearch/RunSearch.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/runSearch/RunSearch.tsx rename ui/litellm-dashboard/src/components/{lens => view_logs/TraceView}/runSearch/RunsToolbar.tsx (60%) rename ui/litellm-dashboard/src/components/{lens => view_logs/TraceView}/runSearch/__fixtures__/runs.ts (92%) create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/runSearch/runQuery.test.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/runSearch/runQuery.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/runSearch/runSql.test.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/runSearch/runSql.ts delete mode 100644 ui/litellm-dashboard/src/components/view_logs/letterShortcut.ts create mode 100644 ui/litellm-dashboard/src/contexts/HotkeysProvider.tsx create mode 100644 ui/litellm-dashboard/src/lib/storage.test.tsx create mode 100644 ui/litellm-dashboard/src/lib/storage.ts create mode 100644 ui/litellm-dashboard/tests/lens-test-utils.tsx diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index b4c886752df..37027f215fe 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -42,6 +42,7 @@ "react-dom": "19.2.8", "react-error-boundary": "6.1.6", "react-hook-form": "7.82.0", + "react-hotkeys-hook": "5.3.3", "react-intersection-observer": "11.0.1", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", @@ -10537,6 +10538,19 @@ "react": "^16.8.0 || ^17 || ^18 || ^19" } }, + "node_modules/react-hotkeys-hook": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/react-hotkeys-hook/-/react-hotkeys-hook-5.3.3.tgz", + "integrity": "sha512-aswgyWUnE25hmhzHTfKDmKzsaSE5DJ4LKaU/o6rQSXkDd/1Bh9TfAFQbHkf6fLy11HvlYkp+cDDarGdhmCDhoQ==", + "license": "MIT", + "workspaces": [ + "packages/*" + ], + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, "node_modules/react-intersection-observer": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/react-intersection-observer/-/react-intersection-observer-11.0.1.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index bba51e9aa2b..388b6c7eab9 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -59,6 +59,7 @@ "react-dom": "19.2.8", "react-error-boundary": "6.1.6", "react-hook-form": "7.82.0", + "react-hotkeys-hook": "5.3.3", "react-intersection-observer": "11.0.1", "react-json-view-lite": "2.5.0", "react-markdown": "9.1.0", diff --git a/ui/litellm-dashboard/src/app/layout.tsx b/ui/litellm-dashboard/src/app/layout.tsx index 60db7104a03..7bb05ef11a7 100644 --- a/ui/litellm-dashboard/src/app/layout.tsx +++ b/ui/litellm-dashboard/src/app/layout.tsx @@ -7,6 +7,7 @@ import { ThemeProvider } from "next-themes"; import { AuthProvider } from "@/contexts/AuthContext"; import ReactQueryProvider from "@/contexts/ReactQueryProvider"; +import HotkeysProvider from "@/contexts/HotkeysProvider"; import { Toaster } from "@/components/ui/sonner"; const inter = Inter({ subsets: ["latin"] }); @@ -30,7 +31,9 @@ export default function RootLayout({ - {children} + + {children} + diff --git a/ui/litellm-dashboard/src/components/lens/HeaderActions.tsx b/ui/litellm-dashboard/src/components/lens/HeaderActions.tsx deleted file mode 100644 index 8df9307c990..00000000000 --- a/ui/litellm-dashboard/src/components/lens/HeaderActions.tsx +++ /dev/null @@ -1,13 +0,0 @@ -"use client"; - -import { useContext, type ReactNode } from "react"; -import { createPortal } from "react-dom"; - -import { LensPreviewContext } from "./LensPreviewButton"; - -export function HeaderActions({ children }: { children: ReactNode }) { - const preview = useContext(LensPreviewContext); - const actions =
    {children}
    ; - if (preview === undefined) return actions; - return preview.target ? createPortal(actions, preview.target) : null; -} diff --git a/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx b/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx new file mode 100644 index 00000000000..11d02567799 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/LensModeSwitch.tsx @@ -0,0 +1,101 @@ +"use client"; + +import { Tabs as TabsPrimitive } from "@base-ui/react/tabs"; +import { Activity, ScanSearch, Settings } from "lucide-react"; +import { StatusDot } from "@/components/shared/StatusDot"; +import { cn } from "@/lib/cva.config"; +import type { InvestigationActivity } from "./model/status"; +import { useWorkerConnected } from "./hooks/useWorkerConnected"; +import type { LensList } from "./model/types"; +import { LENS_TABS, type LensTab } from "./route"; +import { frameCorner, frameTab } from "./ui/frame"; + +const MODE_ICONS = { traces: Activity, investigations: ScanSearch, settings: Settings } as const; + +const ACTIVITY_DOT: Record, { className: string; label: string }> = { + running: { className: "bg-info motion-safe:animate-pulse", label: "An investigation is running" }, + queued: { className: "bg-muted-foreground/60", label: "An investigation is queued" }, +}; + +function ActivityDot({ activity }: { activity: InvestigationActivity }) { + if (activity === "idle") return null; + return ( +