diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1a06906fdf9..bb330d00756 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3,7 +3,7 @@ import json import os from collections.abc import Callable, Mapping from datetime import datetime -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple import httpx from pydantic import ( @@ -73,6 +73,27 @@ else: Span = Any +class ReconcileOutcome(NamedTuple): + """What a model reconcile observed, captured while it still held the reconcile + lock. + + Both fields have to be read under that lock to be worth anything. ``live_after`` + in particular is the router's serving state the instant this reconcile finished, + which is NOT the same as what a later snapshot would see: any other model write + admitted in between briefly un-serves every db model (see ``clear_cache``), so a + caller that re-snapshots at verdict time can observe that hole and blame its own + reload for it. + + - ``still_desired``: the db + config ids the reconcile reconciled against, or None + when no reconcile ran and the desired set is therefore unknown. + - ``live_after``: the ids the router served immediately after the reconcile, or + None when no reconcile ran. + """ + + still_desired: frozenset[str] | None + live_after: frozenset[str] | None + + class SupportedDBObjectType(str, enum.Enum): """ Supported database object types for fine-grained DB storage control. diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 8a52b0d1abb..47566d6b6d5 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -35,6 +35,7 @@ from litellm.proxy._types import ( PrismaCompatibleUpdateDBModel, ProxyErrorTypes, ProxyException, + ReconcileOutcome, TeamModelAddRequest, TeamModelDeleteRequest, UserAPIKeyAuth, @@ -534,7 +535,7 @@ async def patch_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload: Final = live_model_ids_snapshot() - still_desired_ids: Final = await clear_cache() + reload_outcome: Final = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( @@ -554,7 +555,8 @@ async def patch_model( before=live_before_reload, written_models=[(model_id, getattr(updated_model, "model_info", None))], action="update", - still_desired=still_desired_ids, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) return updated_model @@ -640,7 +642,7 @@ async def _set_model_blocked_status( ) live_before_reload: Final = live_model_ids_snapshot() - still_desired_ids: Final = await clear_cache() + reload_outcome: Final = await clear_cache() asyncio.create_task( create_object_audit_log( @@ -661,7 +663,8 @@ async def _set_model_blocked_status( before=live_before_reload, written_models=[(data.model_id, getattr(updated_model, "model_info", None))], action=action, - still_desired=still_desired_ids, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) return updated_model @@ -1033,9 +1036,15 @@ async def delete_team_models( if deleted_model_ids: await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable") + # Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are + # gone, but a reconcile holding a pre-delete snapshot would upsert these ids back + # onto this pod. The lock orders the eviction after any in-flight reconcile. if llm_router is not None: - for model_id in deleted_model_ids: - llm_router.delete_deployment(id=model_id) + from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK + + async with MODEL_RECONCILE_LOCK: + for model_id in deleted_model_ids: + llm_router.delete_deployment(id=model_id) return deleted_model_ids @@ -1355,6 +1364,7 @@ async def delete_model( """ from litellm.proxy.proxy_server import ( + MODEL_RECONCILE_LOCK, llm_router, premium_user, prisma_client, @@ -1403,8 +1413,15 @@ async def delete_model( ) ## DELETE FROM ROUTER ## + # Under MODEL_RECONCILE_LOCK. The db row is already gone, but a reconcile + # that snapshotted the db BEFORE that delete still lists this id as desired, + # and its _add_deployment upserts the deployment straight back -- leaving + # this pod serving a model the database no longer has, until the next + # reconcile. Taking the lock orders this eviction after any such in-flight + # reconcile's re-add, so the eviction is the last word. if llm_router is not None: - llm_router.delete_deployment(id=model_info.id) + async with MODEL_RECONCILE_LOCK: + llm_router.delete_deployment(id=model_info.id) # Runs after the row delete so the sibling check sees post-delete state. if model_params.model_info.team_id is not None: @@ -1579,7 +1596,7 @@ async def add_new_model( """ live_before_reload: Final = live_model_ids_snapshot() - still_desired_ids: frozenset[str] | None = None + reload_outcome: ReconcileOutcome = ReconcileOutcome(still_desired=None, live_after=None) try: _original_litellm_model_name: Final = model_params.model_name if model_params.model_info.team_id is None: @@ -1594,7 +1611,7 @@ async def add_new_model( user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, ) - still_desired_ids = await proxy_config.add_deployment( + reload_outcome = await proxy_config.add_deployment( prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj ) # don't let failed slack alert block the /model/new response @@ -1641,7 +1658,8 @@ async def add_new_model( before=live_before_reload, written_models=[(model_response.model_id, getattr(model_response, "model_info", None))], action="create", - still_desired=still_desired_ids, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) return model_response @@ -1768,7 +1786,7 @@ async def update_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload: Final = live_model_ids_snapshot() - still_desired_ids: Final = await clear_cache() + reload_outcome: Final = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( create_object_audit_log( @@ -1795,7 +1813,8 @@ async def update_model( before=live_before_reload, written_models=[(_model_id, getattr(model_response, "model_info", None))], action="update", - still_desired=still_desired_ids, + still_desired=reload_outcome.still_desired, + live_after=reload_outcome.live_after, ) return model_response @@ -2100,6 +2119,7 @@ def reload_serving_verdict( written_models: Sequence[tuple[str, object]], written_must_serve: bool, still_desired: frozenset[str] | None = None, + live_after: frozenset[str] | None = None, ) -> tuple[tuple[str, ...], tuple[str, ...]]: """Judge a write-triggered reload by diffing the router's serving state instead of trusting any layer of the reload stack to report its own failure. @@ -2121,9 +2141,16 @@ def reload_serving_verdict( yet polled, so the reload dropping it is the reconcile working rather than damage. Without it (no reconcile ran) every drop is reported, which is the safe direction. + ``live_after`` is the router's serving state captured by the reload itself, while it + still held MODEL_RECONCILE_LOCK. Pass it whenever the caller has it: re-reading the + router here instead means sampling it after the lock was released, where the NEXT + reconcile's leading wipe (clear_cache un-serves every db model before reloading + them) shows up as this reload having dropped them. Falling back to a fresh read is + only correct when no reconcile ran and there is nothing to be concurrent with. + Returns (written ids violating their obligation, collateral ids no longer served). """ - now: Final = live_model_ids_snapshot() + now: Final = live_model_ids_snapshot() if live_after is None else live_after written_ids: Final = frozenset(model_id for model_id, _ in written_models) if written_must_serve: missing = tuple( @@ -2143,16 +2170,23 @@ def raise_if_reload_degraded_serving( written_models: Sequence[tuple[str, object]], action: str, still_desired: frozenset[str] | None = None, + live_after: frozenset[str] | None = None, ) -> None: """The caller-visible error this pod's model-write endpoints owe their caller when the model they wrote is not being served after the reload they triggered. The DB write is durable either way and every other pod reloads on its own interval; this - speaks only for the handling pod.""" + speaks only for the handling pod. + + Callers hold a ReconcileOutcome from the reload; pass BOTH of its fields. Supplying + still_desired without live_after mixes a snapshot taken under the reconcile lock + with one taken after it was released, which is what makes a concurrent model write + look like collateral damage.""" missing, collateral = reload_serving_verdict( before=before, written_models=written_models, written_must_serve=True, still_desired=still_desired, + live_after=live_after, ) if not missing and not collateral: return @@ -2179,14 +2213,20 @@ def raise_if_reload_degraded_serving( ) -async def clear_cache() -> frozenset[str] | None: +async def clear_cache() -> ReconcileOutcome: """ Clear router caches and reload models. - Returns the db + config id set the reload reconciled against, or None when no - reload ran, so callers can pass it to raise_if_reload_degraded_serving. + Returns what the reload saw (see ReconcileOutcome) so callers can pass it to + raise_if_reload_degraded_serving. + + Runs under MODEL_RECONCILE_LOCK for its whole extent, not just the reload at the + end, so the auto-router reset and the reload that rebuilds those routers are atomic + to any other reconcile. The inner call is _add_deployment_locked because + add_deployment would re-acquire the same non-reentrant lock and deadlock. """ from litellm.proxy.proxy_server import ( + MODEL_RECONCILE_LOCK, llm_router, prisma_client, proxy_config, @@ -2196,61 +2236,88 @@ async def clear_cache() -> frozenset[str] | None: if llm_router is None or prisma_client is None: verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear") - return None + return ReconcileOutcome(still_desired=None, live_after=None) - try: - # Only clear DB models, preserve config models - verbose_proxy_logger.debug("Clearing only DB models, preserving config models") + async with MODEL_RECONCILE_LOCK: + try: + # Only clear DB models, preserve config models + verbose_proxy_logger.debug("Clearing only DB models, preserving config models") - # Get current models and filter out DB models - current_models: Final = llm_router.model_list.copy() - config_models: Final = [] - db_model_ids: Final = [] + # Get current models and filter out DB models + current_models: Final = llm_router.model_list.copy() + config_models: Final = [] + db_model_ids: Final = [] - for model in current_models: - model_info = model.get("model_info", {}) - if model_info.get("db_model", False): - # This is a DB model, mark for deletion - db_model_ids.append(model_info.get("id")) - else: - # This is a config model, preserve it - config_models.append(model) + db_router_names: Final = set() - # Clear only DB models - for model_id in db_model_ids: - llm_router.delete_deployment(id=model_id) + for model in current_models: + model_info = model.get("model_info", {}) + if model_info.get("db_model", False): + db_model_ids.append(model_info.get("id")) + # Auto-router deployments (and only those) are wiped here, in the + # same pass, so the reload rebuilds them -- see the comment below. + model_name = model.get("model_name") + if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith( + "auto_router/" + ): + db_router_names.add(model_name) + router_model_id = model_info.get("id") + if router_model_id is not None: + llm_router.delete_deployment(id=router_model_id) + else: + # This is a config model, preserved by the reconcile below + config_models.append(model) - # Clear only DB-backed auto-router-family entries, keyed by model_name, so the - # reload below rebuilds them fresh. A blanket .clear() would also drop config-defined - # routers, which are never re-added below (add_deployment only reloads DB models), - # leaving them permanently unroutable until a full proxy restart for every tenant. - # Restrict to deployments whose model is actually an auto_router/* so a config - # router that merely shares a model_name with a regular DB model isn't evicted. The - # auto_router/ prefix also covers quality_router/ and adaptive_router/, so pop the - # name from every router registry (no-op where absent); missing quality/adaptive - # entries would otherwise make init raise "already exists" on reload and abort it. - db_router_names: Final = { - model.get("model_name") - for model in current_models - if model.get("model_name") is not None - and model.get("model_info", {}).get("db_model", False) - and str(model.get("litellm_params", {}).get("model", "")).startswith("auto_router/") - } - for model_name in db_router_names: - llm_router.auto_routers.pop(model_name, None) - llm_router.complexity_routers.pop(model_name, None) - llm_router.adaptive_routers.pop(model_name, None) - llm_router.quality_routers.pop(model_name, None) + # ORDINARY db deployments are deliberately NOT wiped. This used to + # delete_deployment() every db model before the reload put them back, which + # left the router serving ZERO db models for the whole width of the reload + # -- a real data-plane hole that every inference request landing in it fell + # into. It was also redundant for them: the reload's _delete_deployment + # evicts exactly the ids the db no longer lists, and upsert_deployment + # pops-and-re-adds a deployment whose params changed while no-opping one + # that did not, so the reconcile converges on its own. Every mutation is + # visible to that comparison -- `blocked` and (for premium) `updated_at` + # are written into model_info. + # + # AUTO-ROUTER db deployments are the exception and ARE wiped -- in the + # classification pass above, together with the strategy entries popped + # just below. Their strategy registries are keyed + # by model_name, which no deployment-id reconcile touches, so they have to + # be popped and rebuilt here. But the rebuild only happens on the ADD path: + # Router.upsert_deployment returns early when a deployment is unchanged and + # never reaches add_deployment -> _add_deployment -> + # init_auto_router_deployment, which is what repopulates the registries. + # Popping without deleting would therefore strip every db-backed auto, + # complexity, adaptive and quality router on this pod and never put it back, + # so ANY unrelated model write would leave them unroutable until a restart. + # Deleting the deployment forces upsert down the add path, which rebuilds + # both the deployment and its strategy entry. + # + # That pass restricts the wipe to deployments whose model is actually an + # auto_router/* so a config router that merely shares a model_name with a + # regular db model isn't evicted -- config routers are never re-added by the + # reload (it only reloads db models) and would be permanently unroutable. + # The auto_router/ prefix also covers quality_router/ and adaptive_router/, + # so pop the name from every registry (no-op where absent); a missing + # quality/adaptive entry would otherwise make init raise "already exists" + # on reload and abort it. + for model_name in db_router_names: + llm_router.auto_routers.pop(model_name, None) + llm_router.complexity_routers.pop(model_name, None) + llm_router.adaptive_routers.pop(model_name, None) + llm_router.quality_routers.pop(model_name, None) - # Reload only DB models - still_desired_ids: Final = await proxy_config.add_deployment( - prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj - ) + # Reload only DB models. _add_deployment_locked, not add_deployment: this + # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not + # reentrant, so the public wrapper would deadlock against itself. + outcome: Final = await proxy_config._add_deployment_locked( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) - verbose_proxy_logger.debug( - "Cleared %s DB models, preserved %s config models", len(db_model_ids), len(config_models) - ) - return still_desired_ids - except Exception as e: - verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e) - return None + verbose_proxy_logger.debug( + "Reconciled %s DB models, preserved %s config models", len(db_model_ids), len(config_models) + ) + return outcome + except Exception as e: + verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e) + return ReconcileOutcome(still_desired=None, live_after=None) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index de891af5674..3ac002ce8cc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -467,6 +467,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( _add_model_to_db, _add_team_model_to_db, _deduplicate_litellm_router_models, + live_model_ids_snapshot, ) from litellm.proxy.management_endpoints.model_management_endpoints import ( router as model_management_router, @@ -2166,6 +2167,15 @@ experimental = False #### GLOBAL VARIABLES #### llm_router: Router | None = None llm_model_list: list | None = None +# Serializes every model reconcile (ProxyConfig.add_deployment and clear_cache) so the +# read-modify-write of llm_router above is atomic. Without it, two concurrent model +# writes each reconcile the router against their OWN db snapshot, and the one holding +# the older snapshot evicts the deployment the newer one just added -- the db keeps the +# row, this pod stops serving it. Control-plane only (model create/update/delete and +# the config-sync tick), never on a completion path, so the serialization is free. +# Module-level rather than per-ProxyConfig because llm_router is a module global and a +# second ProxyConfig instance must not get its own independent lock over it. +MODEL_RECONCILE_LOCK: Final = asyncio.Lock() general_settings: dict = {} config_passthrough_endpoints: list[dict[str, Any]] | None = None log_file: Final = "api_log.json" @@ -6456,16 +6466,37 @@ class ProxyConfig: self, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, - ) -> frozenset[str] | None: + ) -> ReconcileOutcome: """ - Check db for new models - Check if model id's in router already - If not, add to router - Returns the ids the db + config say should be served after the reconcile, or - None when no reconcile ran. Callers that judge their own reload need it to tell - a deliberate eviction from a deployment that went missing. + Serialized against every other model reconcile by MODEL_RECONCILE_LOCK, because + the work below is a read-modify-write of the shared ``llm_router`` global: it + reads the db into a snapshot and then makes the router match that snapshot. Two + of those interleaving is not a lost update but an eviction -- the request whose + snapshot predates the other's commit reconciles the newer model *out* of the + router, since _delete_deployment removes every live deployment absent from the + snapshot it was handed. The model stays in the db and this pod stops serving it + until some later reload puts it back. + + Returns what the reconcile saw, captured before the lock is released so a + caller's verdict cannot be corrupted by the next reconcile's own in-flight + window. See ReconcileOutcome. """ + async with MODEL_RECONCILE_LOCK: + return await self._add_deployment_locked(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + + async def _add_deployment_locked( + self, + prisma_client: PrismaClient, + proxy_logging_obj: ProxyLogging, + ) -> ReconcileOutcome: + """add_deployment's body, minus the locking. MODEL_RECONCILE_LOCK MUST already + be held. Split out for the one caller that has to hold the lock across more than + this reconcile -- clear_cache, which un-serves every db model before calling it + and would deadlock on a re-acquire.""" global llm_router, llm_model_list, master_key, general_settings still_desired_ids: frozenset[str] | None = None @@ -6508,7 +6539,12 @@ class ProxyConfig: except Exception as e: verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e) - return still_desired_ids + # Read while the lock is still held: once it is released the next reconcile can + # begin, and clear_cache's leading wipe would make this look like a mass drop. + return ReconcileOutcome( + still_desired=still_desired_ids, + live_after=None if still_desired_ids is None else live_model_ids_snapshot(), + ) def start_config_sync_subscriber( self, diff --git a/tests/e2e/ui/fixtures/mock_llm_server/server.py b/tests/e2e/ui/fixtures/mock_llm_server/server.py index 8e92065c696..82c90a9dd64 100644 --- a/tests/e2e/ui/fixtures/mock_llm_server/server.py +++ b/tests/e2e/ui/fixtures/mock_llm_server/server.py @@ -3,6 +3,7 @@ Mock LLM server for UI e2e tests. Responds to OpenAI-format endpoints with canned responses. """ +import os import time import json import uuid @@ -117,4 +118,12 @@ async def embeddings(request: Request): if __name__ == "__main__": - uvicorn.run(app, host="127.0.0.1", port=8090) + # The port is overridable so two checkouts can run the harness at the same + # time; the default keeps every existing caller (run_e2e.sh, the CircleCI + # job, the e2e chart's sidecar) working untouched. + # + # The HOST is deliberately NOT configurable. Binding loopback is what makes + # this reachable at 127.0.0.1:8090 from inside the proxy's own pod, which is + # the contract the deployed config.yml and the e2e values file are written + # against. + uvicorn.run(app, host="127.0.0.1", port=int(os.environ.get("MOCK_LLM_PORT", "8090"))) diff --git a/tests/e2e/ui/helpers/mcp.ts b/tests/e2e/ui/helpers/mcp.ts new file mode 100644 index 00000000000..b41aec59ded --- /dev/null +++ b/tests/e2e/ui/helpers/mcp.ts @@ -0,0 +1,65 @@ +import { expect, Page as PwPage } from "@playwright/test"; +import { navigateToPage } from "./navigation"; +import { Page } from "../fixtures/pages"; +import { masterKey } from "./traffic"; + +/** Creates an MCP server through the UI's discovery to custom-form flow and returns its name. */ +export async function createMcpServer(page: PwPage, url: string): Promise { + await navigateToPage(page, Page.McpServers); + + await page.getByRole("button", { name: /Add New MCP Server/i }).click(); + const discovery = page.getByRole("dialog").filter({ hasText: "Add MCP Server" }); + await expect(discovery).toBeVisible({ timeout: 5_000 }); + await discovery.getByRole("button", { name: /Custom Server/i }).click(); + + const formModal = page.locator(".ant-modal:visible").filter({ hasText: "MCP Server Name" }); + await expect(formModal).toBeVisible({ timeout: 5_000 }); + + // validateMCPServerName rejects spaces and hyphens; the worker index avoids a same-millisecond collision. + const name = `e2e_mcp_${process.env.TEST_WORKER_INDEX ?? "0"}_${Date.now()}`; + await formModal.locator('input[id="server_name"]').fill(name); + + const transportField = formModal.locator(".ant-form-item", { hasText: "Transport Type" }); + await transportField.locator(".ant-select").click(); + await page.locator(".ant-select-dropdown:visible").getByText("Streamable HTTP").click(); + + await formModal.locator('input[id="url"]').fill(url); + + // The auth_type Form.Item has no label prop, so anchor on the enclosing Collapse panel. + const authSection = formModal.locator(".ant-collapse-item", { hasText: /^Authentication/ }); + await authSection.locator(".ant-form-item").first().locator(".ant-select").click(); + await page.locator(".ant-select-dropdown:visible").getByText("None", { exact: true }).click(); + + await formModal.getByRole("button", { name: /^Add MCP Server$/ }).click(); + await expect(page.getByText("MCP Server created successfully").first()).toBeVisible({ timeout: 15_000 }); + + const card = page.getByTestId("mcp-servers-grid").getByText(name).first(); + await expect(card).toBeVisible({ timeout: 10_000 }); + return name; +} + +/** + * Deletes every server carrying `serverName`. Leaked servers break unrelated MCP specs: the page + * reaches out to each one it lists, so unreachable leftovers stall networkidle until it times out. + * Errors are swallowed because this runs from afterEach. + */ +export async function deleteMcpServerByName(page: PwPage, serverName: string): Promise { + const headers = { Authorization: `Bearer ${masterKey()}` }; + try { + const res = await page.request.get("/v1/mcp/server", { headers }); + if (!res.ok()) return; + const servers = (await res.json()) as { server_id: string; server_name?: string }[]; + for (const server of servers.filter((candidate) => candidate.server_name === serverName)) { + await page.request.delete(`/v1/mcp/server/${server.server_id}`, { headers }); + } + } catch { + // best effort, see above + } +} + +/** Opens a server from the grid and switches to its MCP Tools tab. */ +export async function openMcpToolsTab(page: PwPage, serverName: string): Promise { + await page.getByTestId("mcp-servers-grid").getByText(serverName).first().click(); + await expect(page.getByRole("button", { name: /Back to All Servers/i })).toBeVisible({ timeout: 10_000 }); + await page.getByRole("tab", { name: "MCP Tools" }).click(); +} diff --git a/tests/e2e/ui/helpers/playground.ts b/tests/e2e/ui/helpers/playground.ts new file mode 100644 index 00000000000..39aae8398a5 --- /dev/null +++ b/tests/e2e/ui/helpers/playground.ts @@ -0,0 +1,46 @@ +import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { navigateToPage, dismissFeedbackPopup } from "./navigation"; +import { Page } from "../fixtures/pages"; + +/** Controls for the Test Key / Playground page, shared with the router-fallback specs. */ + +/** + * The configuration panel is rendered twice, docked and overlay, with one visible at a time. + * Every control is narrowed to the visible copy or it trips strict mode against its hidden twin. + */ +export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first(); + +/** The model dropdown, addressed by the placeholder it shows before selection. */ +export const modelSelect = (page: PlaywrightPage): Locator => + onlyVisible(page.locator('.ant-select:has(.ant-select-selection-placeholder:text-is("Select a Model"))')); + +/** Send button is icon-only (an up-arrow), so there is no accessible name. */ +export const sendButton = (page: PlaywrightPage): Locator => onlyVisible(page.locator("button:has(.anticon-arrow-up)")); + +/** The Virtual Key Source dropdown, addressed by its currently selected label. */ +export const keySourceSelect = (page: PlaywrightPage, current: string): Locator => + onlyVisible(page.locator(`.ant-select:has(.ant-select-selection-item[title="${current}"])`)); + +export async function openPlayground(page: PlaywrightPage): Promise { + await navigateToPage(page, Page.LlmPlayground); + await dismissFeedbackPopup(page); + await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({ + timeout: 20_000, + }); +} + +export async function selectModel(page: PlaywrightPage, model: string): Promise { + const select = modelSelect(page); + await select.click(); + // Virtualized: options outside the rendered window are absent from the DOM, so search first. + await select.locator("input.ant-select-selection-search-input").fill(model); + // antd portals its dropdown to the body; options carry the value as `title`. + await onlyVisible(page.locator(`.ant-select-item-option[title="${model}"]`)).click({ timeout: 15_000 }); +} + +export async function sendMessage(page: PlaywrightPage, message: string): Promise { + const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false })); + await expect(input).toBeVisible({ timeout: 15_000 }); + await input.fill(message); + await sendButton(page).click(); +} diff --git a/tests/e2e/ui/helpers/roundTrip.ts b/tests/e2e/ui/helpers/roundTrip.ts new file mode 100644 index 00000000000..8d6e264e622 --- /dev/null +++ b/tests/e2e/ui/helpers/roundTrip.ts @@ -0,0 +1,28 @@ +import { expect, Page } from "@playwright/test"; +import { masterKey } from "./traffic"; + +/** + * Runs `action` and returns the parsed body of the first matching request. + * + * `action` is a callback so the listener is armed before the click; awaiting the + * click first lets the request go by, and the test then hangs until timeout. + */ +export async function captureRequestBody( + page: Page, + match: { method: string; urlIncludes: string }, + action: () => Promise, +): Promise> { + const pending = page.waitForRequest((req) => req.method() === match.method && req.url().includes(match.urlIncludes)); + await action(); + const request = await pending; + return JSON.parse(request.postData() ?? "{}") as Record; +} + +/** Reads an endpoint as the master key, so a failure is bad data and not an expired UI token. */ +export async function readBack(page: Page, endpoint: string): Promise { + const res = await page.request.get(endpoint, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + expect(res.ok(), `GET ${endpoint}`).toBe(true); + return (await res.json()) as T; +} diff --git a/tests/e2e/ui/helpers/traffic.ts b/tests/e2e/ui/helpers/traffic.ts new file mode 100644 index 00000000000..a2fc9463c94 --- /dev/null +++ b/tests/e2e/ui/helpers/traffic.ts @@ -0,0 +1,125 @@ +import { APIRequestContext, expect } from "@playwright/test"; + +/** Model names served by fixtures/config.yml, both backed by the mock LLM server. */ +export const CHAT_MODEL_A = "fake-openai-gpt-4"; +export const CHAT_MODEL_B = "fake-anthropic-claude"; + +/** The only completion text fixtures/mock_llm_server/server.py ever returns. */ +export const MOCK_RESPONSE_TEXT = "This is a mock response."; + +export const masterKey = (): string => process.env.LITELLM_MASTER_KEY || "sk-1234"; + +const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? ""; + +interface ChatOptions { + model: string; + prompt: string; + apiKey?: string; + /** Sent as `user`, which lands in the spend log's end_user column. */ + endUser?: string; +} + +/** POST /v1/chat/completions and return the completion id (the Logs Request ID). */ +export async function sendChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise { + const res = await request.post(`${rootPath()}/v1/chat/completions`, { + headers: { + Authorization: `Bearer ${opts.apiKey ?? masterKey()}`, + "Content-Type": "application/json", + }, + data: { + model: opts.model, + messages: [{ role: "user", content: opts.prompt }], + ...(opts.endUser ? { user: opts.endUser } : {}), + }, + }); + expect(res.ok(), `chat completion for ${opts.model} failed (${res.status()}): ${await res.text()}`).toBe(true); + const body = await res.json(); + expect(body.choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT); + return body.id as string; +} + +/** `key` is the sk- value to authenticate with; `token` is its hash, which spend aggregates are keyed by. */ +export async function createVirtualKey( + request: APIRequestContext, + data: Record = {}, +): Promise<{ key: string; token: string; alias?: string }> { + const res = await request.post(`${rootPath()}/key/generate`, { + headers: { + Authorization: `Bearer ${masterKey()}`, + "Content-Type": "application/json", + }, + data, + }); + expect(res.ok(), `key generate failed (${res.status()}): ${await res.text()}`).toBe(true); + const body = await res.json(); + return { + key: body.key as string, + token: (body.token ?? body.token_id) as string, + alias: body.key_alias as string | undefined, + }; +} + +/** Spend logs are flushed on a timer, so an assertion straight after a completion races the writer. */ +export async function waitForSpendLog( + request: APIRequestContext, + requestId: string, + timeoutMs = 60_000, +): Promise { + const deadline = Date.now() + timeoutMs; + let lastStatus = 0; + while (Date.now() < deadline) { + const res = await request.get(`${rootPath()}/spend/logs?request_id=${encodeURIComponent(requestId)}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + lastStatus = res.status(); + if (res.ok()) { + const body = await res.json(); + const rows = Array.isArray(body) ? body : (body?.data ?? []); + if (rows.length > 0) { + return; + } + } + await new Promise((r) => setTimeout(r, 2_000)); + } + throw new Error(`spend log for request ${requestId} never appeared (last /spend/logs status ${lastStatus})`); +} + +const isoDay = (d: Date): string => d.toISOString().slice(0, 10); + +/** + * The Usage page reads /user/daily/activity, a rollup written by a background job, and fetches it once + * on mount. Navigating before the rollup lands leaves a stale render that never refreshes. + */ +export async function waitForKeyInDailyActivity( + request: APIRequestContext, + keyToken: string, + timeoutMs = 120_000, +): Promise { + const now = new Date(); + const start = new Date(now); + start.setDate(start.getDate() - 7); + const query = `start_date=${isoDay(start)}&end_date=${isoDay(now)}`; + + const deadline = Date.now() + timeoutMs; + let lastStatus = 0; + while (Date.now() < deadline) { + const res = await request.get(`${rootPath()}/user/daily/activity?${query}`, { + headers: { Authorization: `Bearer ${masterKey()}` }, + }); + lastStatus = res.status(); + if (res.ok()) { + const body = await res.json(); + const seen = (body?.results ?? []).some( + (day: { breakdown?: { api_keys?: Record } }) => keyToken in (day.breakdown?.api_keys ?? {}), + ); + if (seen) { + return; + } + } + await new Promise((r) => setTimeout(r, 3_000)); + } + throw new Error( + `key ${keyToken} never appeared in /user/daily/activity (last status ${lastStatus}); ` + + "the daily spend rollup may not be running", + ); +} diff --git a/tests/e2e/ui/run_e2e.sh b/tests/e2e/ui/run_e2e.sh index 858eb401c8e..67e3225f668 100755 --- a/tests/e2e/ui/run_e2e.sh +++ b/tests/e2e/ui/run_e2e.sh @@ -12,6 +12,10 @@ set -euo pipefail # ./run_e2e.sh --repeat-each=5 # Run each test 5 times # ./run_e2e.sh --headed # Run with browser visible # +# Ports default to 4000 / 5432 / 8090 and can be moved when another checkout +# already holds them: +# PROXY_PORT=4100 POSTGRES_PORT=5532 MOCK_LLM_PORT=8190 ./run_e2e.sh +# # In CI (CI=true), expects: # - PostgreSQL already running on 127.0.0.1:5432 # - DATABASE_URL already set @@ -28,12 +32,50 @@ MOCK_PID="" PROXY_PID="" PROXY_LOG="" +# Ports, overridable so two checkouts can run this harness at the same time -- +# otherwise a second run aborts on "port 4000 is in use" and the only way out is +# to stop someone else's stack. Defaults are the historical values, so an unset +# environment behaves exactly as before (CI, the CircleCI job and the docs all +# assume 4000/5432/8090). +PROXY_PORT="${PROXY_PORT:-4000}" +POSTGRES_PORT="${POSTGRES_PORT:-5432}" +MOCK_LLM_PORT="${MOCK_LLM_PORT:-8090}" +export MOCK_LLM_PORT + # --- Ensure common tool paths are available (local dev only) --- if [ "$IS_CI" = "false" ]; then for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do [ -d "$p" ] && export PATH="$p:$PATH" done - [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh" + # Sourcing nvm only makes `nvm` available -- it leaves you on whatever the + # default alias points at, which is frequently an older Node than the + # dashboard's engines allow. `npm install` then fails EBADENGINE, npm exits + # non-zero, and because the install below is `--silent ... || true` the error + # is swallowed and the run dies later with the far less obvious + # "sh: next: command not found". + # + # So select a Node that satisfies ui/litellm-dashboard's engines.node, and if + # none is available say so here rather than 200 lines downstream. + if [ -s "$HOME/.nvm/nvm.sh" ]; then + # shellcheck disable=SC1091 + source "$HOME/.nvm/nvm.sh" + required_major="$(sed -nE 's/.*"node"[[:space:]]*:[[:space:]]*">=?([0-9]+).*/\1/p' \ + "$DASHBOARD_DIR/package.json" 2>/dev/null | head -1)" + if [ -n "$required_major" ]; then + current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')" + if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then + echo "Node $(node --version 2>/dev/null || echo 'not found') is below the dashboard's required v${required_major}; selecting a newer one via nvm" + nvm use "$required_major" >/dev/null 2>&1 || nvm use --lts >/dev/null 2>&1 || true + current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')" + if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then + echo "Error: ui/litellm-dashboard requires Node >= v${required_major}, and no such version is installed." + echo " Install one with: nvm install ${required_major}" + exit 1 + fi + fi + echo "Using Node $(node --version) / npm $(npm --version)" + fi + fi fi # --- Cleanup on exit --- @@ -47,7 +89,11 @@ cleanup() { fi echo "Done." } -trap cleanup EXIT INT TERM +on_signal() { + exit 130 +} +trap cleanup EXIT +trap on_signal INT TERM # --- Pre-flight checks --- for cmd in python3 npx uv; do @@ -59,9 +105,14 @@ if [ "$IS_CI" = "false" ]; then for cmd in docker psql; do command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; } done - for port in 4000 5432 8090; do - if lsof -ti ":$port" >/dev/null 2>&1; then - echo "Error: port $port is in use" + # Only a LISTENER conflicts with us. Without -sTCP:LISTEN this also matches + # ESTABLISHED sockets, so an unrelated *outbound* connection from this machine + # to someone else's :5432 (a psql session, a running app, a Prisma engine + # talking to a remote database) aborts the run with "port 5432 is in use" + # while nothing is actually bound locally. + for port in "$PROXY_PORT" "$POSTGRES_PORT" "$MOCK_LLM_PORT"; do + if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then + echo "Error: port $port is in use (override with PROXY_PORT / POSTGRES_PORT / MOCK_LLM_PORT)" exit 1 fi done @@ -69,12 +120,12 @@ if [ "$IS_CI" = "false" ]; then export POSTGRES_USER="e2euser" export POSTGRES_PASSWORD="$(openssl rand -hex 32)" export POSTGRES_DB="litellm_e2e" - export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}" + export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_PORT}/${POSTGRES_DB}" echo "=== Starting PostgreSQL ===" docker run -d --rm --name "$CONTAINER_NAME" \ -e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \ - -p 127.0.0.1:5432:5432 \ + -p "127.0.0.1:${POSTGRES_PORT}:5432" \ postgres:16 echo "Waiting for PostgreSQL..." @@ -91,8 +142,13 @@ fi # --- Credentials --- export LITELLM_MASTER_KEY="sk-1234" -export MOCK_LLM_URL="http://127.0.0.1:8090/v1" +export MOCK_LLM_URL="http://127.0.0.1:${MOCK_LLM_PORT}/v1" export DISABLE_SCHEMA_UPDATE="true" +# The suite resolves its target from E2E_UI_BASE_URL (constants.ts), which +# otherwise defaults to :4000 -- so without this a relocated stack would be +# built and booted correctly and then tested against whatever happens to be +# listening on the default port. +export E2E_UI_BASE_URL="${E2E_UI_BASE_URL:-http://127.0.0.1:${PROXY_PORT}}" # Ensure the proxy serves UI at /ui (not behind a subpath) export SERVER_ROOT_PATH="" # Boot with an external logout URL so proxyLogoutUrl.spec.ts can assert the @@ -108,7 +164,11 @@ export LITELLM_LICENSE="${LITELLM_LICENSE:-}" # --- Rebuild UI from source --- echo "=== Building UI from source ===" cd "$DASHBOARD_DIR" -npm install --silent 2>/dev/null || true +# NOT silenced, and NOT `|| true`. Swallowing this is what turns a one-line +# EBADENGINE ("dashboard requires node >=24, you have v20") into the +# considerably less helpful "sh: next: command not found" from the build below, +# because the deps that provide `next` were never installed. +npm install npm run build # Copy the fresh build to the proxy's static UI directory cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/" @@ -139,7 +199,7 @@ uv run --no-sync python "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" & MOCK_PID=$! for i in $(seq 1 15); do - if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi + if curl -sf http://127.0.0.1:${MOCK_LLM_PORT}/health >/dev/null 2>&1; then break; fi sleep 1 done @@ -149,7 +209,7 @@ cd "$REPO_ROOT" PROXY_LOG="${TMPDIR:-/tmp}/litellm-e2e-proxy-$$.log" uv run --no-sync python -m litellm.proxy.proxy_cli \ --config "$SCRIPT_DIR/fixtures/config.yml" \ - --port 4000 >"$PROXY_LOG" 2>&1 & + --port "$PROXY_PORT" >"$PROXY_LOG" 2>&1 & PROXY_PID=$! echo "Waiting for proxy (logs: $PROXY_LOG)..." @@ -160,7 +220,7 @@ for i in $(seq 1 180); do tail -n 100 "$PROXY_LOG" exit 1 fi - HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:${PROXY_PORT}/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true) if [ "$HTTP_CODE" = "200" ]; then PROXY_READY=1 break @@ -188,9 +248,38 @@ PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAM # --- Playwright --- echo "=== Installing Playwright dependencies ===" cd "$SCRIPT_DIR" -npm install --silent 2>/dev/null || true +# Same reasoning as the dashboard install above: a failure here means the suite +# has no @playwright/test, and the run should say that rather than fail later. +npm install npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium +# Authoring a new spec means running it over and over against a stack that is +# already up -- rebuilding the UI and re-seeding for every iteration costs +# minutes each time. E2E_KEEP_ALIVE brings the stack up, then blocks, so you can +# run `npx playwright test ` yourself from another shell against it. +# Ctrl-C here tears everything down through the usual trap. +if [ "${E2E_KEEP_ALIVE:-0}" = "1" ]; then + cat < + +Press Ctrl-C to tear the stack down. +EOF + while kill -0 "$PROXY_PID" 2>/dev/null; do + sleep 5 + done + echo "Error: proxy process exited unexpectedly. Proxy output:" + tail -n 100 "$PROXY_LOG" + exit 1 +fi + echo "=== Running Playwright tests ===" npx playwright test --config playwright.config.ts "$@" EXIT_CODE=$? diff --git a/tests/e2e/ui/tests/logs/logs.spec.ts b/tests/e2e/ui/tests/logs/logs.spec.ts new file mode 100644 index 00000000000..fc5cce53511 --- /dev/null +++ b/tests/e2e/ui/tests/logs/logs.spec.ts @@ -0,0 +1,221 @@ +import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; +import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation"; +import { Page } from "../../fixtures/pages"; +import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic"; + +/** + * Anchored to traffic this spec generates itself, with a unique prompt and end user per run, so it + * neither depends on seeded spend rows nor collides with other specs under parallelism. + */ + +const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + +/** + * Walking up from the label is the only stable handle: the header carries no role, test id or class, + * and its copy button is icon-only with a hover-only tooltip. + */ +const sectionHeader = (drawer: Locator, label: "Input" | "Output"): Locator => + drawer.getByText(label, { exact: true }).locator("xpath=../../.."); + +/** Every tab stays mounted, so the DOM holds four tables at once; scope to the visible one. */ +const requestLogsRows = (page: PlaywrightPage): Locator => + page.locator("table").filter({ visible: true }).first().locator("tbody tr"); + +const visibleTestId = (page: PlaywrightPage, id: string): Locator => page.getByTestId(id).filter({ visible: true }); + +/** Open the Logs page and filter the table down to a single request id. */ +async function openLogsForRequest(page: PlaywrightPage, requestId: string): Promise { + await navigateToPage(page, Page.Logs); + await dismissFeedbackPopup(page); + + const search = visibleTestId(page, "datatable-search"); + await expect(search).toBeVisible({ timeout: 20_000 }); + await search.fill(requestId); + + const row = requestLogsRows(page).filter({ hasText: requestId }); + await expect(row, `no logs row for request ${requestId}`).toHaveCount(1, { + timeout: 30_000, + }); + return row; +} + +test.describe("Logs page", () => { + test.use({ + storageState: ADMIN_STORAGE_PATH, + // The copy buttons go through navigator.clipboard, which rejects without these. + permissions: ["clipboard-read", "clipboard-write"], + }); + + test("a served request expands to its request and response", async ({ page, request }) => { + const prompt = `logs-detail-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + + // Expand: clicking the row opens the detail drawer for that request. + await row.click(); + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer).toBeVisible({ timeout: 20_000 }); + await expect(drawer.getByText("Request & Response")).toBeVisible({ + timeout: 20_000, + }); + + // The prompt we sent and the mock server's reply are both rendered. + await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({ + timeout: 20_000, + }); + await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 20_000 }); + }); + + // Split out because only the copy path needs a secure context; folding it in would + // take the drawer-rendering coverage down with it. + test("the drawer copies the request and the response to the clipboard", async ({ page, request }) => { + // `navigator.clipboard` is undefined outside a secure context, and handleCopy calls + // writeText unguarded, so on plain HTTP served from a hostname the click throws and no + // toast renders. Skipped rather than weakened so the product gap stays visible. + await page.goto("/ui"); + const isSecure = await page.evaluate(() => window.isSecureContext); + test.skip(!isSecure, "origin is not a secure context, so navigator.clipboard is unavailable"); + + const prompt = `logs-copy-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + await row.click(); + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer).toBeVisible({ timeout: 20_000 }); + + // Copy request: the Input card's copy button puts the prompt on the clipboard. + await sectionHeader(drawer, "Input").getByRole("button").click(); + await expect(page.getByText("Input copied")).toBeVisible({ + timeout: 10_000, + }); + expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(prompt); + + // Copy response: the Output card's copy button puts the completion on it. + await sectionHeader(drawer, "Output").getByRole("button").click(); + await expect(page.getByText("Output copied")).toBeVisible({ + timeout: 10_000, + }); + expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(MOCK_RESPONSE_TEXT); + }); + + test("the Input card collapses and expands", async ({ page, request }) => { + const prompt = `logs-collapse-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + await row.click(); + + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer.getByText("Request & Response")).toBeVisible({ + timeout: 20_000, + }); + + // The body collapses via `max-height: 0; overflow: hidden`, which zeroes its own bounding + // box, so the wrapper reads as hidden while the clipped text node inside it does not. + const header = sectionHeader(drawer, "Input"); + const body = header.locator("xpath=following-sibling::div[1]"); + await expect(header.locator(".anticon-up")).toBeVisible(); + await expect(body).toBeVisible(); + + await header.click(); + await expect(header.locator(".anticon-down")).toBeVisible({ + timeout: 10_000, + }); + await expect(body).toBeHidden({ timeout: 10_000 }); + + await header.click(); + await expect(header.locator(".anticon-up")).toBeVisible({ + timeout: 10_000, + }); + await expect(body).toBeVisible({ timeout: 10_000 }); + await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({ + timeout: 10_000, + }); + }); + + test("the JSON view exposes Request and Response tabs", async ({ page, request }) => { + const prompt = `logs-json-prompt-${uniqueSuffix()}`; + const requestId = await sendChatCompletion(request, { + model: CHAT_MODEL_A, + prompt, + }); + await waitForSpendLog(request, requestId); + + const row = await openLogsForRequest(page, requestId); + await row.click(); + + const drawer = page.locator(".ant-drawer-content").first(); + await expect(drawer.getByText("Request & Response")).toBeVisible({ + timeout: 20_000, + }); + + // antd Radio.Button hides the under its