Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_shadcn_shared_search_user_0812

litellm/proxy/_types.py

@@ -3,7 +3,7 @@ import json
 import os
 from collections.abc import Callable, Mapping
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final, Literal
+from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple
 
 import httpx
 from pydantic import (
@@ -73,6 +73,27 @@ else:
     Span = Any
 
 
+class ReconcileOutcome(NamedTuple):
+    """What a model reconcile observed, captured while it still held the reconcile
+    lock.
+
+    Both fields have to be read under that lock to be worth anything. ``live_after``
+    in particular is the router's serving state the instant this reconcile finished,
+    which is NOT the same as what a later snapshot would see: any other model write
+    admitted in between briefly un-serves every db model (see ``clear_cache``), so a
+    caller that re-snapshots at verdict time can observe that hole and blame its own
+    reload for it.
+
+    - ``still_desired``: the db + config ids the reconcile reconciled against, or None
+      when no reconcile ran and the desired set is therefore unknown.
+    - ``live_after``: the ids the router served immediately after the reconcile, or
+      None when no reconcile ran.
+    """
+
+    still_desired: frozenset[str] | None
+    live_after: frozenset[str] | None
+
+
 class SupportedDBObjectType(str, enum.Enum):
     """
     Supported database object types for fine-grained DB storage control.

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -35,6 +35,7 @@ from litellm.proxy._types import (
     PrismaCompatibleUpdateDBModel,
     ProxyErrorTypes,
     ProxyException,
+    ReconcileOutcome,
     TeamModelAddRequest,
     TeamModelDeleteRequest,
     UserAPIKeyAuth,
@@ -534,7 +535,7 @@ async def patch_model(
 
         # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
         live_before_reload: Final = live_model_ids_snapshot()
-        still_desired_ids: Final = await clear_cache()
+        reload_outcome: Final = await clear_cache()
 
         ## CREATE AUDIT LOG ##
         asyncio.create_task(
@@ -554,7 +555,8 @@ async def patch_model(
             before=live_before_reload,
             written_models=[(model_id, getattr(updated_model, "model_info", None))],
             action="update",
-            still_desired=still_desired_ids,
+            still_desired=reload_outcome.still_desired,
+            live_after=reload_outcome.live_after,
         )
 
         return updated_model
@@ -640,7 +642,7 @@ async def _set_model_blocked_status(
         )
 
         live_before_reload: Final = live_model_ids_snapshot()
-        still_desired_ids: Final = await clear_cache()
+        reload_outcome: Final = await clear_cache()
 
         asyncio.create_task(
             create_object_audit_log(
@@ -661,7 +663,8 @@ async def _set_model_blocked_status(
             before=live_before_reload,
             written_models=[(data.model_id, getattr(updated_model, "model_info", None))],
             action=action,
-            still_desired=still_desired_ids,
+            still_desired=reload_outcome.still_desired,
+            live_after=reload_outcome.live_after,
         )
 
         return updated_model
@@ -1033,9 +1036,15 @@ async def delete_team_models(
     if deleted_model_ids:
         await publish_config_change(redis_cache=coordination_redis_cache(), object_type="litellm_proxymodeltable")
 
+    # Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are
+    # gone, but a reconcile holding a pre-delete snapshot would upsert these ids back
+    # onto this pod. The lock orders the eviction after any in-flight reconcile.
     if llm_router is not None:
-        for model_id in deleted_model_ids:
-            llm_router.delete_deployment(id=model_id)
+        from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK
+
+        async with MODEL_RECONCILE_LOCK:
+            for model_id in deleted_model_ids:
+                llm_router.delete_deployment(id=model_id)
 
     return deleted_model_ids
 
@@ -1355,6 +1364,7 @@ async def delete_model(
         """
 
         from litellm.proxy.proxy_server import (
+            MODEL_RECONCILE_LOCK,
             llm_router,
             premium_user,
             prisma_client,
@@ -1403,8 +1413,15 @@ async def delete_model(
                 )
 
             ## DELETE FROM ROUTER ##
+            # Under MODEL_RECONCILE_LOCK. The db row is already gone, but a reconcile
+            # that snapshotted the db BEFORE that delete still lists this id as desired,
+            # and its _add_deployment upserts the deployment straight back -- leaving
+            # this pod serving a model the database no longer has, until the next
+            # reconcile. Taking the lock orders this eviction after any such in-flight
+            # reconcile's re-add, so the eviction is the last word.
             if llm_router is not None:
-                llm_router.delete_deployment(id=model_info.id)
+                async with MODEL_RECONCILE_LOCK:
+                    llm_router.delete_deployment(id=model_info.id)
 
             # Runs after the row delete so the sibling check sees post-delete state.
             if model_params.model_info.team_id is not None:
@@ -1579,7 +1596,7 @@ async def add_new_model(
             """
 
             live_before_reload: Final = live_model_ids_snapshot()
-            still_desired_ids: frozenset[str] | None = None
+            reload_outcome: ReconcileOutcome = ReconcileOutcome(still_desired=None, live_after=None)
             try:
                 _original_litellm_model_name: Final = model_params.model_name
                 if model_params.model_info.team_id is None:
@@ -1594,7 +1611,7 @@ async def add_new_model(
                         user_api_key_dict=user_api_key_dict,
                         prisma_client=prisma_client,
                     )
-                still_desired_ids = await proxy_config.add_deployment(
+                reload_outcome = await proxy_config.add_deployment(
                     prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
                 )
                 # don't let failed slack alert block the /model/new response
@@ -1641,7 +1658,8 @@ async def add_new_model(
             before=live_before_reload,
             written_models=[(model_response.model_id, getattr(model_response, "model_info", None))],
             action="create",
-            still_desired=still_desired_ids,
+            still_desired=reload_outcome.still_desired,
+            live_after=reload_outcome.live_after,
         )
 
         return model_response
@@ -1768,7 +1786,7 @@ async def update_model(
 
             # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
             live_before_reload: Final = live_model_ids_snapshot()
-            still_desired_ids: Final = await clear_cache()
+            reload_outcome: Final = await clear_cache()
             ## CREATE AUDIT LOG ##
             asyncio.create_task(
                 create_object_audit_log(
@@ -1795,7 +1813,8 @@ async def update_model(
                 before=live_before_reload,
                 written_models=[(_model_id, getattr(model_response, "model_info", None))],
                 action="update",
-                still_desired=still_desired_ids,
+                still_desired=reload_outcome.still_desired,
+                live_after=reload_outcome.live_after,
             )
 
             return model_response
@@ -2100,6 +2119,7 @@ def reload_serving_verdict(
     written_models: Sequence[tuple[str, object]],
     written_must_serve: bool,
     still_desired: frozenset[str] | None = None,
+    live_after: frozenset[str] | None = None,
 ) -> tuple[tuple[str, ...], tuple[str, ...]]:
     """Judge a write-triggered reload by diffing the router's serving state instead of
     trusting any layer of the reload stack to report its own failure.
@@ -2121,9 +2141,16 @@ def reload_serving_verdict(
     yet polled, so the reload dropping it is the reconcile working rather than damage.
     Without it (no reconcile ran) every drop is reported, which is the safe direction.
 
+    ``live_after`` is the router's serving state captured by the reload itself, while it
+    still held MODEL_RECONCILE_LOCK. Pass it whenever the caller has it: re-reading the
+    router here instead means sampling it after the lock was released, where the NEXT
+    reconcile's leading wipe (clear_cache un-serves every db model before reloading
+    them) shows up as this reload having dropped them. Falling back to a fresh read is
+    only correct when no reconcile ran and there is nothing to be concurrent with.
+
     Returns (written ids violating their obligation, collateral ids no longer served).
     """
-    now: Final = live_model_ids_snapshot()
+    now: Final = live_model_ids_snapshot() if live_after is None else live_after
     written_ids: Final = frozenset(model_id for model_id, _ in written_models)
     if written_must_serve:
         missing = tuple(
@@ -2143,16 +2170,23 @@ def raise_if_reload_degraded_serving(
     written_models: Sequence[tuple[str, object]],
     action: str,
     still_desired: frozenset[str] | None = None,
+    live_after: frozenset[str] | None = None,
 ) -> None:
     """The caller-visible error this pod's model-write endpoints owe their caller when
     the model they wrote is not being served after the reload they triggered. The DB
     write is durable either way and every other pod reloads on its own interval; this
-    speaks only for the handling pod."""
+    speaks only for the handling pod.
+
+    Callers hold a ReconcileOutcome from the reload; pass BOTH of its fields. Supplying
+    still_desired without live_after mixes a snapshot taken under the reconcile lock
+    with one taken after it was released, which is what makes a concurrent model write
+    look like collateral damage."""
     missing, collateral = reload_serving_verdict(
         before=before,
         written_models=written_models,
         written_must_serve=True,
         still_desired=still_desired,
+        live_after=live_after,
     )
     if not missing and not collateral:
         return
@@ -2179,14 +2213,20 @@ def raise_if_reload_degraded_serving(
     )
 
 
-async def clear_cache() -> frozenset[str] | None:
+async def clear_cache() -> ReconcileOutcome:
     """
     Clear router caches and reload models.
 
-    Returns the db + config id set the reload reconciled against, or None when no
-    reload ran, so callers can pass it to raise_if_reload_degraded_serving.
+    Returns what the reload saw (see ReconcileOutcome) so callers can pass it to
+    raise_if_reload_degraded_serving.
+
+    Runs under MODEL_RECONCILE_LOCK for its whole extent, not just the reload at the
+    end, so the auto-router reset and the reload that rebuilds those routers are atomic
+    to any other reconcile. The inner call is _add_deployment_locked because
+    add_deployment would re-acquire the same non-reentrant lock and deadlock.
     """
     from litellm.proxy.proxy_server import (
+        MODEL_RECONCILE_LOCK,
         llm_router,
         prisma_client,
         proxy_config,
@@ -2196,61 +2236,88 @@ async def clear_cache() -> frozenset[str] | None:
 
     if llm_router is None or prisma_client is None:
         verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear")
-        return None
+        return ReconcileOutcome(still_desired=None, live_after=None)
 
-    try:
-        # Only clear DB models, preserve config models
-        verbose_proxy_logger.debug("Clearing only DB models, preserving config models")
+    async with MODEL_RECONCILE_LOCK:
+        try:
+            # Only clear DB models, preserve config models
+            verbose_proxy_logger.debug("Clearing only DB models, preserving config models")
 
-        # Get current models and filter out DB models
-        current_models: Final = llm_router.model_list.copy()
-        config_models: Final = []
-        db_model_ids: Final = []
+            # Get current models and filter out DB models
+            current_models: Final = llm_router.model_list.copy()
+            config_models: Final = []
+            db_model_ids: Final = []
 
-        for model in current_models:
-            model_info = model.get("model_info", {})
-            if model_info.get("db_model", False):
-                # This is a DB model, mark for deletion
-                db_model_ids.append(model_info.get("id"))
-            else:
-                # This is a config model, preserve it
-                config_models.append(model)
+            db_router_names: Final = set()
 
-        # Clear only DB models
-        for model_id in db_model_ids:
-            llm_router.delete_deployment(id=model_id)
+            for model in current_models:
+                model_info = model.get("model_info", {})
+                if model_info.get("db_model", False):
+                    db_model_ids.append(model_info.get("id"))
+                    # Auto-router deployments (and only those) are wiped here, in the
+                    # same pass, so the reload rebuilds them -- see the comment below.
+                    model_name = model.get("model_name")
+                    if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith(
+                        "auto_router/"
+                    ):
+                        db_router_names.add(model_name)
+                        router_model_id = model_info.get("id")
+                        if router_model_id is not None:
+                            llm_router.delete_deployment(id=router_model_id)
+                else:
+                    # This is a config model, preserved by the reconcile below
+                    config_models.append(model)
 
-        # Clear only DB-backed auto-router-family entries, keyed by model_name, so the
-        # reload below rebuilds them fresh. A blanket .clear() would also drop config-defined
-        # routers, which are never re-added below (add_deployment only reloads DB models),
-        # leaving them permanently unroutable until a full proxy restart for every tenant.
-        # Restrict to deployments whose model is actually an auto_router/* so a config
-        # router that merely shares a model_name with a regular DB model isn't evicted. The
-        # auto_router/ prefix also covers quality_router/ and adaptive_router/, so pop the
-        # name from every router registry (no-op where absent); missing quality/adaptive
-        # entries would otherwise make init raise "already exists" on reload and abort it.
-        db_router_names: Final = {
-            model.get("model_name")
-            for model in current_models
-            if model.get("model_name") is not None
-            and model.get("model_info", {}).get("db_model", False)
-            and str(model.get("litellm_params", {}).get("model", "")).startswith("auto_router/")
-        }
-        for model_name in db_router_names:
-            llm_router.auto_routers.pop(model_name, None)
-            llm_router.complexity_routers.pop(model_name, None)
-            llm_router.adaptive_routers.pop(model_name, None)
-            llm_router.quality_routers.pop(model_name, None)
+            # ORDINARY db deployments are deliberately NOT wiped. This used to
+            # delete_deployment() every db model before the reload put them back, which
+            # left the router serving ZERO db models for the whole width of the reload
+            # -- a real data-plane hole that every inference request landing in it fell
+            # into. It was also redundant for them: the reload's _delete_deployment
+            # evicts exactly the ids the db no longer lists, and upsert_deployment
+            # pops-and-re-adds a deployment whose params changed while no-opping one
+            # that did not, so the reconcile converges on its own. Every mutation is
+            # visible to that comparison -- `blocked` and (for premium) `updated_at`
+            # are written into model_info.
+            #
+            # AUTO-ROUTER db deployments are the exception and ARE wiped -- in the
+            # classification pass above, together with the strategy entries popped
+            # just below. Their strategy registries are keyed
+            # by model_name, which no deployment-id reconcile touches, so they have to
+            # be popped and rebuilt here. But the rebuild only happens on the ADD path:
+            # Router.upsert_deployment returns early when a deployment is unchanged and
+            # never reaches add_deployment -> _add_deployment ->
+            # init_auto_router_deployment, which is what repopulates the registries.
+            # Popping without deleting would therefore strip every db-backed auto,
+            # complexity, adaptive and quality router on this pod and never put it back,
+            # so ANY unrelated model write would leave them unroutable until a restart.
+            # Deleting the deployment forces upsert down the add path, which rebuilds
+            # both the deployment and its strategy entry.
+            #
+            # That pass restricts the wipe to deployments whose model is actually an
+            # auto_router/* so a config router that merely shares a model_name with a
+            # regular db model isn't evicted -- config routers are never re-added by the
+            # reload (it only reloads db models) and would be permanently unroutable.
+            # The auto_router/ prefix also covers quality_router/ and adaptive_router/,
+            # so pop the name from every registry (no-op where absent); a missing
+            # quality/adaptive entry would otherwise make init raise "already exists"
+            # on reload and abort it.
+            for model_name in db_router_names:
+                llm_router.auto_routers.pop(model_name, None)
+                llm_router.complexity_routers.pop(model_name, None)
+                llm_router.adaptive_routers.pop(model_name, None)
+                llm_router.quality_routers.pop(model_name, None)
 
-        # Reload only DB models
-        still_desired_ids: Final = await proxy_config.add_deployment(
-            prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
-        )
+            # Reload only DB models. _add_deployment_locked, not add_deployment: this
+            # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not
+            # reentrant, so the public wrapper would deadlock against itself.
+            outcome: Final = await proxy_config._add_deployment_locked(
+                prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj
+            )
 
-        verbose_proxy_logger.debug(
-            "Cleared %s DB models, preserved %s config models", len(db_model_ids), len(config_models)
-        )
-        return still_desired_ids
-    except Exception as e:
-        verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e)
-        return None
+            verbose_proxy_logger.debug(
+                "Reconciled %s DB models, preserved %s config models", len(db_model_ids), len(config_models)
+            )
+            return outcome
+        except Exception as e:
+            verbose_proxy_logger.exception("Failed to clear cache and reload models. Due to error - %s", e)
+            return ReconcileOutcome(still_desired=None, live_after=None)

litellm/proxy/proxy_server.py

@@ -467,6 +467,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
     _add_model_to_db,
     _add_team_model_to_db,
     _deduplicate_litellm_router_models,
+    live_model_ids_snapshot,
 )
 from litellm.proxy.management_endpoints.model_management_endpoints import (
     router as model_management_router,
@@ -2166,6 +2167,15 @@ experimental = False
 #### GLOBAL VARIABLES ####
 llm_router: Router | None = None
 llm_model_list: list | None = None
+# Serializes every model reconcile (ProxyConfig.add_deployment and clear_cache) so the
+# read-modify-write of llm_router above is atomic. Without it, two concurrent model
+# writes each reconcile the router against their OWN db snapshot, and the one holding
+# the older snapshot evicts the deployment the newer one just added -- the db keeps the
+# row, this pod stops serving it. Control-plane only (model create/update/delete and
+# the config-sync tick), never on a completion path, so the serialization is free.
+# Module-level rather than per-ProxyConfig because llm_router is a module global and a
+# second ProxyConfig instance must not get its own independent lock over it.
+MODEL_RECONCILE_LOCK: Final = asyncio.Lock()
 general_settings: dict = {}
 config_passthrough_endpoints: list[dict[str, Any]] | None = None
 log_file: Final = "api_log.json"
@@ -6456,16 +6466,37 @@ class ProxyConfig:
         self,
         prisma_client: PrismaClient,
         proxy_logging_obj: ProxyLogging,
-    ) -> frozenset[str] | None:
+    ) -> ReconcileOutcome:
         """
         - Check db for new models
         - Check if model id's in router already
         - If not, add to router
 
-        Returns the ids the db + config say should be served after the reconcile, or
-        None when no reconcile ran. Callers that judge their own reload need it to tell
-        a deliberate eviction from a deployment that went missing.
+        Serialized against every other model reconcile by MODEL_RECONCILE_LOCK, because
+        the work below is a read-modify-write of the shared ``llm_router`` global: it
+        reads the db into a snapshot and then makes the router match that snapshot. Two
+        of those interleaving is not a lost update but an eviction -- the request whose
+        snapshot predates the other's commit reconciles the newer model *out* of the
+        router, since _delete_deployment removes every live deployment absent from the
+        snapshot it was handed. The model stays in the db and this pod stops serving it
+        until some later reload puts it back.
+
+        Returns what the reconcile saw, captured before the lock is released so a
+        caller's verdict cannot be corrupted by the next reconcile's own in-flight
+        window. See ReconcileOutcome.
         """
+        async with MODEL_RECONCILE_LOCK:
+            return await self._add_deployment_locked(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
+
+    async def _add_deployment_locked(
+        self,
+        prisma_client: PrismaClient,
+        proxy_logging_obj: ProxyLogging,
+    ) -> ReconcileOutcome:
+        """add_deployment's body, minus the locking. MODEL_RECONCILE_LOCK MUST already
+        be held. Split out for the one caller that has to hold the lock across more than
+        this reconcile -- clear_cache, which un-serves every db model before calling it
+        and would deadlock on a re-acquire."""
         global llm_router, llm_model_list, master_key, general_settings
 
         still_desired_ids: frozenset[str] | None = None
@@ -6508,7 +6539,12 @@ class ProxyConfig:
         except Exception as e:
             verbose_proxy_logger.exception("litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - %s", e)
 
-        return still_desired_ids
+        # Read while the lock is still held: once it is released the next reconcile can
+        # begin, and clear_cache's leading wipe would make this look like a mass drop.
+        return ReconcileOutcome(
+            still_desired=still_desired_ids,
+            live_after=None if still_desired_ids is None else live_model_ids_snapshot(),
+        )
 
     def start_config_sync_subscriber(
         self,

tests/e2e/ui/fixtures/mock_llm_server/server.py

@@ -3,6 +3,7 @@ Mock LLM server for UI e2e tests.
 Responds to OpenAI-format endpoints with canned responses.
 """
 
+import os
 import time
 import json
 import uuid
@@ -117,4 +118,12 @@ async def embeddings(request: Request):
 
 
 if __name__ == "__main__":
-    uvicorn.run(app, host="127.0.0.1", port=8090)
+    # The port is overridable so two checkouts can run the harness at the same
+    # time; the default keeps every existing caller (run_e2e.sh, the CircleCI
+    # job, the e2e chart's sidecar) working untouched.
+    #
+    # The HOST is deliberately NOT configurable. Binding loopback is what makes
+    # this reachable at 127.0.0.1:8090 from inside the proxy's own pod, which is
+    # the contract the deployed config.yml and the e2e values file are written
+    # against.
+    uvicorn.run(app, host="127.0.0.1", port=int(os.environ.get("MOCK_LLM_PORT", "8090")))

tests/e2e/ui/helpers/mcp.ts

@@ -0,0 +1,65 @@
+import { expect, Page as PwPage } from "@playwright/test";
+import { navigateToPage } from "./navigation";
+import { Page } from "../fixtures/pages";
+import { masterKey } from "./traffic";
+
+/** Creates an MCP server through the UI's discovery to custom-form flow and returns its name. */
+export async function createMcpServer(page: PwPage, url: string): Promise<string> {
+  await navigateToPage(page, Page.McpServers);
+
+  await page.getByRole("button", { name: /Add New MCP Server/i }).click();
+  const discovery = page.getByRole("dialog").filter({ hasText: "Add MCP Server" });
+  await expect(discovery).toBeVisible({ timeout: 5_000 });
+  await discovery.getByRole("button", { name: /Custom Server/i }).click();
+
+  const formModal = page.locator(".ant-modal:visible").filter({ hasText: "MCP Server Name" });
+  await expect(formModal).toBeVisible({ timeout: 5_000 });
+
+  // validateMCPServerName rejects spaces and hyphens; the worker index avoids a same-millisecond collision.
+  const name = `e2e_mcp_${process.env.TEST_WORKER_INDEX ?? "0"}_${Date.now()}`;
+  await formModal.locator('input[id="server_name"]').fill(name);
+
+  const transportField = formModal.locator(".ant-form-item", { hasText: "Transport Type" });
+  await transportField.locator(".ant-select").click();
+  await page.locator(".ant-select-dropdown:visible").getByText("Streamable HTTP").click();
+
+  await formModal.locator('input[id="url"]').fill(url);
+
+  // The auth_type Form.Item has no label prop, so anchor on the enclosing Collapse panel.
+  const authSection = formModal.locator(".ant-collapse-item", { hasText: /^Authentication/ });
+  await authSection.locator(".ant-form-item").first().locator(".ant-select").click();
+  await page.locator(".ant-select-dropdown:visible").getByText("None", { exact: true }).click();
+
+  await formModal.getByRole("button", { name: /^Add MCP Server$/ }).click();
+  await expect(page.getByText("MCP Server created successfully").first()).toBeVisible({ timeout: 15_000 });
+
+  const card = page.getByTestId("mcp-servers-grid").getByText(name).first();
+  await expect(card).toBeVisible({ timeout: 10_000 });
+  return name;
+}
+
+/**
+ * Deletes every server carrying `serverName`. Leaked servers break unrelated MCP specs: the page
+ * reaches out to each one it lists, so unreachable leftovers stall networkidle until it times out.
+ * Errors are swallowed because this runs from afterEach.
+ */
+export async function deleteMcpServerByName(page: PwPage, serverName: string): Promise<void> {
+  const headers = { Authorization: `Bearer ${masterKey()}` };
+  try {
+    const res = await page.request.get("/v1/mcp/server", { headers });
+    if (!res.ok()) return;
+    const servers = (await res.json()) as { server_id: string; server_name?: string }[];
+    for (const server of servers.filter((candidate) => candidate.server_name === serverName)) {
+      await page.request.delete(`/v1/mcp/server/${server.server_id}`, { headers });
+    }
+  } catch {
+    // best effort, see above
+  }
+}
+
+/** Opens a server from the grid and switches to its MCP Tools tab. */
+export async function openMcpToolsTab(page: PwPage, serverName: string): Promise<void> {
+  await page.getByTestId("mcp-servers-grid").getByText(serverName).first().click();
+  await expect(page.getByRole("button", { name: /Back to All Servers/i })).toBeVisible({ timeout: 10_000 });
+  await page.getByRole("tab", { name: "MCP Tools" }).click();
+}

tests/e2e/ui/helpers/playground.ts

@@ -0,0 +1,46 @@
+import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/test";
+import { navigateToPage, dismissFeedbackPopup } from "./navigation";
+import { Page } from "../fixtures/pages";
+
+/** Controls for the Test Key / Playground page, shared with the router-fallback specs. */
+
+/**
+ * The configuration panel is rendered twice, docked and overlay, with one visible at a time.
+ * Every control is narrowed to the visible copy or it trips strict mode against its hidden twin.
+ */
+export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first();
+
+/** The model dropdown, addressed by the placeholder it shows before selection. */
+export const modelSelect = (page: PlaywrightPage): Locator =>
+  onlyVisible(page.locator('.ant-select:has(.ant-select-selection-placeholder:text-is("Select a Model"))'));
+
+/** Send button is icon-only (an up-arrow), so there is no accessible name. */
+export const sendButton = (page: PlaywrightPage): Locator => onlyVisible(page.locator("button:has(.anticon-arrow-up)"));
+
+/** The Virtual Key Source dropdown, addressed by its currently selected label. */
+export const keySourceSelect = (page: PlaywrightPage, current: string): Locator =>
+  onlyVisible(page.locator(`.ant-select:has(.ant-select-selection-item[title="${current}"])`));
+
+export async function openPlayground(page: PlaywrightPage): Promise<void> {
+  await navigateToPage(page, Page.LlmPlayground);
+  await dismissFeedbackPopup(page);
+  await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({
+    timeout: 20_000,
+  });
+}
+
+export async function selectModel(page: PlaywrightPage, model: string): Promise<void> {
+  const select = modelSelect(page);
+  await select.click();
+  // Virtualized: options outside the rendered window are absent from the DOM, so search first.
+  await select.locator("input.ant-select-selection-search-input").fill(model);
+  // antd portals its dropdown to the body; options carry the value as `title`.
+  await onlyVisible(page.locator(`.ant-select-item-option[title="${model}"]`)).click({ timeout: 15_000 });
+}
+
+export async function sendMessage(page: PlaywrightPage, message: string): Promise<void> {
+  const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
+  await expect(input).toBeVisible({ timeout: 15_000 });
+  await input.fill(message);
+  await sendButton(page).click();
+}

tests/e2e/ui/helpers/roundTrip.ts

@@ -0,0 +1,28 @@
+import { expect, Page } from "@playwright/test";
+import { masterKey } from "./traffic";
+
+/**
+ * Runs `action` and returns the parsed body of the first matching request.
+ *
+ * `action` is a callback so the listener is armed before the click; awaiting the
+ * click first lets the request go by, and the test then hangs until timeout.
+ */
+export async function captureRequestBody(
+  page: Page,
+  match: { method: string; urlIncludes: string },
+  action: () => Promise<void>,
+): Promise<Record<string, any>> {
+  const pending = page.waitForRequest((req) => req.method() === match.method && req.url().includes(match.urlIncludes));
+  await action();
+  const request = await pending;
+  return JSON.parse(request.postData() ?? "{}") as Record<string, any>;
+}
+
+/** Reads an endpoint as the master key, so a failure is bad data and not an expired UI token. */
+export async function readBack<T = any>(page: Page, endpoint: string): Promise<T> {
+  const res = await page.request.get(endpoint, {
+    headers: { Authorization: `Bearer ${masterKey()}` },
+  });
+  expect(res.ok(), `GET ${endpoint}`).toBe(true);
+  return (await res.json()) as T;
+}

tests/e2e/ui/helpers/traffic.ts

@@ -0,0 +1,125 @@
+import { APIRequestContext, expect } from "@playwright/test";
+
+/** Model names served by fixtures/config.yml, both backed by the mock LLM server. */
+export const CHAT_MODEL_A = "fake-openai-gpt-4";
+export const CHAT_MODEL_B = "fake-anthropic-claude";
+
+/** The only completion text fixtures/mock_llm_server/server.py ever returns. */
+export const MOCK_RESPONSE_TEXT = "This is a mock response.";
+
+export const masterKey = (): string => process.env.LITELLM_MASTER_KEY || "sk-1234";
+
+const rootPath = (): string => process.env.SERVER_ROOT_PATH ?? "";
+
+interface ChatOptions {
+  model: string;
+  prompt: string;
+  apiKey?: string;
+  /** Sent as `user`, which lands in the spend log's end_user column. */
+  endUser?: string;
+}
+
+/** POST /v1/chat/completions and return the completion id (the Logs Request ID). */
+export async function sendChatCompletion(request: APIRequestContext, opts: ChatOptions): Promise<string> {
+  const res = await request.post(`${rootPath()}/v1/chat/completions`, {
+    headers: {
+      Authorization: `Bearer ${opts.apiKey ?? masterKey()}`,
+      "Content-Type": "application/json",
+    },
+    data: {
+      model: opts.model,
+      messages: [{ role: "user", content: opts.prompt }],
+      ...(opts.endUser ? { user: opts.endUser } : {}),
+    },
+  });
+  expect(res.ok(), `chat completion for ${opts.model} failed (${res.status()}): ${await res.text()}`).toBe(true);
+  const body = await res.json();
+  expect(body.choices?.[0]?.message?.content).toContain(MOCK_RESPONSE_TEXT);
+  return body.id as string;
+}
+
+/** `key` is the sk- value to authenticate with; `token` is its hash, which spend aggregates are keyed by. */
+export async function createVirtualKey(
+  request: APIRequestContext,
+  data: Record<string, unknown> = {},
+): Promise<{ key: string; token: string; alias?: string }> {
+  const res = await request.post(`${rootPath()}/key/generate`, {
+    headers: {
+      Authorization: `Bearer ${masterKey()}`,
+      "Content-Type": "application/json",
+    },
+    data,
+  });
+  expect(res.ok(), `key generate failed (${res.status()}): ${await res.text()}`).toBe(true);
+  const body = await res.json();
+  return {
+    key: body.key as string,
+    token: (body.token ?? body.token_id) as string,
+    alias: body.key_alias as string | undefined,
+  };
+}
+
+/** Spend logs are flushed on a timer, so an assertion straight after a completion races the writer. */
+export async function waitForSpendLog(
+  request: APIRequestContext,
+  requestId: string,
+  timeoutMs = 60_000,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  let lastStatus = 0;
+  while (Date.now() < deadline) {
+    const res = await request.get(`${rootPath()}/spend/logs?request_id=${encodeURIComponent(requestId)}`, {
+      headers: { Authorization: `Bearer ${masterKey()}` },
+    });
+    lastStatus = res.status();
+    if (res.ok()) {
+      const body = await res.json();
+      const rows = Array.isArray(body) ? body : (body?.data ?? []);
+      if (rows.length > 0) {
+        return;
+      }
+    }
+    await new Promise((r) => setTimeout(r, 2_000));
+  }
+  throw new Error(`spend log for request ${requestId} never appeared (last /spend/logs status ${lastStatus})`);
+}
+
+const isoDay = (d: Date): string => d.toISOString().slice(0, 10);
+
+/**
+ * The Usage page reads /user/daily/activity, a rollup written by a background job, and fetches it once
+ * on mount. Navigating before the rollup lands leaves a stale render that never refreshes.
+ */
+export async function waitForKeyInDailyActivity(
+  request: APIRequestContext,
+  keyToken: string,
+  timeoutMs = 120_000,
+): Promise<void> {
+  const now = new Date();
+  const start = new Date(now);
+  start.setDate(start.getDate() - 7);
+  const query = `start_date=${isoDay(start)}&end_date=${isoDay(now)}`;
+
+  const deadline = Date.now() + timeoutMs;
+  let lastStatus = 0;
+  while (Date.now() < deadline) {
+    const res = await request.get(`${rootPath()}/user/daily/activity?${query}`, {
+      headers: { Authorization: `Bearer ${masterKey()}` },
+    });
+    lastStatus = res.status();
+    if (res.ok()) {
+      const body = await res.json();
+      const seen = (body?.results ?? []).some(
+        (day: { breakdown?: { api_keys?: Record<string, unknown> } }) => keyToken in (day.breakdown?.api_keys ?? {}),
+      );
+      if (seen) {
+        return;
+      }
+    }
+    await new Promise((r) => setTimeout(r, 3_000));
+  }
+  throw new Error(
+    `key ${keyToken} never appeared in /user/daily/activity (last status ${lastStatus}); ` +
+      "the daily spend rollup may not be running",
+  );
+}

tests/e2e/ui/run_e2e.sh

@@ -12,6 +12,10 @@ set -euo pipefail
 #   ./run_e2e.sh --repeat-each=5    # Run each test 5 times
 #   ./run_e2e.sh --headed           # Run with browser visible
 #
+# Ports default to 4000 / 5432 / 8090 and can be moved when another checkout
+# already holds them:
+#   PROXY_PORT=4100 POSTGRES_PORT=5532 MOCK_LLM_PORT=8190 ./run_e2e.sh
+#
 # In CI (CI=true), expects:
 #   - PostgreSQL already running on 127.0.0.1:5432
 #   - DATABASE_URL already set
@@ -28,12 +32,50 @@ MOCK_PID=""
 PROXY_PID=""
 PROXY_LOG=""
 
+# Ports, overridable so two checkouts can run this harness at the same time --
+# otherwise a second run aborts on "port 4000 is in use" and the only way out is
+# to stop someone else's stack. Defaults are the historical values, so an unset
+# environment behaves exactly as before (CI, the CircleCI job and the docs all
+# assume 4000/5432/8090).
+PROXY_PORT="${PROXY_PORT:-4000}"
+POSTGRES_PORT="${POSTGRES_PORT:-5432}"
+MOCK_LLM_PORT="${MOCK_LLM_PORT:-8090}"
+export MOCK_LLM_PORT
+
 # --- Ensure common tool paths are available (local dev only) ---
 if [ "$IS_CI" = "false" ]; then
   for p in /usr/local/bin /opt/homebrew/bin "$HOME/.local/bin" /opt/homebrew/opt/postgresql@14/bin /opt/homebrew/opt/libpq/bin; do
     [ -d "$p" ] && export PATH="$p:$PATH"
   done
-  [ -s "$HOME/.nvm/nvm.sh" ] && source "$HOME/.nvm/nvm.sh"
+  # Sourcing nvm only makes `nvm` available -- it leaves you on whatever the
+  # default alias points at, which is frequently an older Node than the
+  # dashboard's engines allow. `npm install` then fails EBADENGINE, npm exits
+  # non-zero, and because the install below is `--silent ... || true` the error
+  # is swallowed and the run dies later with the far less obvious
+  # "sh: next: command not found".
+  #
+  # So select a Node that satisfies ui/litellm-dashboard's engines.node, and if
+  # none is available say so here rather than 200 lines downstream.
+  if [ -s "$HOME/.nvm/nvm.sh" ]; then
+    # shellcheck disable=SC1091
+    source "$HOME/.nvm/nvm.sh"
+    required_major="$(sed -nE 's/.*"node"[[:space:]]*:[[:space:]]*">=?([0-9]+).*/\1/p' \
+      "$DASHBOARD_DIR/package.json" 2>/dev/null | head -1)"
+    if [ -n "$required_major" ]; then
+      current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')"
+      if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then
+        echo "Node $(node --version 2>/dev/null || echo 'not found') is below the dashboard's required v${required_major}; selecting a newer one via nvm"
+        nvm use "$required_major" >/dev/null 2>&1 || nvm use --lts >/dev/null 2>&1 || true
+        current_major="$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/')"
+        if [ -z "$current_major" ] || [ "$current_major" -lt "$required_major" ]; then
+          echo "Error: ui/litellm-dashboard requires Node >= v${required_major}, and no such version is installed."
+          echo "       Install one with:  nvm install ${required_major}"
+          exit 1
+        fi
+      fi
+      echo "Using Node $(node --version) / npm $(npm --version)"
+    fi
+  fi
 fi
 
 # --- Cleanup on exit ---
@@ -47,7 +89,11 @@ cleanup() {
   fi
   echo "Done."
 }
-trap cleanup EXIT INT TERM
+on_signal() {
+  exit 130
+}
+trap cleanup EXIT
+trap on_signal INT TERM
 
 # --- Pre-flight checks ---
 for cmd in python3 npx uv; do
@@ -59,9 +105,14 @@ if [ "$IS_CI" = "false" ]; then
   for cmd in docker psql; do
     command -v "$cmd" >/dev/null 2>&1 || { echo "Error: $cmd not found."; exit 1; }
   done
-  for port in 4000 5432 8090; do
-    if lsof -ti ":$port" >/dev/null 2>&1; then
-      echo "Error: port $port is in use"
+  # Only a LISTENER conflicts with us. Without -sTCP:LISTEN this also matches
+  # ESTABLISHED sockets, so an unrelated *outbound* connection from this machine
+  # to someone else's :5432 (a psql session, a running app, a Prisma engine
+  # talking to a remote database) aborts the run with "port 5432 is in use"
+  # while nothing is actually bound locally.
+  for port in "$PROXY_PORT" "$POSTGRES_PORT" "$MOCK_LLM_PORT"; do
+    if lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1; then
+      echo "Error: port $port is in use (override with PROXY_PORT / POSTGRES_PORT / MOCK_LLM_PORT)"
       exit 1
     fi
   done
@@ -69,12 +120,12 @@ if [ "$IS_CI" = "false" ]; then
   export POSTGRES_USER="e2euser"
   export POSTGRES_PASSWORD="$(openssl rand -hex 32)"
   export POSTGRES_DB="litellm_e2e"
-  export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:5432/${POSTGRES_DB}"
+  export DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_PORT}/${POSTGRES_DB}"
 
   echo "=== Starting PostgreSQL ==="
   docker run -d --rm --name "$CONTAINER_NAME" \
     -e POSTGRES_USER -e POSTGRES_PASSWORD -e POSTGRES_DB \
-    -p 127.0.0.1:5432:5432 \
+    -p "127.0.0.1:${POSTGRES_PORT}:5432" \
     postgres:16
 
   echo "Waiting for PostgreSQL..."
@@ -91,8 +142,13 @@ fi
 
 # --- Credentials ---
 export LITELLM_MASTER_KEY="sk-1234"
-export MOCK_LLM_URL="http://127.0.0.1:8090/v1"
+export MOCK_LLM_URL="http://127.0.0.1:${MOCK_LLM_PORT}/v1"
 export DISABLE_SCHEMA_UPDATE="true"
+# The suite resolves its target from E2E_UI_BASE_URL (constants.ts), which
+# otherwise defaults to :4000 -- so without this a relocated stack would be
+# built and booted correctly and then tested against whatever happens to be
+# listening on the default port.
+export E2E_UI_BASE_URL="${E2E_UI_BASE_URL:-http://127.0.0.1:${PROXY_PORT}}"
 # Ensure the proxy serves UI at /ui (not behind a subpath)
 export SERVER_ROOT_PATH=""
 # Boot with an external logout URL so proxyLogoutUrl.spec.ts can assert the
@@ -108,7 +164,11 @@ export LITELLM_LICENSE="${LITELLM_LICENSE:-}"
 # --- Rebuild UI from source ---
 echo "=== Building UI from source ==="
 cd "$DASHBOARD_DIR"
-npm install --silent 2>/dev/null || true
+# NOT silenced, and NOT `|| true`. Swallowing this is what turns a one-line
+# EBADENGINE ("dashboard requires node >=24, you have v20") into the
+# considerably less helpful "sh: next: command not found" from the build below,
+# because the deps that provide `next` were never installed.
+npm install
 npm run build
 # Copy the fresh build to the proxy's static UI directory
 cp -r "$DASHBOARD_DIR/out/" "$REPO_ROOT/litellm/proxy/_experimental/out/"
@@ -139,7 +199,7 @@ uv run --no-sync python "$SCRIPT_DIR/fixtures/mock_llm_server/server.py" &
 MOCK_PID=$!
 
 for i in $(seq 1 15); do
-  if curl -sf http://127.0.0.1:8090/health >/dev/null 2>&1; then break; fi
+  if curl -sf http://127.0.0.1:${MOCK_LLM_PORT}/health >/dev/null 2>&1; then break; fi
   sleep 1
 done
 
@@ -149,7 +209,7 @@ cd "$REPO_ROOT"
 PROXY_LOG="${TMPDIR:-/tmp}/litellm-e2e-proxy-$$.log"
 uv run --no-sync python -m litellm.proxy.proxy_cli \
   --config "$SCRIPT_DIR/fixtures/config.yml" \
-  --port 4000 >"$PROXY_LOG" 2>&1 &
+  --port "$PROXY_PORT" >"$PROXY_LOG" 2>&1 &
 PROXY_PID=$!
 
 echo "Waiting for proxy (logs: $PROXY_LOG)..."
@@ -160,7 +220,7 @@ for i in $(seq 1 180); do
     tail -n 100 "$PROXY_LOG"
     exit 1
   fi
-  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:4000/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
+  HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:${PROXY_PORT}/health -H "Authorization: Bearer $LITELLM_MASTER_KEY" 2>/dev/null || true)
   if [ "$HTTP_CODE" = "200" ]; then
     PROXY_READY=1
     break
@@ -188,9 +248,38 @@ PGPASSWORD="$DB_PASS" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAM
 # --- Playwright ---
 echo "=== Installing Playwright dependencies ==="
 cd "$SCRIPT_DIR"
-npm install --silent 2>/dev/null || true
+# Same reasoning as the dashboard install above: a failure here means the suite
+# has no @playwright/test, and the run should say that rather than fail later.
+npm install
 npx playwright install chromium --with-deps 2>/dev/null || npx playwright install chromium
 
+# Authoring a new spec means running it over and over against a stack that is
+# already up -- rebuilding the UI and re-seeding for every iteration costs
+# minutes each time. E2E_KEEP_ALIVE brings the stack up, then blocks, so you can
+# run `npx playwright test <spec>` yourself from another shell against it.
+# Ctrl-C here tears everything down through the usual trap.
+if [ "${E2E_KEEP_ALIVE:-0}" = "1" ]; then
+  cat <<EOF
+
+=== Stack is up (E2E_KEEP_ALIVE=1); not running tests ===
+  UI / API : http://127.0.0.1:${PROXY_PORT}
+  Mock LLM : http://127.0.0.1:${MOCK_LLM_PORT}/v1
+  Database : $DATABASE_URL
+  Proxy log: $PROXY_LOG
+
+Run specs against it from $SCRIPT_DIR:
+  npx playwright test --config playwright.config.ts <spec>
+
+Press Ctrl-C to tear the stack down.
+EOF
+  while kill -0 "$PROXY_PID" 2>/dev/null; do
+    sleep 5
+  done
+  echo "Error: proxy process exited unexpectedly. Proxy output:"
+  tail -n 100 "$PROXY_LOG"
+  exit 1
+fi
+
 echo "=== Running Playwright tests ==="
 npx playwright test --config playwright.config.ts "$@"
 EXIT_CODE=$?

tests/e2e/ui/tests/logs/logs.spec.ts

@@ -0,0 +1,221 @@
+import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
+import { Page } from "../../fixtures/pages";
+import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic";
+
+/**
+ * Anchored to traffic this spec generates itself, with a unique prompt and end user per run, so it
+ * neither depends on seeded spend rows nor collides with other specs under parallelism.
+ */
+
+const uniqueSuffix = (): string => `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+
+/**
+ * Walking up from the label is the only stable handle: the header carries no role, test id or class,
+ * and its copy button is icon-only with a hover-only tooltip.
+ */
+const sectionHeader = (drawer: Locator, label: "Input" | "Output"): Locator =>
+  drawer.getByText(label, { exact: true }).locator("xpath=../../..");
+
+/** Every tab stays mounted, so the DOM holds four tables at once; scope to the visible one. */
+const requestLogsRows = (page: PlaywrightPage): Locator =>
+  page.locator("table").filter({ visible: true }).first().locator("tbody tr");
+
+const visibleTestId = (page: PlaywrightPage, id: string): Locator => page.getByTestId(id).filter({ visible: true });
+
+/** Open the Logs page and filter the table down to a single request id. */
+async function openLogsForRequest(page: PlaywrightPage, requestId: string): Promise<Locator> {
+  await navigateToPage(page, Page.Logs);
+  await dismissFeedbackPopup(page);
+
+  const search = visibleTestId(page, "datatable-search");
+  await expect(search).toBeVisible({ timeout: 20_000 });
+  await search.fill(requestId);
+
+  const row = requestLogsRows(page).filter({ hasText: requestId });
+  await expect(row, `no logs row for request ${requestId}`).toHaveCount(1, {
+    timeout: 30_000,
+  });
+  return row;
+}
+
+test.describe("Logs page", () => {
+  test.use({
+    storageState: ADMIN_STORAGE_PATH,
+    // The copy buttons go through navigator.clipboard, which rejects without these.
+    permissions: ["clipboard-read", "clipboard-write"],
+  });
+
+  test("a served request expands to its request and response", async ({ page, request }) => {
+    const prompt = `logs-detail-prompt-${uniqueSuffix()}`;
+    const requestId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt,
+    });
+    await waitForSpendLog(request, requestId);
+
+    const row = await openLogsForRequest(page, requestId);
+
+    // Expand: clicking the row opens the detail drawer for that request.
+    await row.click();
+    const drawer = page.locator(".ant-drawer-content").first();
+    await expect(drawer).toBeVisible({ timeout: 20_000 });
+    await expect(drawer.getByText("Request & Response")).toBeVisible({
+      timeout: 20_000,
+    });
+
+    // The prompt we sent and the mock server's reply are both rendered.
+    await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({
+      timeout: 20_000,
+    });
+    await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 20_000 });
+  });
+
+  // Split out because only the copy path needs a secure context; folding it in would
+  // take the drawer-rendering coverage down with it.
+  test("the drawer copies the request and the response to the clipboard", async ({ page, request }) => {
+    // `navigator.clipboard` is undefined outside a secure context, and handleCopy calls
+    // writeText unguarded, so on plain HTTP served from a hostname the click throws and no
+    // toast renders. Skipped rather than weakened so the product gap stays visible.
+    await page.goto("/ui");
+    const isSecure = await page.evaluate(() => window.isSecureContext);
+    test.skip(!isSecure, "origin is not a secure context, so navigator.clipboard is unavailable");
+
+    const prompt = `logs-copy-prompt-${uniqueSuffix()}`;
+    const requestId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt,
+    });
+    await waitForSpendLog(request, requestId);
+
+    const row = await openLogsForRequest(page, requestId);
+    await row.click();
+    const drawer = page.locator(".ant-drawer-content").first();
+    await expect(drawer).toBeVisible({ timeout: 20_000 });
+
+    // Copy request: the Input card's copy button puts the prompt on the clipboard.
+    await sectionHeader(drawer, "Input").getByRole("button").click();
+    await expect(page.getByText("Input copied")).toBeVisible({
+      timeout: 10_000,
+    });
+    expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(prompt);
+
+    // Copy response: the Output card's copy button puts the completion on it.
+    await sectionHeader(drawer, "Output").getByRole("button").click();
+    await expect(page.getByText("Output copied")).toBeVisible({
+      timeout: 10_000,
+    });
+    expect(await page.evaluate(() => navigator.clipboard.readText())).toContain(MOCK_RESPONSE_TEXT);
+  });
+
+  test("the Input card collapses and expands", async ({ page, request }) => {
+    const prompt = `logs-collapse-prompt-${uniqueSuffix()}`;
+    const requestId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt,
+    });
+    await waitForSpendLog(request, requestId);
+
+    const row = await openLogsForRequest(page, requestId);
+    await row.click();
+
+    const drawer = page.locator(".ant-drawer-content").first();
+    await expect(drawer.getByText("Request & Response")).toBeVisible({
+      timeout: 20_000,
+    });
+
+    // The body collapses via `max-height: 0; overflow: hidden`, which zeroes its own bounding
+    // box, so the wrapper reads as hidden while the clipped text node inside it does not.
+    const header = sectionHeader(drawer, "Input");
+    const body = header.locator("xpath=following-sibling::div[1]");
+    await expect(header.locator(".anticon-up")).toBeVisible();
+    await expect(body).toBeVisible();
+
+    await header.click();
+    await expect(header.locator(".anticon-down")).toBeVisible({
+      timeout: 10_000,
+    });
+    await expect(body).toBeHidden({ timeout: 10_000 });
+
+    await header.click();
+    await expect(header.locator(".anticon-up")).toBeVisible({
+      timeout: 10_000,
+    });
+    await expect(body).toBeVisible({ timeout: 10_000 });
+    await expect(drawer.getByText(prompt, { exact: false })).toBeVisible({
+      timeout: 10_000,
+    });
+  });
+
+  test("the JSON view exposes Request and Response tabs", async ({ page, request }) => {
+    const prompt = `logs-json-prompt-${uniqueSuffix()}`;
+    const requestId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt,
+    });
+    await waitForSpendLog(request, requestId);
+
+    const row = await openLogsForRequest(page, requestId);
+    await row.click();
+
+    const drawer = page.locator(".ant-drawer-content").first();
+    await expect(drawer.getByText("Request & Response")).toBeVisible({
+      timeout: 20_000,
+    });
+
+    // antd Radio.Button hides the <input> under its <label>, which intercepts
+    // the pointer event — click the label, not the radio.
+    await drawer.locator("label.ant-radio-button-wrapper").filter({ hasText: "JSON" }).click();
+
+    const requestTab = drawer.getByRole("tab", { name: "Request" });
+    await expect(requestTab).toBeVisible({ timeout: 10_000 });
+    await requestTab.click();
+    await expect(drawer.getByText(prompt, { exact: false }).first()).toBeVisible({ timeout: 10_000 });
+
+    await drawer.getByRole("tab", { name: "Response" }).click();
+    await expect(drawer.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 10_000 });
+  });
+
+  test("the End User filter narrows the table to that customer", async ({ page, request }) => {
+    const endUser = `logs-end-user-${uniqueSuffix()}`;
+    const minePrompt = `logs-filter-mine-${uniqueSuffix()}`;
+    const otherPrompt = `logs-filter-other-${uniqueSuffix()}`;
+
+    const mineId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt: minePrompt,
+      endUser,
+    });
+    const otherId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt: otherPrompt,
+    });
+    await waitForSpendLog(request, mineId);
+    await waitForSpendLog(request, otherId);
+
+    await navigateToPage(page, Page.Logs);
+    await dismissFeedbackPopup(page);
+
+    // Both requests are in the unfiltered table.
+    await expect(requestLogsRows(page).filter({ hasText: mineId })).toHaveCount(1, { timeout: 30_000 });
+    await expect(requestLogsRows(page).filter({ hasText: otherId })).toHaveCount(1, { timeout: 30_000 });
+
+    await visibleTestId(page, "datatable-filters-trigger").click();
+    const filters = page.getByRole("dialog").filter({ hasText: "Narrow down request logs" });
+    await expect(filters).toBeVisible({ timeout: 10_000 });
+
+    const endUserInput = filters.getByPlaceholder("Search an end user");
+    await endUserInput.click();
+    await endUserInput.fill(endUser);
+    // The combobox popup is portaled to the body, so it is outside the filter
+    // dialog's subtree — scope the option lookup to the page, not the dialog.
+    await page.getByRole("option", { name: endUser, exact: true }).click({ timeout: 30_000 });
+    await filters.getByRole("button", { name: "Apply Filters" }).click();
+
+    // Only the request tagged with this end user survives the filter.
+    await expect(requestLogsRows(page).filter({ hasText: otherId })).toHaveCount(0, { timeout: 30_000 });
+    await expect(requestLogsRows(page).filter({ hasText: mineId })).toHaveCount(1);
+    await expect(requestLogsRows(page)).toHaveCount(1);
+  });
+});

tests/e2e/ui/tests/mcp/mcpServerEdit.spec.ts

@@ -0,0 +1,92 @@
+import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { createMcpServer, deleteMcpServerByName } from "../../helpers/mcp";
+import { captureRequestBody, readBack } from "../../helpers/roundTrip";
+
+/**
+ * Editing and deleting an MCP server, verified against the API. The reported failures are all on
+ * this side: renames that need repeating, deletes that need two attempts, each toasting success on
+ * the failing attempt. The URL is unreachable on purpose; only persistence is under test here.
+ */
+const UNREACHABLE_URL = "https://e2e-fake-mcp.test.local/mcp";
+
+/** GET /v1/mcp/server returns a bare array of servers (useMCPServers types it MCPServer[]). */
+async function findServerByName(page: PlaywrightPage, serverName: string): Promise<Record<string, any> | undefined> {
+  const servers = await readBack<Record<string, any>[]>(page, "/v1/mcp/server");
+  return servers.find((server) => server.server_name === serverName);
+}
+
+test.describe("MCP Servers - edit and delete", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  let serverName: string;
+
+  test.beforeEach(async ({ page }) => {
+    serverName = await createMcpServer(page, UNREACHABLE_URL);
+  });
+
+  // The rename test leaves an unreachable server behind, which slows the MCP page for later tests.
+  test.afterEach(async ({ page }) => {
+    await deleteMcpServerByName(page, serverName);
+  });
+
+  test("Renaming a server's alias persists", async ({ page }) => {
+    const before = await findServerByName(page, serverName);
+    expect(before, `created server ${serverName} readable from /v1/mcp/server`).toBeTruthy();
+
+    await page.getByTestId("mcp-servers-grid").getByText(serverName).first().click();
+    await expect(page.getByRole("button", { name: /Back to All Servers/i })).toBeVisible({ timeout: 10_000 });
+
+    // exact: the server view also renders a "Network Settings" tab.
+    await page.getByRole("tab", { name: "Settings", exact: true }).click();
+
+    // A card click may land straight in edit mode, so only click the button when it rendered.
+    const editSettings = page.getByRole("button", { name: "Edit Settings" });
+    if (await editSettings.isVisible().catch(() => false)) {
+      await editSettings.click();
+    }
+
+    // The create modal stays mounted behind the view with its own #alias and Save.
+    const settingsPanel = page.getByRole("tabpanel", { name: "Settings" });
+
+    const newAlias = `${serverName}_renamed`;
+    const aliasInput = settingsPanel.locator('input[id="alias"]');
+    await expect(aliasInput).toBeVisible({ timeout: 10_000 });
+    await aliasInput.fill(newAlias);
+
+    const update = await captureRequestBody(page, { method: "PUT", urlIncludes: "/v1/mcp/server" }, async () => {
+      await settingsPanel.getByRole("button", { name: "Save Changes" }).click();
+    });
+    expect(update.alias, "new alias on the wire").toBe(newAlias);
+    // An unidentified target is one way a save succeeds and changes nothing.
+    expect(update.server_id, "update targets the server being edited").toBe(before?.server_id);
+
+    // The reported symptom is a first save that returns success and does not stick.
+    await expect
+      .poll(async () => (await findServerByName(page, serverName))?.alias, {
+        message: `alias for ${serverName} did not persist after one save`,
+        timeout: 15_000,
+      })
+      .toBe(newAlias);
+  });
+
+  test("Deleting a server removes it", async ({ page }) => {
+    expect(await findServerByName(page, serverName), `created server ${serverName} exists`).toBeTruthy();
+
+    const card = page.getByTestId("mcp-servers-grid").locator("div").filter({ hasText: serverName }).first();
+    await card.getByRole("button", { name: "Server actions" }).click();
+    await page.getByRole("menuitem", { name: "Delete" }).click();
+
+    const dialog = page.getByRole("alertdialog");
+    await expect(dialog.getByText("Delete MCP Server?")).toBeVisible({ timeout: 5_000 });
+    await dialog.getByRole("button", { name: "Delete", exact: true }).click();
+
+    // One attempt has to be enough; the report is a delete that needs two.
+    await expect
+      .poll(async () => await findServerByName(page, serverName), {
+        message: `server ${serverName} still present after one delete`,
+        timeout: 15_000,
+      })
+      .toBeUndefined();
+  });
+});

tests/e2e/ui/tests/mcp/mcpServers.spec.ts

@@ -2,6 +2,7 @@ import { test, expect } from "@playwright/test";
 import { ADMIN_STORAGE_PATH } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
+import { deleteMcpServerByName } from "../../helpers/mcp";
 
 // Coverage scope: only the happy-path Streamable HTTP + None auth create flow.
 // See E2E_COVERAGE.md (#29 row) for the full list of uncovered MCP surfaces
@@ -11,6 +12,15 @@ import { Page } from "../../fixtures/pages";
 test.describe("MCP Servers", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
+  let createdServerName = "";
+
+  // The server this test creates is unreachable, and the MCP page contacts
+  // every server it lists, so leaving it behind slows down every later MCP
+  // test. See deleteMcpServerByName for what that actually cost.
+  test.afterEach(async ({ page }) => {
+    if (createdServerName) await deleteMcpServerByName(page, createdServerName);
+  });
+
   test("Add a custom MCP server via the discovery → custom form", async ({ page }) => {
     await navigateToPage(page, Page.McpServers);
 
@@ -25,6 +35,7 @@ test.describe("MCP Servers", () => {
 
     // Name — no spaces or hyphens per validateMCPServerName
     const uniqueName = `e2e_mcp_${Date.now()}`;
+    createdServerName = uniqueName;
     await formModal.locator('input[id="server_name"]').fill(uniqueName);
 
     // Transport: Streamable HTTP — the only value the proxy actually accepts is "http"
@@ -48,8 +59,6 @@ test.describe("MCP Servers", () => {
     // Submit
     await formModal.getByRole("button", { name: /^Add MCP Server$/ }).click();
 
-    // No teardown needed — the e2e runner spins up a fresh DB per invocation.
-
     // Success toast and the new card in the server grid. Scope the lookup to
     // the MCP servers grid so the form modal's `server_name` input — which
     // still holds the timestamped value during its close animation — can't

tests/e2e/ui/tests/mcp/mcpTools.spec.ts

@@ -0,0 +1,80 @@
+import { test, expect, Locator } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { createMcpServer, deleteMcpServerByName, openMcpToolsTab } from "../../helpers/mcp";
+
+// Listing and calling MCP tools, which needs a server that really answers; the create-only spec
+// points at an unreachable URL on purpose.
+//
+// This spec makes a read-only network call to DeepWiki's public MCP server, from the proxy rather
+// than the browser. It needs no credentials, so there is no secret to leak from a public repo.
+//
+// A DeepWiki outage turns this red for something that is not a litellm regression. That is left
+// visible rather than auto-skipped: skipping on connection trouble also skips when the proxy's own
+// MCP client breaks, which is the regression this exists to catch. E2E_SKIP_EXTERNAL_MCP=1 opts out.
+const MCP_SERVER_URL = "https://mcp.deepwiki.com/mcp";
+const TOOL_NAME = "read_wiki_structure";
+const TOOL_ARG_REPO = "BerriAI/litellm";
+
+// Match the h4 heading, not page text: a tool whose description names another tool trips strict mode.
+const toolCard = (list: Locator, name: string): Locator =>
+  list.locator("h4.font-mono").filter({ hasText: new RegExp(`^${name}$`) });
+
+test.describe("MCP Tools", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+  test.skip(!!process.env.E2E_SKIP_EXTERNAL_MCP, "E2E_SKIP_EXTERNAL_MCP is set");
+
+  let serverName: string;
+
+  test.beforeEach(async ({ page }) => {
+    serverName = await createMcpServer(page, MCP_SERVER_URL);
+    await openMcpToolsTab(page, serverName);
+  });
+
+  // The MCP page contacts every server it lists, so leaks slow later tests run by run.
+  test.afterEach(async ({ page }) => {
+    await deleteMcpServerByName(page, serverName);
+  });
+
+  test("MCP Tools tab lists the tools the upstream server advertises", async ({ page }) => {
+    // Fetched through the proxy on mount, so allow for a cold upstream connection.
+    const toolList = page.locator(".mcp-tools-scrollable");
+    await expect(toolList).toBeVisible({ timeout: 30_000 });
+
+    // Non-empty would still pass if the proxy returned some other server's tools.
+    await expect(toolCard(toolList, TOOL_NAME)).toBeVisible();
+    await expect(toolCard(toolList, "ask_question")).toBeVisible();
+    await expect(toolCard(toolList, "read_wiki_contents")).toBeVisible();
+
+    // No other tool's name or description contains this string, so exactly one card survives.
+    await page.getByPlaceholder("Search tools...").fill(TOOL_NAME);
+    await expect(toolList.locator("h4.font-mono")).toHaveCount(1);
+    await expect(toolCard(toolList, TOOL_NAME)).toBeVisible();
+  });
+
+  test("Calling a tool from the Test Tool panel returns the upstream result", async ({ page }) => {
+    const toolList = page.locator(".mcp-tools-scrollable");
+    await expect(toolList).toBeVisible({ timeout: 30_000 });
+
+    await toolCard(toolList, TOOL_NAME).click();
+
+    // Selecting a tool swaps the right-hand pane in for the empty state.
+    await expect(page.getByText("Test Tool:", { exact: true })).toBeVisible({ timeout: 10_000 });
+    await expect(page.getByText("Ready to Call Tool")).toBeVisible();
+
+    // The form is generated from the tool's inputSchema, so `repoName` proves the schema
+    // round-tripped through the proxy instead of the panel falling back to a generic field.
+    const repoInput = page.locator('input[id="repoName"]');
+    await expect(repoInput).toBeVisible();
+    await repoInput.fill(TOOL_ARG_REPO);
+
+    await page.getByRole("button", { name: "Call Tool", exact: true }).click();
+
+    await expect(page.getByText("Tool executed successfully")).toBeVisible({ timeout: 60_000 });
+    // read_wiki_structure answers with the repo's outline, so the pane must name the repo.
+    await expect(page.getByText(TOOL_ARG_REPO).first()).toBeVisible();
+
+    // A second call is offered rather than the button resetting to its
+    // first-run label.
+    await expect(page.getByRole("button", { name: "Call Again", exact: true })).toBeVisible();
+  });
+});

tests/e2e/ui/tests/modelsPage/addModel.spec.ts

@@ -1,8 +1,25 @@
-import { test, expect } from "@playwright/test";
+import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
 import { ADMIN_STORAGE_PATH, E2E_TEAM_CRUD_ID } from "../../constants";
 import { Role, users } from "../../fixtures/users";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
+import { captureRequestBody, readBack } from "../../helpers/roundTrip";
+import { sendChatCompletion } from "../../helpers/traffic";
+
+/** The mock LLM as the proxy reaches it: same host locally, a sidecar in the deployed stack. */
+const MOCK_LLM_BASE = `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`;
+
+/** GET /model/info?litellm_model_id= returns {data: [row]}, the deployment as stored. */
+async function readDeployment(page: PlaywrightPage, modelId: string): Promise<Record<string, any> | undefined> {
+  const body = await readBack<{ data: Record<string, any>[] }>(page, `/model/info?litellm_model_id=${modelId}`);
+  return body.data[0];
+}
+
+/** GET /v2/model/info lists every deployment; created models are found by model_name. */
+async function findDeploymentByName(page: PlaywrightPage, modelName: string): Promise<Record<string, any> | undefined> {
+  const body = await readBack<{ data: Record<string, any>[] }>(page, "/v2/model/info");
+  return body.data.find((row) => row.model_name === modelName);
+}
 
 /**
  * Helper to select a provider from the Add Model form dropdown.
@@ -18,6 +35,28 @@ async function selectProvider(page: any, providerName: string) {
 test.describe("Add Model", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
 
+  // Set by the UI-add test below. The deployed stack keeps its database, so a leak
+  // pollutes every later Models table and readback.
+  let uiAddedModelName = "";
+
+  test.afterEach(async ({ page }) => {
+    if (!uiAddedModelName) return;
+    const name = uiAddedModelName;
+    uiAddedModelName = "";
+    try {
+      const stored = await findDeploymentByName(page, name);
+      const id = stored?.model_info?.id;
+      if (id) {
+        await page.request.post("/model/delete", {
+          headers: { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` },
+          data: { id },
+        });
+      }
+    } catch {
+      // Teardown must never turn a passing test red or mask a real failure.
+    }
+  });
+
   test("Able to see all models for a specific provider in the model dropdown", async ({ page }) => {
     await navigateToPage(page, Page.Models);
     await page.getByRole("tab", { name: "Add Model" }).click();
@@ -37,15 +76,14 @@ test.describe("Add Model", () => {
     const modelName = `e2e-team-model-${Date.now()}`;
 
     // Create a team-scoped model via API so the test has something to edit.
-    // The e2e runner spins up a fresh postgres container per invocation, so
-    // there's no cleanup step — the DB is thrown away at the end of the run.
     const createResponse = await page.request.post("/model/new", {
       headers: { Authorization: `Bearer ${masterKey}` },
       data: {
         model_name: modelName,
         litellm_params: {
           model: "openai/fake-gpt-4",
-          api_base: "http://127.0.0.1:8090/v1",
+          // Never called, but the port moves when two checkouts run side by side.
+          api_base: `http://127.0.0.1:${process.env.MOCK_LLM_PORT ?? "8090"}/v1`,
           api_key: "fake-key",
           tpm: 100,
           rpm: 200,
@@ -55,7 +93,10 @@ test.describe("Add Model", () => {
         },
       },
     });
-    expect(createResponse.ok()).toBe(true);
+    // A bare toBe(true) sends you looking at the UI for a setup call that never landed.
+    expect(createResponse.ok(), `/model/new failed: ${createResponse.status()} ${await createResponse.text()}`).toBe(
+      true,
+    );
     const createdModelId = (await createResponse.json()).model_info?.id;
     expect(createdModelId, "model id from /model/new").toBeTruthy();
 
@@ -76,11 +117,93 @@ test.describe("Add Model", () => {
     await page.getByPlaceholder("Enter TPM").fill("999");
     await page.getByPlaceholder("Enter RPM").fill("888");
 
-    await page.getByRole("button", { name: "Save Changes" }).click();
+    // handleModelUpdate PATCHes the whole litellm_params blob, so pin what goes on the wire.
+    const patch = await captureRequestBody(
+      page,
+      { method: "PATCH", urlIncludes: `/model/${createdModelId}/update` },
+      async () => {
+        await page.getByRole("button", { name: "Save Changes" }).click();
+      },
+    );
+    expect(Number(patch.litellm_params?.tpm), "new TPM on the wire").toBe(999);
+    expect(Number(patch.litellm_params?.rpm), "new RPM on the wire").toBe(888);
 
     // Verify the new values render back in view mode
     await expect(page.getByText("999", { exact: true })).toBeVisible({ timeout: 10_000 });
     await expect(page.getByText("888", { exact: true })).toBeVisible({ timeout: 10_000 });
+
+    // View mode re-renders from the form's own state, so read the deployment back.
+    await expect
+      .poll(
+        async () => {
+          const stored = await readDeployment(page, createdModelId);
+          return [Number(stored?.litellm_params?.tpm), Number(stored?.litellm_params?.rpm)];
+        },
+        { message: "TPM/RPM did not persist on the deployment", timeout: 15_000 },
+      )
+      .toEqual([999, 888]);
+
+    // Pin the fields this edit had no business changing; dropping them looks identical in the UI.
+    const after = await readDeployment(page, createdModelId);
+    expect(after?.litellm_params?.model, "upstream model untouched by a limits edit").toBe("openai/fake-gpt-4");
+    expect(after?.model_info?.team_id, "team ownership untouched by a limits edit").toBe(E2E_TEAM_CRUD_ID);
+  });
+
+  test("Add a model through the UI, pass Test Connect, and serve traffic with it", async ({ page, request }) => {
+    // Every other test here stops at "the row appears", which an unroutable model also does.
+    // OpenAI-Compatible exposes API Base, so this points at the mock LLM and needs no credential.
+    await navigateToPage(page, Page.Models);
+    await page.getByRole("tab", { name: "Add Model" }).click();
+
+    // Labels come from /public/providers/fields, not the frontend Providers enum, and the two differ.
+    await selectProvider(page, "OpenAI-Compatible Endpoints");
+
+    const publicName = `e2e-ui-added-${Date.now()}`;
+    uiAddedModelName = publicName;
+
+    // The model picker's "custom" entry reveals the free-text name field.
+    await page.locator(".ant-select-selection-overflow").first().click();
+    await page.locator(".ant-select-dropdown:visible").getByText("Custom Model Name (Enter below)").click();
+    await page.keyboard.press("Escape");
+    await page.getByPlaceholder("Enter custom model name").fill(publicName);
+
+    // By Form.Item id, not placeholder: placeholders change with the provider selection.
+    await page.locator("#api_base").fill(MOCK_LLM_BASE);
+    await page.locator("#api_key").fill("fake-key");
+
+    await page.getByRole("button", { name: "Test Connect" }).click();
+    await expect(page.getByText("Connection Test Results")).toBeVisible({ timeout: 10_000 });
+    // Assert the success panel is present; "no failure yet" is also true mid-flight.
+    await expect(page.getByTestId("connection-success-msg")).toBeVisible({ timeout: 30_000 });
+
+    // The modal swallows the Add click. Scope to the footer: the dismiss X is also named "Close".
+    const resultsModal = page.locator(".ant-modal:visible").filter({ hasText: "Connection Test Results" });
+    await resultsModal.locator(".ant-modal-footer").getByRole("button", { name: "Close" }).click();
+    await expect(resultsModal).toBeHidden({ timeout: 5_000 });
+
+    const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
+      await page.getByRole("button", { name: "Add Model" }).last().click();
+    });
+    expect(created.model_name, "the model is created under the name that was typed").toBe(publicName);
+    expect(created.litellm_params?.api_base, "the api base survives the form").toBe(MOCK_LLM_BASE);
+
+    await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 });
+
+    // Serving one request is the only assertion that rules out a dropped api_base or an
+    // unregistered name. Polled because /model/new returns before the router reloads.
+    await expect
+      .poll(
+        async () => {
+          try {
+            await sendChatCompletion(request, { model: publicName, prompt: `hello from ${publicName}` });
+            return true;
+          } catch {
+            return false;
+          }
+        },
+        { message: `model ${publicName} was added through the UI but never served a request`, timeout: 30_000 },
+      )
+      .toBe(true);
   });
 
   test("Test connection with bad credentials shows failure", async ({ page }) => {
@@ -126,7 +249,13 @@ test.describe("Add Model", () => {
     await apiKeyInput.fill("sk-any-key-for-add-test");
 
     // Click Add Model button by its text
-    await page.getByRole("button", { name: "Add Model" }).last().click();
+    const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
+      await page.getByRole("button", { name: "Add Model" }).last().click();
+    });
+    // The form sends custom_llm_provider separately from the name, so both halves have to arrive.
+    expect(created.model_name, "the selected model is what goes on the wire").toBe("claude-haiku-4-5");
+    expect(created.litellm_params?.model, "the model name goes on the wire").toBe("claude-haiku-4-5");
+    expect(created.litellm_params?.custom_llm_provider, "the picked provider goes on the wire").toBe("anthropic");
 
     // Wait for success notification
     await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 });
@@ -148,19 +277,20 @@ test.describe("Add Model", () => {
     // Verify the model name appears in the table body
     const tableBody = page.locator("table tbody");
     await expect(tableBody.getByText("claude-haiku-4-5").first()).toBeVisible({ timeout: 15_000 });
+
+    // A row proves the name is there, not what the deployment routes to.
+    const stored = await findDeploymentByName(page, "claude-haiku-4-5");
+    expect(stored, "created model readable from /v2/model/info").toBeTruthy();
+    expect(stored?.litellm_params?.model, "stored deployment keeps the model name").toBe("claude-haiku-4-5");
+    expect(stored?.litellm_params?.custom_llm_provider, "stored deployment keeps its provider").toBe("anthropic");
   });
 
   test("Add team-only model via Team-BYOK toggle and verify it appears with the team", async ({ page, request }) => {
-    // The Team-BYOK switch is gated on `premiumUser` — without a license set
-    // for the proxy under test, the toggle is disabled and this manual-QA
-    // step cannot be exercised.
+    // The Team-BYOK switch is gated on premiumUser; without a license the toggle is disabled.
     test.skip(!process.env.LITELLM_LICENSE, "LITELLM_LICENSE not set in test env — Team-BYOK switch is disabled");
 
-    // Make the test idempotent across retries and local reruns: delete any
-    // Cohere model already scoped to the e2e team before we start, and again
-    // after we finish. The sibling "Add wildcard route" test creates a
-    // team-less Cohere wildcard, so we only target rows that have BOTH the
-    // cohere/* model_name AND team_id == e2e-team-crud.
+    // Idempotent across reruns. Only target rows with both the cohere name and the e2e team,
+    // so the sibling wildcard test's team-less model is left alone.
     const masterKey = users[Role.ProxyAdmin].password;
     const auth = { Authorization: `Bearer ${masterKey}` };
     const deleteTeamScopedCohereModels = async () => {
@@ -198,11 +328,7 @@ test.describe("Add Model", () => {
       const teamByokRow = page.locator(".ant-form-item", { hasText: "Team-BYOK Model" });
       await teamByokRow.getByRole("switch").click();
 
-      // The Team dropdown appears underneath once the switch is on. TeamDropdown
-      // renders its Select.Option children with custom <span>/<Text> markup, so
-      // the popup items don't carry role="option" — match by text content,
-      // scoped to the visible dropdown so a stale tag elsewhere in the form
-      // can't satisfy it.
+      // TeamDropdown's options carry custom markup and no role="option", so match by text.
       const teamDropdown = page.getByTestId("team-dropdown");
       await expect(teamDropdown).toBeVisible({ timeout: 5_000 });
       await teamDropdown.click();
@@ -212,36 +338,27 @@ test.describe("Add Model", () => {
 
       await page.getByRole("button", { name: "Add Model" }).last().click();
 
-      // Scope the success toast to antd's notification container so a stale
-      // success message from an earlier test in the same context can't satisfy
-      // the assertion.
+      // Scope to antd's notification container so a stale toast can't satisfy this.
       await expect(page.locator(".ant-notification").getByText("created successfully").last()).toBeVisible({
         timeout: 15_000,
       });
 
-      // Verify the model is now in All Models with the team_id attached. The
-      // Models table renders team-scoped models with the team id in the row.
+      // The Models table renders team-scoped models with the team id in the row.
       await page.getByRole("tab", { name: "All Models" }).click();
       await page.waitForLoadState("networkidle");
-      // Match the sibling tests in this file — networkidle fires before the
-      // table finishes re-rendering, so give it the same 2s settle before
-      // searching.
+      // networkidle fires before the table finishes re-rendering.
       await page.waitForTimeout(2000);
 
       await page.getByPlaceholder("Search model names").fill("cohere");
       await page.waitForTimeout(1000);
 
-      // Confirm the search returned at least one result — gives a clear
-      // failure message when the table is empty instead of timing out on a
-      // row assertion.
+      // Clearer failure than timing out on a row assertion when the table is empty.
       await expect(page.getByTestId("pagination-range")).toHaveText(/Showing \d+-\d+ of \d+/, {
         timeout: 15_000,
       });
 
-      // Stronger than "the team appears somewhere in tbody" — pin the assertion
-      // to a single row that has BOTH the cohere model_name AND the seeded
-      // team, so a stale cohere row from "Add wildcard route" (no team) can't
-      // satisfy the check. The Team ID column renders the id, not the alias.
+      // Pin to one row carrying both the name and the team, so the sibling test's
+      // team-less cohere row can't satisfy it.
       const teamCohereRow = page
         .locator("table tbody tr")
         .filter({ hasText: "cohere/" })
@@ -270,7 +387,11 @@ test.describe("Add Model", () => {
     await apiKeyInput.fill("sk-any-key-for-wildcard-test");
 
     // Click Add Model button by its text
-    await page.getByRole("button", { name: "Add Model" }).last().click();
+    const created = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
+      await page.getByRole("button", { name: "Add Model" }).last().click();
+    });
+    // A wildcard with the star stripped becomes a plain "cohere" deployment that matches nothing.
+    expect(created.model_name, "the wildcard route goes on the wire intact").toBe("cohere/*");
 
     // Wait for success notification
     await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 });
@@ -292,5 +413,10 @@ test.describe("Add Model", () => {
     // Verify the wildcard model appears in the table body (wildcard models show as "cohere/*")
     const tableBody = page.locator("table tbody");
     await expect(tableBody.getByText("cohere/").first()).toBeVisible({ timeout: 15_000 });
+
+    // "cohere/" in the table also matches a plain cohere deployment; require the wildcard exactly.
+    const stored = await findDeploymentByName(page, "cohere/*");
+    expect(stored, "wildcard deployment readable from /v2/model/info").toBeTruthy();
+    expect(stored?.litellm_params?.model, "stored deployment keeps the wildcard route").toBe("cohere/*");
   });
 });

tests/e2e/ui/tests/playground/playground.spec.ts

@@ -0,0 +1,50 @@
+import { test, expect } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { CHAT_MODEL_A, CHAT_MODEL_B, MOCK_RESPONSE_TEXT, createVirtualKey } from "../../helpers/traffic";
+import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground";
+
+/**
+ * The one flow that exercises the dashboard's own LLM call path rather than an admin CRUD endpoint,
+ * so it covers the UI's auth header, endpoint selection and streaming render.
+ */
+test.describe("Playground", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  for (const model of [CHAT_MODEL_A, CHAT_MODEL_B]) {
+    test(`chats with ${model} using the current UI session`, async ({ page }) => {
+      await openPlayground(page);
+
+      // "Current UI Session" is the default: the logged-in admin's key, nothing pasted.
+      await expect(onlyVisible(page.getByTitle("Current UI Session"))).toBeVisible();
+
+      await selectModel(page, model);
+      const prompt = `playground ping for ${model}`;
+      await sendMessage(page, prompt);
+
+      // Our prompt is echoed into the transcript, and the mock server replies.
+      await expect(page.getByText(prompt, { exact: false }).first()).toBeVisible({ timeout: 20_000 });
+      await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 });
+    });
+  }
+
+  test("chats using a pasted virtual key instead of the UI session", async ({ page, request }) => {
+    const { key } = await createVirtualKey(request, {
+      key_alias: `e2e-playground-${Date.now()}`,
+    });
+
+    await openPlayground(page);
+
+    // Switch the source to "Virtual Key" and paste the key we just minted.
+    await keySourceSelect(page, "Current UI Session").click();
+    await onlyVisible(page.locator('.ant-select-item-option[title="Virtual Key"]')).click({ timeout: 15_000 });
+
+    const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
+    await expect(keyInput).toBeVisible({ timeout: 10_000 });
+    await keyInput.fill(key);
+
+    await selectModel(page, CHAT_MODEL_A);
+    await sendMessage(page, "playground ping via virtual key");
+
+    await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 });
+  });
+});

tests/e2e/ui/tests/proxy-admin/keys.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
 import {
   ADMIN_STORAGE_PATH,
   E2E_DELETE_KEY_ALIAS,
@@ -9,6 +9,19 @@ import {
 } from "../../constants";
 import { Page } from "../../fixtures/pages";
 import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
+import { captureRequestBody, readBack } from "../../helpers/roundTrip";
+
+/**
+ * Looks a key up by alias, undefined when none carries it. `return_full_object=true` is what makes
+ * the row carry token / models / tpm_limit; without it the response is aliases only.
+ */
+async function findKeyByAlias(page: PlaywrightPage, alias: string): Promise<Record<string, any> | undefined> {
+  const body = await readBack<{ keys: Record<string, any>[] }>(
+    page,
+    `/key/list?key_alias=${encodeURIComponent(alias)}&return_full_object=true&size=100`,
+  );
+  return body.keys.find((row) => row.key_alias === alias);
+}
 
 test.describe("Proxy Admin - Keys", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
@@ -47,12 +60,21 @@ test.describe("Proxy Admin - Keys", () => {
 
     // Verify the new key appears in the table
     await expect(page.getByText(keyName)).toBeVisible({ timeout: 10_000 });
+
+    // The row above renders from the create response the UI already holds, so it proves nothing.
+    const persisted = await findKeyByAlias(page, keyName);
+    expect(persisted, `key ${keyName} readable from /key/list`).toBeTruthy();
+    expect(typeof persisted?.team_id, "created key is owned by a team, not orphaned").toBe("string");
   });
 
   test("Regenerate key", async ({ page }) => {
     await navigateToPage(page, Page.ApiKeys);
     await dismissFeedbackPopup(page);
 
+    // Capture the old token first: a modal with a Copy button only proves the UI rendered.
+    const before = await findKeyByAlias(page, E2E_REGENERATE_KEY_ALIAS);
+    expect(before?.token, `seeded key ${E2E_REGENERATE_KEY_ALIAS} has a token`).toBeTruthy();
+
     // Key IDs are rendered as buttons in the table
     const keyRow = page.locator("tr", { hasText: E2E_REGENERATE_KEY_ALIAS });
     await expect(keyRow).toBeVisible({ timeout: 10_000 });
@@ -70,12 +92,24 @@ test.describe("Proxy Admin - Keys", () => {
 
     // Success view shows a Copy button in the footer (text varies between modal versions)
     await expect(modal.getByRole("button", { name: /Copy.*Key/ })).toBeVisible({ timeout: 20_000 });
+
+    // The token must be replaced and the alias kept; orphaning it looks identical from the modal.
+    await expect
+      .poll(async () => (await findKeyByAlias(page, E2E_REGENERATE_KEY_ALIAS))?.token, {
+        message: `token for ${E2E_REGENERATE_KEY_ALIAS} did not change after regenerate`,
+        timeout: 15_000,
+      })
+      .not.toBe(before?.token);
   });
 
   test("Update key TPM and RPM limits", async ({ page }) => {
     await navigateToPage(page, Page.ApiKeys);
     await dismissFeedbackPopup(page);
 
+    // Snapshot first, so the end assertions can tell an isolated edit from a collateral one.
+    const before = await findKeyByAlias(page, E2E_UPDATE_LIMITS_KEY_ALIAS);
+    expect(before, `seeded key ${E2E_UPDATE_LIMITS_KEY_ALIAS} exists`).toBeTruthy();
+
     const keyRow = page.locator("tr", { hasText: E2E_UPDATE_LIMITS_KEY_ALIAS });
     await expect(keyRow).toBeVisible({ timeout: 10_000 });
     await keyRow.locator("button").first().click();
@@ -87,10 +121,27 @@ test.describe("Proxy Admin - Keys", () => {
 
     await page.getByRole("spinbutton", { name: "TPM Limit" }).fill("123");
     await page.getByRole("spinbutton", { name: "RPM Limit" }).fill("456");
-    await page.getByRole("button", { name: "Save Changes" }).click();
+
+    const update = await captureRequestBody(page, { method: "POST", urlIncludes: "/key/update" }, async () => {
+      await page.getByRole("button", { name: "Save Changes" }).click();
+    });
+
+    // The form posts limits at the top level. Compare numerically: the spinbutton yields either type.
+    expect(Number(update.tpm_limit), "TPM limit on the wire").toBe(123);
+    expect(Number(update.rpm_limit), "RPM limit on the wire").toBe(456);
 
     await expect(page.getByRole("paragraph").filter({ hasText: "TPM: 123" })).toBeVisible({ timeout: 10_000 });
     await expect(page.getByRole("paragraph").filter({ hasText: "RPM: 456" })).toBeVisible({ timeout: 10_000 });
+
+    // Read the key back; the rendering above comes from a response the UI already holds.
+    const after = await findKeyByAlias(page, E2E_UPDATE_LIMITS_KEY_ALIAS);
+    expect(after, "key still readable after update").toBeTruthy();
+    expect(Number(after?.tpm_limit), "TPM limit persisted").toBe(123);
+    expect(Number(after?.rpm_limit), "RPM limit persisted").toBe(456);
+
+    // Not hypothetical: bumping a key's budget wiped its MCP toolset (PR #34452), toast said success.
+    expect(after?.models, "editing limits left the key's models untouched").toEqual(before?.models);
+    expect(after?.team_id, "editing limits left the key's team untouched").toEqual(before?.team_id);
   });
 
   test("Delete key", async ({ page }) => {
@@ -115,6 +166,14 @@ test.describe("Proxy Admin - Keys", () => {
     await deleteButton.click();
 
     await expect(page.getByText(/Key deleted/i).first()).toBeVisible({ timeout: 10_000 });
+
+    // The key is gone when the management API stops returning it, not when the toast says so.
+    await expect
+      .poll(async () => await findKeyByAlias(page, E2E_DELETE_KEY_ALIAS), {
+        message: `key ${E2E_DELETE_KEY_ALIAS} still readable from /key/list after delete`,
+        timeout: 15_000,
+      })
+      .toBeUndefined();
   });
 
   test("See internal user keys in team", async ({ page }) => {

tests/e2e/ui/tests/proxy-admin/teams.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
 import {
   ADMIN_STORAGE_PATH,
   E2E_TEAM_CRUD_ID,
@@ -8,6 +8,22 @@ import {
 } from "../../constants";
 import { Page } from "../../fixtures/pages";
 import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
+import { readBack } from "../../helpers/roundTrip";
+
+/** GET /team/list returns a bare array of teams, each carrying team_alias/team_id. */
+async function findTeamByAlias(page: PlaywrightPage, alias: string): Promise<Record<string, any> | undefined> {
+  const teams = await readBack<Record<string, any>[]>(page, "/team/list");
+  return teams.find((team) => team.team_alias === alias);
+}
+
+/** GET /team/info nests the record under `team_info`; membership lives in members_with_roles. */
+async function teamMemberEmails(page: PlaywrightPage, teamId: string): Promise<string[]> {
+  const info = await readBack<{ team_info: { members_with_roles?: { user_email?: string }[] } }>(
+    page,
+    `/team/info?team_id=${encodeURIComponent(teamId)}`,
+  );
+  return (info.team_info.members_with_roles ?? []).map((member) => member.user_email ?? "").filter(Boolean);
+}
 
 test.describe("Proxy Admin - Teams", () => {
   test.use({ storageState: ADMIN_STORAGE_PATH });
@@ -42,6 +58,11 @@ test.describe("Proxy Admin - Teams", () => {
 
     // Verify success notification
     await expect(page.getByText("Team created").first()).toBeVisible({ timeout: 10_000 });
+
+    // A create that drops its model selection still toasts success.
+    const created = await findTeamByAlias(page, uniqueAlias);
+    expect(created, `team ${uniqueAlias} readable from /team/list`).toBeTruthy();
+    expect(created?.models, "created team kept its model selection").toBeTruthy();
   });
 
   test("Invite a user to a team", async ({ page }) => {
@@ -71,6 +92,14 @@ test.describe("Proxy Admin - Teams", () => {
     await modal.getByRole("button", { name: /Add Member/i }).click();
 
     await expect(page.getByText(/member.*added|success/i).first()).toBeVisible({ timeout: 10_000 });
+
+    // The toast is matched loosely enough (/success/i) that almost any notification satisfies it.
+    await expect
+      .poll(async () => await teamMemberEmails(page, E2E_TEAM_CRUD_ID), {
+        message: "invited user never appeared in the team's members",
+        timeout: 15_000,
+      })
+      .toContain("invitable@test.local");
   });
 
   test("Edit team member for team proxy admin does not belong to", async ({ page }) => {
@@ -106,6 +135,14 @@ test.describe("Proxy Admin - Teams", () => {
     await modal.getByRole("button", { name: /Force Delete|Delete/i }).click();
 
     await expect(teamRow).not.toBeVisible({ timeout: 10_000 });
+
+    // A row vanishing is local state, which happens whether or not the delete landed.
+    await expect
+      .poll(async () => await findTeamByAlias(page, E2E_TEAM_DELETE_ALIAS), {
+        message: `team ${E2E_TEAM_DELETE_ALIAS} still readable from /team/list after delete`,
+        timeout: 15_000,
+      })
+      .toBeUndefined();
   });
 
   test("Team in org - edit team member", async ({ page }) => {

tests/e2e/ui/tests/settings/routerSettings.spec.ts

@@ -3,6 +3,8 @@ import { ADMIN_STORAGE_PATH } from "../../constants";
 import { navigateToPage } from "../../helpers/navigation";
 import { Page } from "../../fixtures/pages";
 import { Role, users } from "../../fixtures/users";
+import { MOCK_RESPONSE_TEXT } from "../../helpers/traffic";
+import { openPlayground, selectModel, sendMessage } from "../../helpers/playground";
 // Type-only import of the OpenAPI-generated backend schema, erased at runtime by
 // esbuild. It types the round-trips below so mistakes surface in the editor; the live
 // test against the real proxy is what actually enforces the contract.
@@ -79,7 +81,9 @@ test.describe("Router Settings - Fallbacks", () => {
     await primarySelect.click();
     await page.keyboard.type(PRIMARY);
     await page.keyboard.press("Enter");
-    await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({ timeout: 10_000 });
+    await expect(modal.getByRole("tab", { name: PRIMARY })).toBeVisible({
+      timeout: 10_000,
+    });
 
     const fallbackSelect = modal.locator(".ant-select").filter({ hasText: "Select fallback models" });
     await fallbackSelect.click();
@@ -88,7 +92,9 @@ test.describe("Router Settings - Fallbacks", () => {
     await page.keyboard.press("Escape");
     // The Fallback Chain helper text reads "(N/10 used)"; once it ticks to 1 the
     // selection has been recorded.
-    await expect(modal.getByText("(1/10 used)")).toBeVisible({ timeout: 10_000 });
+    await expect(modal.getByText("(1/10 used)")).toBeVisible({
+      timeout: 10_000,
+    });
 
     // Save
     await modal.getByRole("button", { name: /Save All Configurations/i }).click();
@@ -111,7 +117,9 @@ test.describe("Router Settings - Fallbacks", () => {
 type ConfigYAML = components["schemas"]["ConfigYAML"];
 type RouterSettingsResponse = components["schemas"]["RouterSettingsResponse"];
 
-const ADMIN_AUTH = { Authorization: `Bearer ${users[Role.ProxyAdmin].password}` };
+const ADMIN_AUTH = {
+  Authorization: `Bearer ${users[Role.ProxyAdmin].password}`,
+};
 
 /**
  * Apply a router_settings patch through the typed /config/update contract. The
@@ -172,13 +180,17 @@ test.describe("Router Settings - Loadbalancing", () => {
     // The ticket's core symptom was that a refresh showed the old value.
     await navigateToPage(page, Page.RouterSettings);
     await page.getByRole("tab", { name: "Loadbalancing" }).click();
-    await expect(page.locator('input[name="num_retries"]')).toHaveValue("5", { timeout: 15_000 });
+    await expect(page.locator('input[name="num_retries"]')).toHaveValue("5", {
+      timeout: 15_000,
+    });
 
     // The typed backend read agrees the change persisted.
     await expect
       .poll(
         async () => {
-          const res = await request.get(`/router/settings`, { headers: ADMIN_AUTH });
+          const res = await request.get(`/router/settings`, {
+            headers: ADMIN_AUTH,
+          });
           const data = (await res.json()) as RouterSettingsResponse;
           return data.current_values?.num_retries;
         },
@@ -187,3 +199,91 @@ test.describe("Router Settings - Loadbalancing", () => {
       .toBe(5);
   });
 });
+
+/**
+ * The test above proves the UI can record a fallback; this proves the fallback is honoured. The
+ * primary is created here because every fixture model is mock-backed and cannot fail on demand.
+ */
+test.describe("Router Settings - Fallbacks serve the request", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  const BROKEN_PRIMARY = "e2e-broken-primary";
+  let brokenModelId: string | null = null;
+
+  /** Drop only this test's fallback entry, leaving any others untouched. */
+  async function clearBrokenFallback(request: import("@playwright/test").APIRequestContext) {
+    const current = await request.get("/get/config/callbacks", {
+      headers: ADMIN_AUTH,
+    });
+    if (!current.ok()) return;
+    const router = (await current.json())?.router_settings ?? {};
+    const existing: Array<Record<string, string[]>> = Array.isArray(router.fallbacks) ? router.fallbacks : [];
+    await patchRouterSettings(request, {
+      fallbacks: existing.filter((entry) => !(entry && BROKEN_PRIMARY in entry)),
+    } as Partial<NonNullable<ConfigYAML["router_settings"]>>);
+  }
+
+  test.beforeEach(async ({ request }) => {
+    await clearBrokenFallback(request);
+
+    // Port 9 is the discard service: nothing listens, so the connection is
+    // refused immediately rather than hanging until a timeout.
+    const res = await request.post("/model/new", {
+      headers: ADMIN_AUTH,
+      data: {
+        model_name: BROKEN_PRIMARY,
+        litellm_params: {
+          model: "openai/broken",
+          api_base: "http://127.0.0.1:9/v1",
+          api_key: "fake",
+          timeout: 5,
+        },
+      },
+    });
+    expect(res.ok(), `creating the broken primary failed: ${res.status()} ${await res.text()}`).toBeTruthy();
+    brokenModelId = (await res.json())?.model_id ?? null;
+  });
+
+  test.afterEach(async ({ request }) => {
+    await clearBrokenFallback(request);
+    if (brokenModelId) {
+      await request.post("/model/delete", {
+        headers: ADMIN_AUTH,
+        data: { id: brokenModelId },
+      });
+      brokenModelId = null;
+    }
+  });
+
+  test("a request to an unreachable model is answered by its fallback", async ({ page, request }) => {
+    const chat = async () =>
+      request.post("/v1/chat/completions", {
+        headers: { ...ADMIN_AUTH, "Content-Type": "application/json" },
+        data: {
+          model: BROKEN_PRIMARY,
+          messages: [{ role: "user", content: "fallback probe" }],
+        },
+      });
+
+    // The control: it proves the reply below could only have come from the fallback.
+    expect((await chat()).status(), "broken primary unexpectedly succeeded on its own").toBeGreaterThanOrEqual(400);
+
+    await patchRouterSettings(request, {
+      fallbacks: [{ [BROKEN_PRIMARY]: [PRIMARY] }],
+    } as Partial<NonNullable<ConfigYAML["router_settings"]>>);
+
+    // Same call now succeeds, served by the fallback model.
+    await expect
+      .poll(async () => (await chat()).status(), {
+        timeout: 30_000,
+        message: "fallback never took effect",
+      })
+      .toBe(200);
+
+    // And the playground renders a reply for a model whose own upstream is down.
+    await openPlayground(page);
+    await selectModel(page, BROKEN_PRIMARY);
+    await sendMessage(page, "fallback probe from the playground");
+    await expect(page.getByText(MOCK_RESPONSE_TEXT, { exact: false }).first()).toBeVisible({ timeout: 60_000 });
+  });
+});

tests/e2e/ui/tests/team-admin/teamAdmin.spec.ts

@@ -1,4 +1,4 @@
-import { test, expect } from "@playwright/test";
+import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
 import {
   E2E_INTERNAL_USER_KEY_ALIAS,
   E2E_TEAM_CRUD_ALIAS,
@@ -6,13 +6,30 @@ import {
   TEAM_ADMIN_STORAGE_PATH,
 } from "../../constants";
 import { Page } from "../../fixtures/pages";
-import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
+import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
+import { captureRequestBody, readBack } from "../../helpers/roundTrip";
 
-async function clickTeamId(page: import("@playwright/test").Page, teamId: string) {
-  const cell = page.locator("td").filter({ hasText: teamId }).first();
-  await expect(cell).toBeVisible({ timeout: 10_000 });
-  await cell.click();
-  await expect(page.getByText("Back to Teams")).toBeVisible({ timeout: 10_000 });
+/**
+ * Every identifier a roster is addressable by. Which of user_id / user_email is populated depends on
+ * how the member got there, so flatten both and let assertions name whichever the test typed.
+ */
+async function teamMemberIdentities(page: PlaywrightPage, teamId: string): Promise<string[]> {
+  const info = await readBack<{ team_info: { members_with_roles?: { user_id?: string; user_email?: string }[] } }>(
+    page,
+    `/team/info?team_id=${encodeURIComponent(teamId)}`,
+  );
+  return (info.team_info.members_with_roles ?? []).flatMap((member) =>
+    [member.user_id, member.user_email].filter((value): value is string => Boolean(value)),
+  );
+}
+
+/** See keys.spec.ts -- return_full_object is what makes the row carry team_id. */
+async function findKeyByAlias(page: PlaywrightPage, alias: string): Promise<Record<string, any> | undefined> {
+  const body = await readBack<{ keys: Record<string, any>[] }>(
+    page,
+    `/key/list?key_alias=${encodeURIComponent(alias)}&return_full_object=true&size=100`,
+  );
+  return body.keys.find((row) => row.key_alias === alias);
 }
 
 test.describe("Team Admin", () => {
@@ -56,9 +73,22 @@ test.describe("Team Admin", () => {
     await expect(emailOption).toBeAttached({ timeout: 10_000 });
     await page.keyboard.press("Enter");
 
-    await modal.getByRole("button", { name: /Add Member/i }).click();
+    const add = await captureRequestBody(page, { method: "POST", urlIncludes: "/team/member_add" }, async () => {
+      await modal.getByRole("button", { name: /Add Member/i }).click();
+    });
+    // An add carrying the wrong team_id still toasts success, and the member lands elsewhere.
+    expect(add.team_id, "add targets the team being viewed").toBe(E2E_TEAM_CRUD_ID);
+    expect(add.member?.user_email, "the typed email is what goes on the wire").toBe("invitable-team@test.local");
 
     await expect(page.getByText("Team member added successfully").first()).toBeVisible({ timeout: 10_000 });
+
+    // Membership is the point of the flow, so read the roster back.
+    await expect
+      .poll(async () => await teamMemberIdentities(page, E2E_TEAM_CRUD_ID), {
+        message: "added member never appeared in the team's roster",
+        timeout: 15_000,
+      })
+      .toContain("invitable-team@test.local");
   });
 
   test("Team admin can remove a member from their team", async ({ page }) => {
@@ -77,9 +107,25 @@ test.describe("Team Admin", () => {
 
     const modal = page.locator(".ant-modal:visible");
     await expect(modal).toBeVisible({ timeout: 5_000 });
-    await modal.getByRole("button", { name: /^Delete$/ }).click();
+
+    const remove = await captureRequestBody(page, { method: "POST", urlIncludes: "/team/member_delete" }, async () => {
+      await modal.getByRole("button", { name: /^Delete$/ }).click();
+    });
+    // Removing the wrong member is exactly what a success toast hides, so pin both halves.
+    expect(remove.team_id, "delete targets the team being viewed").toBe(E2E_TEAM_CRUD_ID);
+    expect([remove.user_id, remove.user_email], "delete identifies the member whose row was clicked").toContain(
+      "e2e-removable-member",
+    );
 
     await expect(page.getByText("Team member removed successfully").first()).toBeVisible({ timeout: 10_000 });
+
+    // The row disappearing is local state, which happens whether or not the write landed.
+    await expect
+      .poll(async () => await teamMemberIdentities(page, E2E_TEAM_CRUD_ID), {
+        message: "removed member is still on the team",
+        timeout: 15_000,
+      })
+      .not.toContain("e2e-removable-member");
   });
 
   test("Team admin can create a team key with All Team Models", async ({ page }) => {
@@ -103,11 +149,20 @@ test.describe("Team Admin", () => {
     await page.locator(".ant-select-dropdown:visible").getByText("All Team Models").click();
     await page.keyboard.press("Escape");
 
-    await page.getByRole("button", { name: "Create Key", exact: true }).click();
+    const generate = await captureRequestBody(page, { method: "POST", urlIncludes: "/key/generate" }, async () => {
+      await page.getByRole("button", { name: "Create Key", exact: true }).click();
+    });
+    expect(generate.team_id, "the selected team goes on the wire").toBe(E2E_TEAM_CRUD_ID);
 
     await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
     await page.keyboard.press("Escape");
 
     await expect(page.getByText(keyName)).toBeVisible({ timeout: 10_000 });
+
+    // A team-admin key that comes back unscoped, or scoped elsewhere, is a privilege and
+    // billing problem that only a read-back sees.
+    const persisted = await findKeyByAlias(page, keyName);
+    expect(persisted, `key ${keyName} readable from /key/list`).toBeTruthy();
+    expect(persisted?.team_id, "the key is owned by the team admin's own team").toBe(E2E_TEAM_CRUD_ID);
   });
 });

tests/e2e/ui/tests/usage/usagePage.spec.ts

@@ -0,0 +1,74 @@
+import { test, expect, type Locator, type Page as PlaywrightPage } from "@playwright/test";
+import { ADMIN_STORAGE_PATH } from "../../constants";
+import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
+import { Page } from "../../fixtures/pages";
+import {
+  CHAT_MODEL_A,
+  createVirtualKey,
+  sendChatCompletion,
+  waitForKeyInDailyActivity,
+  waitForSpendLog,
+} from "../../helpers/traffic";
+
+/** Covers /ui/usage. The legacy /ui/old-usage view is deprecated and deliberately not covered. */
+
+/** Stepping up from the title is exact; the page renders several other tables. */
+const topKeysCard = (page: PlaywrightPage): Locator =>
+  page.getByText("Top Virtual Keys", { exact: true }).locator("xpath=..");
+
+async function openUsage(page: PlaywrightPage): Promise<Locator> {
+  await navigateToPage(page, Page.NewUsage);
+  await dismissFeedbackPopup(page);
+  const card = topKeysCard(page);
+  await expect(card).toBeVisible({ timeout: 30_000 });
+  // Widen past the default top-5 so other keys in the database cannot crowd this one out.
+  await card.locator(".ant-segmented-item").filter({ hasText: /^50$/ }).click();
+  return card;
+}
+
+test.describe("Usage page", () => {
+  test.use({ storageState: ADMIN_STORAGE_PATH });
+
+  test("Top Virtual Keys lists a key that served traffic, toggles views, and opens key info", async ({
+    page,
+    request,
+  }) => {
+    const alias = `e2e-usage-key-${Date.now()}`;
+    const { key, token } = await createVirtualKey(request, {
+      key_alias: alias,
+    });
+
+    const requestId = await sendChatCompletion(request, {
+      model: CHAT_MODEL_A,
+      prompt: `usage ping for ${alias}`,
+      apiKey: key,
+    });
+    await waitForSpendLog(request, requestId);
+    // Must land in the aggregate before the page mounts — it fetches once.
+    await waitForKeyInDailyActivity(request, token);
+
+    const card = await openUsage(page);
+
+    // Table view (the default): the key is listed by its alias.
+    const row = card.locator("tbody tr").filter({ hasText: alias });
+    await expect(row, `${alias} missing from Top Virtual Keys`).toHaveCount(1, {
+      timeout: 30_000,
+    });
+
+    // Chart view swaps the table out for the bar chart, and back.
+    await card.getByText("Chart View", { exact: true }).click();
+    await expect(card.locator("tbody tr")).toHaveCount(0, { timeout: 10_000 });
+    await card.getByText("Table View", { exact: true }).click();
+    await expect(row).toHaveCount(1, { timeout: 10_000 });
+
+    // Clicking the Key ID cell fetches key info and opens the detail panel.
+    // The alias is already in the row behind the modal, so match the panel's own controls.
+    await row.locator("td").first().click();
+    const keyInfo = page.getByRole("tab", { name: "Overview", exact: true });
+    await expect(keyInfo, "key info panel did not open").toBeVisible({
+      timeout: 20_000,
+    });
+    await expect(page.getByRole("tab", { name: "Settings", exact: true })).toBeVisible();
+    await expect(page.getByText("Back to Keys", { exact: false })).toBeVisible();
+  });
+});

tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import os
 import sys
@@ -18,6 +19,7 @@ from litellm.proxy._types import (
     LiteLLM_TeamTable,
     LitellmUserRoles,
     Member,
+    ReconcileOutcome,
     UserAPIKeyAuth,
 )
 from litellm.proxy.management_endpoints.model_management_endpoints import (
@@ -103,11 +105,11 @@ class MockProxyConfig:
         self.success = success
         self.deployment_called = False
 
-    async def add_deployment(self, prisma_client, proxy_logging_obj):
+    async def _add_deployment_locked(self, prisma_client, proxy_logging_obj):
         self.deployment_called = True
         if not self.success:
             raise Exception("Failed to add deployment")
-        return True
+        return ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
 
 
 class TestModelManagementAuthChecks:
@@ -409,7 +411,9 @@ class TestClearCache:
         mock_router.model_list = ["openai/gpt-4o", "openai/gpt-4o-mini"]
 
         mock_config = MagicMock()
-        mock_config.add_deployment = AsyncMock(return_value=True)
+        mock_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
 
         mock_prisma = MagicMock()
         mock_logging = MagicMock()
@@ -430,7 +434,9 @@ class TestClearCache:
     @pytest.mark.asyncio
     async def test_clear_cache_preserve_config_models(self):
         """
-        Test that clear_cache clears DB models and preserves config models.
+        clear_cache resets DB-backed auto-router entries and delegates every deployment
+        change to the reload, leaving config models untouched. It must not wipe
+        deployments itself -- see the delete_deployment assertion below.
         """
         from litellm.proxy.management_endpoints.model_management_endpoints import (
             clear_cache,
@@ -463,7 +469,9 @@ class TestClearCache:
         mock_router.complexity_routers = {"db-complexity-router": MagicMock(), "config-router": MagicMock()}
 
         mock_config = MagicMock()
-        mock_config.add_deployment = AsyncMock(return_value=True)
+        mock_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
 
         mock_prisma = MagicMock()
         mock_logging = MagicMock()
@@ -477,7 +485,10 @@ class TestClearCache:
         ):
             await clear_cache()
 
-            # Should have called delete_deployment for both DB models
+            # clear_cache must wipe ONLY the db auto-router deployments -- the ones whose
+            # strategy entries are popped below and can only be rebuilt via the add path.
+            # Ordinary db models are left alone: wiping them un-served every db model for
+            # the width of the reload, and the reconcile converges without it.
             assert mock_router.delete_deployment.call_count == 2
             mock_router.delete_deployment.assert_any_call(id="db-model-1")
             mock_router.delete_deployment.assert_any_call(id="db-model-2")
@@ -491,11 +502,70 @@ class TestClearCache:
             assert "config-router" in mock_router.auto_routers
             assert "config-router" in mock_router.complexity_routers
 
-            # Should have called add_deployment to reload DB models
-            mock_config.add_deployment.assert_called_once_with(
+            # Should have called the already-locked reload to restore DB models
+            mock_config._add_deployment_locked.assert_called_once_with(
                 prisma_client=mock_prisma, proxy_logging_obj=mock_logging
             )
 
+    @pytest.mark.asyncio
+    async def test_clear_cache_wipes_auto_routers_but_leaves_ordinary_db_models(self):
+        """An ordinary db model must survive clear_cache; a db auto-router must not.
+
+        Two separate hazards meet here, and fixing one naively breaks the other:
+
+        - Wiping ordinary db models un-serves EVERY db model for the width of the
+          reload. The reconcile converges without that, so the wipe is a pure
+          data-plane hole.
+        - NOT wiping a db auto-router strands it. Its strategy registries are keyed by
+          model_name and are popped here, but Router.upsert_deployment returns early
+          for an unchanged deployment and never reaches the add path that rebuilds
+          them. Any unrelated model write would then leave every db-backed auto,
+          complexity, adaptive and quality router unroutable until a restart.
+
+        So the wipe is scoped to exactly the auto-router deployments.
+        """
+        from litellm.proxy.management_endpoints.model_management_endpoints import (
+            clear_cache,
+        )
+
+        mock_router = MagicMock()
+        mock_router.model_list = [
+            {
+                "model_name": "ordinary-db-model",
+                "model_info": {"id": "db-ordinary-1", "db_model": True},
+                "litellm_params": {"model": "openai/gpt-4o"},
+            },
+            {
+                "model_name": "db-auto-router",
+                "model_info": {"id": "db-auto-1", "db_model": True},
+                "litellm_params": {"model": "auto_router/db-auto-router"},
+            },
+        ]
+        mock_router.delete_deployment = MagicMock(return_value=True)
+        mock_router.auto_routers = {"db-auto-router": MagicMock()}
+        mock_router.complexity_routers = {}
+        mock_router.adaptive_routers = {}
+        mock_router.quality_routers = {}
+
+        mock_config = MagicMock()
+        mock_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
+
+        with (
+            patch("litellm.proxy.proxy_server.llm_router", mock_router),
+            patch("litellm.proxy.proxy_server.proxy_config", mock_config),
+            patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
+            patch("litellm.proxy.proxy_server.proxy_logging_obj", MagicMock()),
+            patch("litellm.proxy.proxy_server.verbose_proxy_logger"),
+        ):
+            await clear_cache()
+
+        # The auto-router deployment is wiped so the reload takes the add path and
+        # rebuilds its strategy entry; the ordinary db model is never touched.
+        mock_router.delete_deployment.assert_called_once_with(id="db-auto-1")
+        assert "db-auto-router" not in mock_router.auto_routers
+
 
 class TestClearCachePreservesConfigRouters:
     """
@@ -534,7 +604,9 @@ class TestClearCachePreservesConfigRouters:
         }
 
         mock_config = MagicMock()
-        mock_config.add_deployment = AsyncMock(return_value=True)
+        mock_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
 
         with (
             patch("litellm.proxy.proxy_server.llm_router", mock_router),
@@ -574,7 +646,9 @@ class TestClearCachePreservesConfigRouters:
         mock_router.complexity_routers = {"shared-name": MagicMock()}
 
         mock_config = MagicMock()
-        mock_config.add_deployment = AsyncMock(return_value=True)
+        mock_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
 
         with (
             patch("litellm.proxy.proxy_server.llm_router", mock_router),
@@ -616,7 +690,9 @@ class TestClearCachePreservesConfigRouters:
         mock_router.adaptive_routers = {"a1": MagicMock()}
 
         mock_config = MagicMock()
-        mock_config.add_deployment = AsyncMock(return_value=True)
+        mock_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
 
         with (
             patch("litellm.proxy.proxy_server.llm_router", mock_router),
@@ -839,7 +915,9 @@ class TestUpdateModel:
             ),
             patch(
                 "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
-                new=AsyncMock(return_value=None),
+                new=AsyncMock(
+                    return_value=ReconcileOutcome(still_desired=None, live_after=None)
+                ),
             ) as mock_clear_cache,
         ):
             await update_model(
@@ -1885,7 +1963,9 @@ class TestAddAndDeleteModelLifecycle:
         mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
 
         mock_proxy_config = MagicMock()
-        mock_proxy_config.add_deployment = AsyncMock()
+        mock_proxy_config._add_deployment_locked = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+        )
 
         mock_router = MagicMock()
         mock_router.delete_deployment = MagicMock()
@@ -3223,7 +3303,9 @@ class TestPatchModelBlockedAuthGate:
             ),
             patch(
                 "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
-                new=AsyncMock(return_value=None),
+                new=AsyncMock(
+                    return_value=ReconcileOutcome(still_desired=None, live_after=None)
+                ),
             ),
         ):
             result = await patch_model(
@@ -3381,6 +3463,281 @@ class TestWriteSurfacesReloadDrop:
             )
 
 
+class TestConcurrentModelWritesDoNotEvictEachOther:
+    """Two model writes racing on one pod must not un-serve each other's deployments,
+    and neither may report the other's in-flight reload as damage of its own.
+
+    The reconcile is a read-modify-write of the shared ``llm_router`` global: read the db
+    into a snapshot, then make the router match that snapshot. Unserialized, the request
+    holding the older snapshot deletes the deployment the newer one just added, because
+    _delete_deployment evicts every live id absent from the snapshot it was handed. The
+    row survives in the db, so the damage is invisible there -- the pod just stops
+    serving a model it was told to serve.
+    """
+
+    @pytest.mark.asyncio
+    async def test_reconciles_serialize_so_no_stale_snapshot_can_evict(self, monkeypatch):
+        """MODEL_RECONCILE_LOCK admits one reconcile at a time.
+
+        The fake body awaits, which is the whole point: without the lock the gather below
+        parks all five inside the critical section at that await and observed depth goes
+        to 5. Asserting depth never exceeds 1 is what pins the fix -- deleting the
+        `async with` makes this fail rather than merely getting slower.
+        """
+        import asyncio
+
+        from litellm.proxy._types import ReconcileOutcome
+        from litellm.proxy.proxy_server import ProxyConfig
+
+        depth = 0
+        observed_max = 0
+
+        async def fake_locked(self, **kwargs):
+            nonlocal depth, observed_max
+            depth += 1
+            observed_max = max(observed_max, depth)
+            await asyncio.sleep(0)
+            depth -= 1
+            return ReconcileOutcome(still_desired=frozenset(), live_after=frozenset())
+
+        monkeypatch.setattr(ProxyConfig, "_add_deployment_locked", fake_locked)
+        config = ProxyConfig()
+
+        await asyncio.gather(
+            *[
+                config.add_deployment(prisma_client=MagicMock(), proxy_logging_obj=MagicMock())
+                for _ in range(5)
+            ]
+        )
+
+        assert observed_max == 1
+
+    @pytest.mark.asyncio
+    async def test_clear_cache_reloads_under_the_lock_without_deadlocking(self, monkeypatch):
+        """clear_cache un-serves every db model before reloading, so it has to hold the
+        lock across the pair -- and therefore must call the already-locked reload.
+
+        asyncio.Lock is not reentrant: routing this back through the public
+        add_deployment would block forever on a lock this coroutine already owns, taking
+        every model write on the pod down with it. The timeout is the assertion.
+        """
+        import asyncio
+
+        import litellm
+        from litellm.proxy._types import ReconcileOutcome
+        from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
+        from litellm.proxy.proxy_server import ProxyConfig
+
+        live_router = litellm.Router(
+            model_list=[
+                {
+                    "model_name": "gpt-4o",
+                    "litellm_params": {"model": "gpt-4o"},
+                    "model_info": {"id": "m-db", "db_model": True},
+                }
+            ]
+        )
+        monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", live_router)
+        monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", MagicMock())
+
+        async def fake_locked(self, **kwargs):
+            return ReconcileOutcome(still_desired=frozenset({"m-db"}), live_after=frozenset({"m-db"}))
+
+        monkeypatch.setattr(ProxyConfig, "_add_deployment_locked", fake_locked)
+
+        outcome = await asyncio.wait_for(clear_cache(), timeout=5)
+
+        assert outcome.still_desired == frozenset({"m-db"})
+        assert outcome.live_after == frozenset({"m-db"})
+
+    def test_verdict_trusts_the_lock_captured_snapshot_over_a_live_reread(self, monkeypatch):
+        """Given live_after, the verdict judges the router as it stood when the reload
+        finished -- not as it stands now.
+
+        Re-reading here would sample the router after the lock was released, which is
+        exactly where the next writer's clear_cache has every db model deleted and not
+        yet re-added. That hole is another request's in-flight state; blaming this
+        request's reload for it is the 500 that made concurrent model creates fail.
+        """
+        import litellm
+        from litellm.proxy._types import ProxyException
+        from litellm.proxy.management_endpoints.model_management_endpoints import (
+            raise_if_reload_degraded_serving,
+            reload_serving_verdict,
+        )
+
+        # The router as another writer's clear_cache leaves it mid-wipe: db models gone.
+        mid_wipe_router = litellm.Router(model_list=[])
+        monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", mid_wipe_router)
+
+        healthy_after_reload = frozenset({"m-live", "m-neighbour"})
+
+        _, collateral = reload_serving_verdict(
+            before=frozenset({"m-live", "m-neighbour"}),
+            written_models=[("m-live", None)],
+            written_must_serve=True,
+            still_desired=healthy_after_reload,
+            live_after=healthy_after_reload,
+        )
+        assert collateral == ()
+
+        assert (
+            raise_if_reload_degraded_serving(
+                before=frozenset({"m-live", "m-neighbour"}),
+                written_models=[("m-live", None)],
+                action="create",
+                still_desired=healthy_after_reload,
+                live_after=healthy_after_reload,
+            )
+            is None
+        )
+
+        # Same inputs, no lock-captured snapshot: the mid-wipe router is read live and
+        # the neighbour looks like collateral. This is the pre-fix behaviour, kept to
+        # show the parameter is what carries the difference.
+        with pytest.raises(ProxyException, match="m-neighbour"):
+            raise_if_reload_degraded_serving(
+                before=frozenset({"m-live", "m-neighbour"}),
+                written_models=[("m-live", None)],
+                action="create",
+                still_desired=healthy_after_reload,
+            )
+
+
+class TestDeleteEvictionsHoldTheReconcileLock:
+    """A delete evicts from ``llm_router`` directly instead of reconciling, so it must
+    take MODEL_RECONCILE_LOCK to do it.
+
+    The db row is gone by then, but a reconcile that snapshotted the db BEFORE the row
+    was deleted still lists that id as desired, and its ``_add_deployment`` upserts the
+    deployment back. Unserialized, the eviction can land while that reconcile is
+    mid-flight and simply be undone -- the pod keeps serving a model the database no
+    longer has, until the next reconcile happens to notice. Taking the lock orders the
+    eviction after any in-flight reconcile, making it the last word.
+    """
+
+    @staticmethod
+    async def _assert_evicts_under_lock(monkeypatch, call_endpoint, model_id: str) -> None:
+        """Run ``call_endpoint`` with the lock already held and assert it blocks.
+
+        Holding MODEL_RECONCILE_LOCK stands in for a reconcile that is mid-flight. If
+        the eviction takes the lock it cannot run until we release; if it does not, it
+        runs straight through and the deployment is evicted while the "reconcile" is
+        still in its critical section -- exactly the interleaving that resurrects it.
+
+        Each test gets a FRESH lock. asyncio.Lock binds itself to the event loop of its
+        first contended acquire and raises on every other loop afterwards, so a shared
+        module-level lock contended here would poison the next asyncio test in this
+        process. The proxy has one event loop for its lifetime, so this is a test-only
+        concern -- but it means any future test that contends this lock must patch its
+        own, exactly as here.
+        """
+        lock = asyncio.Lock()
+        monkeypatch.setattr("litellm.proxy.proxy_server.MODEL_RECONCILE_LOCK", lock)
+
+        async with lock:
+            task = asyncio.create_task(call_endpoint())
+            # Give the endpoint every chance to reach (and get stuck on) the lock.
+            for _ in range(50):
+                await asyncio.sleep(0)
+            assert not task.done(), (
+                f"deleting {model_id} did not wait for MODEL_RECONCILE_LOCK -- an "
+                f"in-flight reconcile can resurrect the deployment it just evicted"
+            )
+        await asyncio.wait_for(task, timeout=5)
+
+    @pytest.mark.asyncio
+    async def test_delete_model_waits_for_an_in_flight_reconcile(self, monkeypatch):
+        from litellm.proxy.management_endpoints.model_management_endpoints import (
+            ModelInfoDelete,
+            delete_model,
+        )
+
+        model_id = "m-doomed"
+        row = MagicMock()
+        row.model_dump.return_value = {
+            "model_name": "gpt-4o",
+            "litellm_params": {"model": "openai/gpt-4o"},
+            "model_info": {"id": model_id},
+        }
+        table = MagicMock()
+        table.find_unique = AsyncMock(return_value=row)
+        table.delete = AsyncMock(return_value=row)
+
+        prisma = MagicMock()
+        prisma.db.litellm_proxymodeltable = table
+
+        router = MagicMock()
+        router.delete_deployment = MagicMock(return_value=True)
+
+        monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma)
+        monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
+        monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True)
+        monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True)
+        monkeypatch.setattr(
+            "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
+            AsyncMock(return_value=True),
+        )
+
+        async def call() -> None:
+            await delete_model(
+                model_info=ModelInfoDelete(id=model_id),
+                user_api_key_dict=UserAPIKeyAuth(
+                    user_id="admin",
+                    user_role=LitellmUserRoles.PROXY_ADMIN,
+                    api_key="sk-admin",
+                ),
+            )
+
+        await self._assert_evicts_under_lock(monkeypatch, call, model_id)
+        router.delete_deployment.assert_called_once_with(id=model_id)
+
+    @pytest.mark.asyncio
+    async def test_delete_team_models_waits_for_an_in_flight_reconcile(self, monkeypatch):
+        from litellm.proxy.management_endpoints.model_management_endpoints import (
+            delete_team_models,
+        )
+
+        model_id = "m-team-doomed"
+        router = MagicMock()
+        router.delete_deployment = MagicMock(return_value=True)
+
+        # _get_team_deployments filters by the model_name prefix, then confirms
+        # model_info["team_id"] Python-side, so the row must satisfy both.
+        deleted_row = MagicMock()
+        deleted_row.model_id = model_id
+        deleted_row.model_name = "model_name_team-1_gpt-4o"
+        deleted_row.model_info = {"id": model_id, "team_id": "team-1"}
+
+        tx = MagicMock()
+        tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[deleted_row])
+        tx.litellm_proxymodeltable.delete_many = AsyncMock(return_value=1)
+
+        tx_ctx = MagicMock()
+        tx_ctx.__aenter__ = AsyncMock(return_value=tx)
+        tx_ctx.__aexit__ = AsyncMock(return_value=False)
+
+        prisma = MagicMock()
+        prisma.db.tx = MagicMock(return_value=tx_ctx)
+
+        monkeypatch.setattr(
+            "litellm.proxy.management_endpoints.model_management_endpoints.publish_config_change",
+            AsyncMock(return_value=None),
+        )
+        monkeypatch.setattr(
+            "litellm.proxy.management_endpoints.model_management_endpoints.coordination_redis_cache",
+            MagicMock(return_value=None),
+        )
+
+        async def call() -> None:
+            await delete_team_models(
+                team_ids=["team-1"], prisma_client=prisma, llm_router=router
+            )
+
+        await self._assert_evicts_under_lock(monkeypatch, call, model_id)
+        router.delete_deployment.assert_called_once_with(id=model_id)
+
+
 class TestModelInfoAsMapping:
     """The model_info column reaches consumers as a dict or as its JSON string; this is
     the single owner of that parse, and None means no usable mapping."""

tests/test_litellm/proxy/management_endpoints/test_ptu_model_settings.py

@@ -8,7 +8,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 from fastapi import HTTPException
 
-from litellm.proxy._types import LiteLLM_ProxyModelTable, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy._types import (
+    LiteLLM_ProxyModelTable,
+    LitellmUserRoles,
+    ReconcileOutcome,
+    UserAPIKeyAuth,
+)
 from litellm.proxy.management_endpoints.model_management_endpoints import (
     _merged_ptu_model_info,
     _raise_if_ptu_cost_attribution_disabled,
@@ -623,7 +628,12 @@ class TestAddNewModelPtuGate:
         add_team_model_to_db = AsyncMock(return_value=db_row)
 
         mock_proxy_config = MagicMock()
-        mock_proxy_config.add_deployment = AsyncMock(return_value=None)
+        # Both fields None: no reconcile state was captured, so the serving verdict
+        # falls back to reading the router live -- which is what mock_router below
+        # drives. These tests are about the PTU gate, not the reload verdict.
+        mock_proxy_config.add_deployment = AsyncMock(
+            return_value=ReconcileOutcome(still_desired=None, live_after=None)
+        )
 
         mock_router = MagicMock()
         mock_router.get_model_ids.return_value = [model_id]

tests/test_litellm/test_model_block_unblock.py

@@ -7,6 +7,7 @@ from litellm.proxy._types import (
     BlockModelRequest,
     LitellmUserRoles,
     ProxyException,
+    ReconcileOutcome,
     UserAPIKeyAuth,
 )
 from litellm.types.router import RouterRateLimitError
@@ -36,7 +37,12 @@ def _setup_model_block_mocks(monkeypatch, *, updated_blocked: bool):
     mock_router = MagicMock()
     mock_router.get_model_ids.return_value = [model_id]
 
-    mock_clear_cache = AsyncMock(return_value=None)
+    # No reconcile ran in these tests, so both fields are None and the verdict falls
+    # back to reading the router live -- which is what the get_model_ids side_effects
+    # below drive.
+    mock_clear_cache = AsyncMock(
+        return_value=ReconcileOutcome(still_desired=None, live_after=None)
+    )
     mock_audit_log = AsyncMock(return_value=None)
 
     monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client)