From ba1cab0b600441b46e85dc529ef696342ecf66a4 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 16 Jul 2026 15:12:52 -0700 Subject: [PATCH] fix(e2e): point UI tests at dashboard service; register complexity router Stage gateway 404s /ui; the Next.js dashboard is litellm-ui:3000. Drive playwright against E2E_UI_BASE_URL and wait on login placeholders after client render. Register complexity-smart-router via /model/new when the proxy does not already list it so stage matches compose config --- tests/e2e/e2e_config.py | 5 ++ tests/e2e/management/conftest.py | 20 +++--- .../test_key_models_dropdown_e2e.py | 6 +- tests/e2e/models.py | 1 + tests/e2e/router/conftest.py | 67 +++++++++++++++++++ 5 files changed, 88 insertions(+), 11 deletions(-) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index e84438430fd..2f6866305ea 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -24,6 +24,11 @@ CONTROL_PLANE_BASE_URL = os.environ.get( UI_USERNAME = os.environ.get("E2E_UI_USERNAME", "admin") UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY) +# Dashboard base URL. On split deployments the Next.js Admin UI is a separate +# service (e.g. litellm-ui:3000); the data-plane gateway often 404s /ui. +# Defaults to PROXY_BASE_URL so compose/monolith (proxy-served /ui) still works. +UI_BASE_URL = os.environ.get("E2E_UI_BASE_URL", PROXY_BASE_URL).rstrip("/") + CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5") CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") diff --git a/tests/e2e/management/conftest.py b/tests/e2e/management/conftest.py index 264108f6089..4dae616f01a 100644 --- a/tests/e2e/management/conftest.py +++ b/tests/e2e/management/conftest.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Iterator import pytest -from e2e_config import PROXY_BASE_URL, UI_PASSWORD, UI_USERNAME +from e2e_config import UI_BASE_URL, UI_PASSWORD, UI_USERNAME from management_client import ManagementClient, build_client if TYPE_CHECKING: @@ -47,13 +47,17 @@ def ui_page(browser: "Browser") -> "Iterator[Page]": context = browser.new_context() try: page = context.new_page() - page.goto(f"{PROXY_BASE_URL}/ui/") - page.fill("#username", UI_USERNAME) - page.fill("#password", UI_PASSWORD) - page.click('button[type="submit"]') - page.wait_for_function( - "() => document.cookie.includes('token=') || !document.querySelector('#username')" - ) + # Split deploys serve the Next.js dashboard on the UI service, not the + # data-plane gateway (which 404s /ui). Login is a client-rendered form + # that appears after LoadingScreen; wait on the placeholder, not #id + # (Ant Design Input does not always set id="username"). + page.goto(f"{UI_BASE_URL}/ui/login") + username = page.get_by_placeholder("Enter your username") + username.wait_for(state="visible", timeout=30_000) + username.fill(UI_USERNAME) + page.get_by_placeholder("Enter your password").fill(UI_PASSWORD) + page.get_by_role("button", name="Login").click() + page.wait_for_function("() => document.cookie.includes('token=')") yield page finally: context.close() diff --git a/tests/e2e/management/test_key_models_dropdown_e2e.py b/tests/e2e/management/test_key_models_dropdown_e2e.py index f0ba21699e0..676e3ccffcf 100644 --- a/tests/e2e/management/test_key_models_dropdown_e2e.py +++ b/tests/e2e/management/test_key_models_dropdown_e2e.py @@ -14,7 +14,7 @@ proxy under test does not serve it. import pytest -from e2e_config import PROXY_BASE_URL, unique_marker +from e2e_config import UI_BASE_URL, unique_marker from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, TeamNewBody @@ -46,7 +46,7 @@ def _models_dropdown_texts(page: Page, must_contain: str) -> list[str]: def _open_create_key_modal(page: Page) -> None: - page.goto(f"{PROXY_BASE_URL}/ui/api-keys/?create=true") + page.goto(f"{UI_BASE_URL}/ui/api-keys/?create=true") expect(page.locator(".ant-modal").first).to_be_visible() @@ -69,7 +69,7 @@ def _submit_create_modal(page: Page, sentinel_label: str) -> str: def _open_key_edit_form(page: Page, key_alias: str) -> None: - page.goto(f"{PROXY_BASE_URL}/ui/api-keys/") + page.goto(f"{UI_BASE_URL}/ui/api-keys/") page.get_by_text(key_alias).first.click() page.get_by_role("tab", name="Settings").click() page.get_by_role("button", name="Edit Settings").click() diff --git a/tests/e2e/models.py b/tests/e2e/models.py index c4ecd0cfa63..82c276d0b64 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -442,6 +442,7 @@ class LiteLLMParamsBody(BaseModel): output_cost_per_token: float | None = None extra_headers: dict[str, str] | None = None use_in_pass_through: bool | None = None + complexity_router_config: dict[str, object] | None = None ModelMode = Literal["batch", "realtime", "image_generation"] diff --git a/tests/e2e/router/conftest.py b/tests/e2e/router/conftest.py index e8c05520b10..025c0afcc6d 100644 --- a/tests/e2e/router/conftest.py +++ b/tests/e2e/router/conftest.py @@ -3,13 +3,80 @@ The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker live in the parent tests/e2e/conftest.py. ComplexityRouterClient holds the shared Gateway, so the `resources` fixture cleans up keys this suite creates. + +Also registers `complexity-smart-router` via management /model/new when the +proxy does not already list it (compose has it in static config; stage does not). """ +from __future__ import annotations + +from collections.abc import Iterator + import pytest +from requests import RequestException from complexity_router_client import ComplexityRouterClient, build_client +from e2e_gateway import Gateway +from e2e_http import NoBody, Success +from models import LiteLLMParamsBody, ModelsListResponse + +ROUTER_MODEL = "complexity-smart-router" +ROUTER_PARAMS = LiteLLMParamsBody( + model="auto_router/complexity_router", + complexity_router_config={ + "classifier_type": "llm", + "classifier_llm_config": {"model": "gpt-5.5"}, + "tiers": { + "SIMPLE": "gpt-5.5", + "MEDIUM": "claude-haiku-4-5", + "COMPLEX": "claude-haiku-4-5", + "REASONING": "claude-haiku-4-5", + }, + }, +) @pytest.fixture(scope="session") def client() -> ComplexityRouterClient: return build_client() + + +def _model_is_servable(gateway: Gateway, model_name: str) -> bool: + result = gateway.transport.get( + "/v1/models", + headers=gateway.transport.master, + params=NoBody(), + response_type=ModelsListResponse, + ) + return isinstance(result, Success) and any(entry.id == model_name for entry in result.data.data) + + +@pytest.fixture(scope="session", autouse=True) +def _ensure_complexity_smart_router( # pyright: ignore[reportUnusedFunction] # pytest autouse session fixture, wired by name + client: ComplexityRouterClient, +) -> Iterator[None]: + """Ensure the complexity router virtual model exists for this session. + + Compose already declares it in docker-compose.yml; stage does not. Register + via /model/new when missing and tear down only what we created. + """ + gateway = client.gateway + if _model_is_servable(gateway, ROUTER_MODEL): + yield + return + + try: + model_id = gateway.create_model(ROUTER_MODEL, ROUTER_PARAMS) + except (AssertionError, RequestException) as exc: + if _model_is_servable(gateway, ROUTER_MODEL): + yield + return + raise AssertionError( + f"failed to register {ROUTER_MODEL!r} for the complexity router e2e " + f"(not listed on /v1/models and /model/new failed): {exc}" + ) from exc + + try: + yield + finally: + gateway.delete_model(model_id)