From dd22dc9132ba2d32cb809fbcb482c2b76b74d352 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 09:18:44 -0700 Subject: [PATCH] test(perf): measure Admin UI journey latency in a browser on the 1TB SpendLogs restore (#47486) * test(perf): measure Admin UI journey latency in a browser on the 1TB restore * test(perf): settle on rendered rows, chart bars and the usage view trigger * test(perf): cover the UI journey summaries in the e2e harness tests * test(perf): fail UI journeys on failed samples and p50/p90 ceilings * test(perf): add Usage, Models and Keys journeys and seed realistic keys, teams, users and models on the disposable restore --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/UI_QUERIES.md | 111 ++--- tests/e2e/perf/setup/ui_journeys.py | 397 ++++++++++++++++++ tests/e2e/perf/setup/ui_queries.py | 227 ---------- tests/e2e/perf/setup/ui_query_cases.py | 206 --------- tests/e2e/perf/setup/ui_query_fixture.py | 72 +++- tests/e2e/perf/setup/ui_query_isolation.py | 4 +- tests/e2e/perf/test_ui_journeys.py | 108 +++++ tests/e2e/perf/test_ui_queries.py | 159 ------- .../perf/setup/test_ui_journeys.py | 144 +++++++ .../e2e_harness/perf/setup/test_ui_queries.py | 257 ------------ tests/e2e_harness/perf/test_upgrade.py | 4 +- 11 files changed, 761 insertions(+), 928 deletions(-) create mode 100644 tests/e2e/perf/setup/ui_journeys.py delete mode 100644 tests/e2e/perf/setup/ui_queries.py delete mode 100644 tests/e2e/perf/setup/ui_query_cases.py create mode 100644 tests/e2e/perf/test_ui_journeys.py delete mode 100644 tests/e2e/perf/test_ui_queries.py create mode 100644 tests/e2e_harness/perf/setup/test_ui_journeys.py delete mode 100644 tests/e2e_harness/perf/setup/test_ui_queries.py diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md index 552f1b953fd..27780537042 100644 --- a/tests/e2e/perf/UI_QUERIES.md +++ b/tests/e2e/perf/UI_QUERIES.md @@ -1,92 +1,55 @@ -# SpendLog UI query performance +# Admin UI journey latency on the 1TB SpendLogs restore -Run `test_ui_queries.py` against a dedicated proxy and a disposable database restored from the metadata-only performance seed. These tests do not call an LLM or measure browser rendering +Run `test_ui_journeys.py` against a dedicated proxy, its Admin UI and a disposable database restored from the metadata-only performance seed. A headless Chromium logs into the dashboard and performs the journeys below, timing each one from the user's click or navigation until the data is on screen. No LLM is called. The test does not look at which routes, SQL or tables served the page, so it stays valid whether the backend gets faster queries or new aggregate tables -The suite measures 42 request shapes across 20 routes used by current dashboard screens. It replays direct HTTP requests against the existing bulk seed, with the seed anchor replacing the wall clock and a UTC browser timezone. It does not run a browser or claim to cover every dashboard request +## Journeys -## UI callers +| name | what the user does | timer stops when | +| --- | --- | --- | +| `logs_open_24h` | opens Logs (default Last 24 Hours) | request rows are rendered and the refresh control is idle | +| `logs_last_7_days` | switches the time range to Last 7 Days | same | +| `logs_next_page` | clicks the next page control | same | +| `logs_filter_key_hash` | filters by the seeded key hash and applies | same | +| `logs_filter_key_alias` | picks the first key alias in the filter drawer and applies | same | +| `logs_filter_team` | picks the first team in the filter drawer and applies | same | +| `logs_filter_model` | picks the first model in the filter drawer and applies | same | +| `usage_open` | opens Usage | summary tiles show dollar values with no skeletons left | +| `usage_last_30_days` | picks Last 30 days in the Usage date picker | same | +| `usage_team_view` | switches the Usage view to Team Usage | same | +| `usage_customer_view` | switches the Usage view to Customer Usage | same | +| `usage_model_activity_30_days` | on the Model Activity tab, picks Last 30 days | same | +| `cache_open` | opens Caching | the Cache Hits tile is visible and the Cache Hits vs API Requests chart has bars | +| `models_open` | opens Models + Endpoints | model rows are rendered and the loading state is gone | +| `keys_open` | opens Virtual Keys | key rows are rendered and the loading state is gone | -All paths below are relative to `ui/litellm-dashboard/src/`. Each case requires an authenticated admin session with access to that screen, and selected resources must exist in the fixture +The mix follows what customers report as slow: the Logs page on `LiteLLM_SpendLogs`, the Usage page on the `LiteLLM_Daily*Spend` rollups (30 day ranges, Customer and Model views), and the Models and Keys pages. -| Cases | UI action and source | -| --- | --- | -| `logs_*` | Logs > Request Logs, change date range, filter, sort or next page. `components/logs/request/useLogFilterLogic.ts` and `RequestLogsFilters.tsx` | -| `usage_*` | Usage overview or entity tab, open key activity, search keys, expand a model group. `app/(dashboard)/usage/_components/components/UsagePageView.tsx`, `EntityUsage/EntityUsage.tsx`, `EntityUsage/entityFetchFns.ts`, `components/UsagePage/components/KeyActivityPanel.tsx` | -| `team_users_*` | Team Usage. `EntityUsage/EntityUsage.tsx` passes selected teams or all available non-dashboard teams to `TeamUserSpendCard.tsx` | -| `cache_activity_*` | Caching > Cache Analytics, optionally filter aliases/models. `app/(dashboard)/hooks/caching/useCacheActivity.ts` | -| `cache_leakage_keys` | Cost Optimization, switch cache leakage to Keys. `app/(dashboard)/cost-optimization/_components/useCacheLeakageKeys.ts` | +## Seeded rows -Request serialization lives in `components/networking.tsx`. Logs use page size 25 and session grouping, including the next-session cursor returned by page one. Individual Logs filters use a 30-day window to exercise larger matching datasets. Their case names end in `_30d` so the measurements do not share history with the old 24-hour filters. The synthetic cursorless page-three request is excluded +The snapshot carries the SpendLogs rows and their Daily* rollups. `setup/ui_query_fixture.py` checks that `LiteLLM_DailyUserSpend` and `LiteLLM_DailyTeamSpend` have rows in the week before the anchor, then fills the tables the snapshot does not have so the Keys, Models and filter journeys see realistic counts: 100 teams (`perf-team-N`), 2,000 users (`perf-user-N`), 5,000 keys whose token hashes and `perf-key-N` aliases match the seeded log rows, and 200 DB models pointed at the perf mock provider. Every insert is `ON CONFLICT DO NOTHING`, so reruns on the same restore are no-ops. Nothing here runs outside the disposable restore -Usage list, search and model drilldown requests do not inject an `api_key` filter. Team activity excludes `litellm-dashboard`. Cache leakage supplies `include_current_utc_day=true`. Dates and filter values are deterministic fixture values, not captured production traffic +Each journey is measured once cold, then `E2E_UI_JOURNEY_REPEATS` more times (default 9) from the same starting state, inside one logged-in tab whose clock is pinned to the seed anchor so relative windows land on seeded rows. The browser timezone is UTC to match the seed -Old Usage is excluded because its expensive-query gate normally hides those calls at this database size. LiteAdmin's agent-selected reports are excluded from this deterministic dashboard set. Missing-token alias recovery is backend functional coverage, not a separate UI action, and its artificial requests and orphan fixtures are removed +## Safety -This mapping is source-verified, not browser-recorded. Recheck the named callers when changing cases. Neither browser capture nor an automatic UI-drift checker is installed by this suite +The seeded source is never the target. `E2E_UI_PERF_DISPOSABLE=1`, `E2E_RESET_SPEND_LOGS=0` and a hostname matching `db-perf-perf-.litellm.svc` are all required, and in-flight SQL is cancelled only on that disposable restore between journeys (see `setup/ui_query_isolation.py`) -## Dataset scope +## Running -Logs, team-user spend and cache activity query the bulk SpendLogs data. User, team, tag and customer usage, plus cache leakage, query daily tables aggregated from that dataset. These are sequential large-dataset queries, not concurrent traffic or saturation tests - -A read-only restore of `db-perf-v1-105-1tb` gave exact request totals from daily aggregates: 9,442,606 across 24 hours, 66,077,435 across seven days and 283,189,684 across 30 days. The retained 30-day key filter matches 1,134,559 requests, user filter 11,316,454, team filter 113,298,119 and model filter 94,395,473. These describe matching requests, not rows scanned or returned by the API, and are documentation rather than performance assertions - -PostgreSQL table-wide estimates (`pg_class.reltuples`) are approximately 2.70 million daily user rows, 2.70 million daily team rows, 5.40 million daily tag rows and 78.27 million daily end-user rows. These are not exact counts within each Usage case's seven-day window - -The individual end-user filter is excluded because it matched only 104 requests in 24 hours and 8,616 across all dates. User/end-user dropdown requests are also excluded because their queries cap the input at 10,000 matching SpendLogs before extracting distinct values. Neither is a broad matching-data workload in this seed - -Excluded because they only had tiny matching fixtures or individual-record lookups: organization/agent usage, guardrail/tool logs, request-ID lookup, payload/session drawers, session/search filters for `perf-ui-*`, the OAuth-token filter, and all prompt-caching request cases. The seed has response-cache hits but no prompt-cache usage or injection metadata, so the prompt-caching cases only matched the 320 added rows. No UI-specific rows are inserted anymore - -| Remaining cases | Count | Variations | -| --- | ---: | --- | -| Logs | 21 | 24h/7d/30d, page 2 with session cursor, 12 individual filters over 30 days, five additional descending sorts | -| User/team/tag/customer usage | 16 | Aggregated totals, keys, key search, model top keys for each | -| Team-user spend | 2 | All 100 seeded teams or `perf-team-0` | -| Cache activity | 2 | All or selected aliases/model | -| Cache leakage keys | 1 | 30-day window | - -## Deployment prerequisites - -Use a fresh proxy against the disposable restore, never the active seed source or a shared test database. Apply the target migrations and wait for readiness first. The source seed's `litellm_perf_seed.checkpoint` must exist and contain rows - -Configure `litellm_settings.service_callback: [prometheus_system]` and expose metrics for every proxy worker/pod. Set `E2E_DATABASE_METRICS_URLS` to their comma-separated URLs, not a load-balanced endpoint that can miss a pod. Absent failure counters count as zero only when the endpoint exposes a positive PostgreSQL request counter. Missing instrumentation stops the run - -No expensive-query override or legacy view creation is needed for these cases. Keep the normal large-database UI restrictions - -The cases use explicit dates from the seed anchor where the UI supplies dates. A missing nonempty fixture result records a contract failure rather than reporting a fast empty query as success - -## Run - -Set `DATABASE_URL` and `LITELLM_MASTER_KEY` securely for this disposable deployment. The URL must name the same database used by the proxy, with hostname `db-perf-perf-.litellm.svc`. The explicit disposable confirmation permits cancellation of leftover queries in this database. Setup reads the existing seed checkpoint without adding UI-specific rows. Destroy the disposable database after the run +The componentized perf stack serves the dashboard behind the same router as the proxy, so `LITELLM_PROXY_URL` covers `/ui`. Set `E2E_UI_BASE_URL` only if the UI lives elsewhere. The admin login uses `E2E_UI_USERNAME` (default `admin`) and `E2E_UI_PASSWORD` (default the master key) ```bash -export LITELLM_PROXY_URL=http://localhost:4000 -export E2E_DATABASE_METRICS_URLS=http://localhost:4000/metrics -export E2E_PERF_TESTS=1 -export E2E_UI_PERF_DISPOSABLE=1 -export E2E_RESET_SPEND_LOGS=0 -export E2E_ARTIFACT_DIR="$PWD/ui-query-results" -mkdir -p "$E2E_ARTIFACT_DIR" -uv run pytest tests/e2e/perf/test_ui_queries.py --reruns 0 --junitxml="$E2E_ARTIFACT_DIR/junit.xml" -q +export E2E_PERF_TESTS=1 E2E_UI_PERF_DISPOSABLE=1 E2E_RESET_SPEND_LOGS=0 +export DATABASE_URL=postgresql://...@db-perf-perf-.litellm.svc:5432/litellm +export E2E_DATABASE_METRICS_URLS=http://:4000/metrics +export E2E_ARTIFACT_DIR=/tmp/e2e-artifacts +uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ARTIFACT_DIR/junit.xml" -q ``` -Do not enable SpendLog retention or run other traffic on this stack. The `quiet_stack` marker coordinates local pytest callers but cannot stop external traffic. Use one pytest worker for comparable timings +`E2E_UI_JOURNEY_TIMEOUT_SECONDS` (default 120) bounds one journey. A journey that never settles is recorded as a `timeout` sample and a journey whose page rendered empty or threw in the browser as a `ui` sample. The test fails when any sample failed, or when the p50 or p90 across all samples is over the journey's `Ceiling` in `UI_JOURNEY_LIMITS` (`setup/ui_journeys.py`, 10s/20s for Logs journeys, 20s/40s for Usage and Caching, 5s/10s for the Models and Keys tables, generous until the first 1TB run sets real numbers). The artifacts are written before the assertion so a failed run still ships its samples -`E2E_UI_QUERY_REPEATS` defaults to 9, giving ten measured requests including the first, and `E2E_UI_QUERY_TIMEOUT_SECONDS` defaults to 120. Every request is issued once without transport retries. A failed first request stops that case's repeated samples. Cursor prerequisites are recorded separately, and failed prerequisites stop that case's measurements +## Artifacts -## Reading results +Each JUnit testcase `test_ui_journey_latency[]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-.json` holds every sample with its on-screen row count and `ui-isolation-.jsonl` the SQL isolation evidence -Each case writes `ui-query-.json` with its exact method, path, parameters and body, response status counts, elapsed times, row counts, errors and before/after PostgreSQL failure counters. Response bodies and authentication headers are not saved - -The dashboard's `perf_ui_seconds_p50`, `_p90` and `_max` include all successful measured requests, including the first. `perf_ui_samples` is their sample count. p50 is the median and p90 uses nearest rank, the ninth sorted value with ten successes. Failures and cursor prerequisites contribute no latency. Pass percentage is `(perf_ui_requests - perf_ui_failures) / perf_ui_requests * 100`, including cursor prerequisites in the request counts - -The first-request and repeated-request metrics are retained for existing consumers. This is not a cold-cache claim: previous cases and previous runs can warm database buffers or application caches. Restart the disposable proxy between independent runs when comparing independent runs - -Repeated p50 is reported for successful samples, p95 only with at least 20 and p99 only with at least 100. First-request latency is omitted when that request fails. HTTP, JSON, response-shape, nonempty-result, cursor failures are recorded, never asserted. Endpoint failures, database counter increases and latency do not fail CI - -Each JUnit testcase `test_ui_query_performance[]` carries `perf_ui_requests`, `perf_ui_failures`, `perf_ui_repeated_samples`, and nonzero `perf_ui_error_` counts. Codes distinguish `http_400`, `http_500`, `http_503`, other HTTP statuses, `timeout`, `network`, `invalid_json` and `contract`. Successful latency properties are `perf_ui_first_seconds` and `perf_ui_repeated_seconds_p50`, `_p95`, `_p99`, `_max`. A failed cursor prerequisite is counted once and contributes no latency - -Before every case the runner requires three idle database observations and stable failure counters. After each case it cancels leftover SQL, terminates idle transactions and waits for idle and stable counters again. `ui-isolation-.jsonl` retains active SQL, waits, elapsed times, raw samples and counter observations. `perf_ui_database_failures` measures the increase before cleanup, while `perf_ui_cleanup_database_failures` measures the increase observed afterward, including cancellations and potentially delayed request failures. Isolation, instrumentation or setup failures still stop the run rather than publish contaminated measurements. Disk and process caches are not reset between cases - -The Buildkite `ui_queries` lane restores the pinned snapshot independently, waits for migrations and indexes to complete, runs these cases with no retries, uploads JUnit and JSON artifacts, and removes the temporary resources. It follows the nightly regular perf and database-upgrade builds using the same LiteLLM SHA. Shinkiro ingestion is a separate change - -Validate a small restore first. Small-database numbers validate the runner only, not full-size performance. Run the same cases against the completed full-size snapshot before publishing large-database results +The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot diff --git a/tests/e2e/perf/setup/ui_journeys.py b/tests/e2e/perf/setup/ui_journeys.py new file mode 100644 index 00000000000..e9d4360d1a4 --- /dev/null +++ b/tests/e2e/perf/setup/ui_journeys.py @@ -0,0 +1,397 @@ +from __future__ import annotations + +import math +import re +import statistics +import time +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime +from itertools import chain +from types import MappingProxyType +from typing import Final + +from playwright.sync_api import Error as PlaywrightError +from playwright.sync_api import Locator, Page, Response, expect, sync_playwright +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError +from prometheus_client.parser import text_string_to_metric_families +from pydantic import BaseModel, ConfigDict + +MONEY: Final = re.compile(r"^\$[\d,]+\.\d{2}$") +LOADING_LOGS: Final = "Loading request logs…" +LOADING_MODELS: Final = "Loading models…" +LOADING_KEYS: Final = "Loading keys..." + + +class Sample(BaseModel): + model_config = ConfigDict(frozen=True) + + seconds: float + rows: int = 0 + error: str = "" + + +class Timings(BaseModel): + model_config = ConfigDict(frozen=True) + + first_seconds: float | None + repeated_samples: int + repeated_p50_seconds: float | None + repeated_p95_seconds: float | None + repeated_p99_seconds: float | None + repeated_max_seconds: float | None + + +@dataclass(frozen=True, slots=True) +class Journey: + """One dashboard interaction. ``start`` brings the page to the state the user acts from (untimed), + ``act`` is the click or navigation being measured and ``settled`` blocks until the data the user + was waiting for is on screen, returning the number of rows or tiles that rendered.""" + + name: str + surface: str + start: Callable[[Page, float], None] + act: Callable[[Page], None] + settled: Callable[[Page, float], int] + + +@dataclass(frozen=True, slots=True) +class Dashboard: + base_url: str + username: str + password: str + anchor: datetime + + +def summarize(samples: tuple[Sample, ...]) -> Timings: + assert samples, "No UI journeys were measured" + repeated: Final = tuple(sorted(sample.seconds for sample in samples[1:] if not sample.error)) + return Timings( + first_seconds=samples[0].seconds if not samples[0].error else None, + repeated_samples=len(repeated), + repeated_p50_seconds=statistics.median(repeated) if repeated else None, + repeated_p95_seconds=repeated[math.ceil(len(repeated) * 0.95) - 1] if len(repeated) >= 20 else None, + repeated_p99_seconds=repeated[math.ceil(len(repeated) * 0.99) - 1] if len(repeated) >= 100 else None, + repeated_max_seconds=max(repeated) if repeated else None, + ) + + +def error_code(sample: Sample) -> str: + if not sample.error: + return "" + return "timeout" if sample.error == "timeout" else "ui" + + +def ui_junit_properties(samples: tuple[Sample, ...]) -> tuple[tuple[str, float], ...]: + codes: Final = tuple(error_code(sample) for sample in samples if sample.error) + timings: Final = summarize(samples) + successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error)) + latency: Final = ( + ("seconds_p50", statistics.median(successful) if successful else None), + ("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None), + ("seconds_max", max(successful) if successful else None), + ("first_seconds", timings.first_seconds), + ("repeated_seconds_p50", timings.repeated_p50_seconds), + ("repeated_seconds_p95", timings.repeated_p95_seconds), + ("repeated_seconds_p99", timings.repeated_p99_seconds), + ("repeated_seconds_max", timings.repeated_max_seconds), + ) + return ( + ("perf_ui_journeys", len(samples)), + ("perf_ui_failures", len(codes)), + ("perf_ui_samples", len(successful)), + ("perf_ui_repeated_samples", timings.repeated_samples), + *((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))), + *((f"perf_ui_{name}", value) for name, value in latency if value is not None), + ) + + +@dataclass(frozen=True, slots=True) +class Ceiling: + p50_seconds: float + p90_seconds: float + + +LOGS_CEILING: Final = Ceiling(p50_seconds=10, p90_seconds=20) +CHART_CEILING: Final = Ceiling(p50_seconds=20, p90_seconds=40) +TABLE_CEILING: Final = Ceiling(p50_seconds=5, p90_seconds=10) +UI_JOURNEY_LIMITS: Final = MappingProxyType( + { + "logs_open_24h": LOGS_CEILING, + "logs_last_7_days": LOGS_CEILING, + "logs_next_page": LOGS_CEILING, + "logs_filter_key_hash": LOGS_CEILING, + "logs_filter_key_alias": LOGS_CEILING, + "logs_filter_team": LOGS_CEILING, + "logs_filter_model": LOGS_CEILING, + "usage_open": CHART_CEILING, + "usage_last_30_days": CHART_CEILING, + "usage_team_view": CHART_CEILING, + "usage_customer_view": CHART_CEILING, + "usage_model_activity_30_days": CHART_CEILING, + "cache_open": CHART_CEILING, + "models_open": TABLE_CEILING, + "keys_open": TABLE_CEILING, + } +) + + +def assert_within_ceiling(journey_name: str, samples: tuple[Sample, ...]) -> None: + failed: Final = tuple(sample.error for sample in samples if sample.error) + assert not failed, f"{len(failed)} of {len(samples)} {journey_name} samples failed: {failed[0]}" + metrics: Final = dict(ui_junit_properties(samples)) + ceiling: Final = UI_JOURNEY_LIMITS[journey_name] + assert metrics["perf_ui_seconds_p50"] <= ceiling.p50_seconds, ( + f"{journey_name} p50 {metrics['perf_ui_seconds_p50']:.1f}s over {ceiling.p50_seconds}s" + ) + assert metrics["perf_ui_seconds_p90"] <= ceiling.p90_seconds, ( + f"{journey_name} p90 {metrics['perf_ui_seconds_p90']:.1f}s over {ceiling.p90_seconds}s" + ) + + +@contextmanager +def dashboard_page(dashboard: Dashboard, timeout: float) -> Generator[Page]: + """A logged-in admin tab whose clock reads the seed anchor, so relative windows such as + Last 24 Hours land on seeded rows. Live Tail is off so its 15 second refetch cannot land + inside a sample. Playwright keeps a UTC timezone to match the seed.""" + with sync_playwright() as playwright: + browser: Final = playwright.chromium.launch() + context: Final = browser.new_context( + base_url=dashboard.base_url, timezone_id="UTC", viewport={"width": 1600, "height": 1000} + ) + context.set_default_timeout(timeout * 1000) + context.add_init_script('sessionStorage.setItem("isLiveTail", "false")') + page: Final = context.new_page() + page.clock.set_fixed_time(dashboard.anchor) + _login(page, dashboard) + try: + yield page + finally: + context.close() + browser.close() + + +def _login(page: Page, dashboard: Dashboard) -> None: + _ = page.goto("/ui/login") + page.get_by_placeholder("Enter your username").fill(dashboard.username) + page.get_by_placeholder("Enter your password").fill(dashboard.password) + page.get_by_role("button", name="Login", exact=True).click() + page.wait_for_url(lambda url: "/ui" in url and "/login" not in url) + expect(page.get_by_label("LiteLLM home")).to_be_visible() + + +def _is_data_fetch(response: Response) -> bool: + return response.request.resource_type in ("fetch", "xhr") and "/_next/" not in response.url + + +def measure(page: Page, journey: Journey, timeout: float) -> Sample: + """Wall time from the user's action to the data being on screen. Every sample starts from the + same pre-action state so repeats measure the same journey, not a warmed client cache.""" + try: + journey.start(page, timeout) + except (PlaywrightError, AssertionError) as error: + return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300]) + started: Final = time.monotonic() + try: + with page.expect_response(_is_data_fetch, timeout=timeout * 1000): + journey.act(page) + rows: Final = journey.settled(page, timeout) + except PlaywrightTimeoutError: + return Sample(seconds=time.monotonic() - started, error="timeout") + except (PlaywrightError, AssertionError) as error: + return Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]) + return Sample(seconds=time.monotonic() - started, rows=rows) + + +def _visible(page: Page, test_id: str) -> Locator: + return page.get_by_test_id(test_id).filter(visible=True).first + + +def _logs_rows(page: Page) -> Locator: + return page.locator("table").filter(visible=True).first.locator("tbody tr") + + +def _logs_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + rows: Final = _logs_rows(page).filter(has_not_text="No results") + expect(rows.first).to_be_visible(timeout=millis) + expect(_visible(page, "datatable-refresh")).to_be_enabled(timeout=millis) + expect(page.get_by_text(LOADING_LOGS)).to_have_count(0, timeout=millis) + return rows.count() + + +def _no_skeletons(page: Page, timeout: float) -> None: + expect(page.locator('[data-slot="skeleton"]')).to_have_count(0, timeout=timeout * 1000) + + +def _usage_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + expect(page.get_by_text("Total Spend", exact=True).first).to_be_visible(timeout=millis) + _no_skeletons(page, timeout) + money: Final = page.get_by_text(MONEY) + expect(money.first).to_be_visible(timeout=millis) + return money.count() + + +def _cache_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + expect(page.get_by_text("Cache Hits", exact=True)).to_be_visible(timeout=millis) + bars: Final = page.locator(".recharts-bar-rectangle") + expect(bars.first).to_be_visible(timeout=millis) + return bars.count() + + +def _table_settled(loading: str) -> Callable[[Page, float], int]: + def settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + rows: Final = page.locator("[data-row-id]") + expect(rows.first).to_be_visible(timeout=millis) + expect(page.get_by_text(loading)).to_have_count(0, timeout=millis) + return rows.count() + + return settled + + +def _open_logs(page: Page, timeout: float) -> None: + _ = page.goto("/ui/logs") + _ = _logs_settled(page, timeout) + + +def _open_usage(page: Page, timeout: float) -> None: + _ = page.goto("/ui/usage") + _ = _usage_settled(page, timeout) + + +def _filters_drawer(page: Page) -> Locator: + _visible(page, "datatable-filters-trigger").click() + drawer: Final = page.get_by_role("dialog", name="Filters") + expect(drawer).to_be_visible() + return drawer + + +def _apply_filters(page: Page) -> None: + page.get_by_role("dialog", name="Filters").get_by_role("button", name="Apply Filters").click() + + +def _pick_first_option(page: Page, combobox: str) -> None: + _filters_drawer(page).get_by_role("combobox", name=combobox).click() + page.get_by_role("option").first.click() + + +def _open_time_range(page: Page, timeout: float) -> None: + _open_logs(page, timeout) + page.get_by_role("button", name="Last 24 Hours").click() + + +def _open_usage_range(page: Page, timeout: float) -> None: + _open_usage(page, timeout) + page.locator('[data-slot="advanced-date-picker-trigger"]').click() + + +def _model_activity_settled(page: Page, timeout: float) -> int: + millis: Final = timeout * 1000 + expect(page.locator('[role="tab"][data-active]', has_text="Model Activity")).to_be_visible(timeout=millis) + _no_skeletons(page, timeout) + money: Final = page.get_by_text(MONEY).filter(visible=True) + expect(money.first).to_be_visible(timeout=millis) + return money.count() + + +def _open_model_activity_range(page: Page, timeout: float) -> None: + _open_usage(page, timeout) + page.get_by_role("tab", name="Model Activity").click() + _ = _model_activity_settled(page, timeout) + page.locator('[data-slot="advanced-date-picker-trigger"]').click() + + +def _pick_usage_preset(preset: str) -> Callable[[Page], None]: + def act(page: Page) -> None: + page.locator('[data-slot="advanced-date-picker-preset"]', has_text=preset).click() + + return act + + +def _select_usage_view(page: Page, view: str) -> None: + page.locator('[data-slot="select-trigger"]').filter(has_text="Usage").first.click() + page.get_by_role("option", name=re.compile(rf"^{re.escape(view)}")).click() + + +def _nothing(page: Page, timeout: float) -> None: + return None + + +def _goto(path: str) -> Callable[[Page], None]: + def act(page: Page) -> None: + _ = page.goto(path) + + return act + + +def _logs_filter(name: str, prepare: Callable[[Page], None]) -> Journey: + def start(page: Page, timeout: float) -> None: + _open_logs(page, timeout) + prepare(page) + + return Journey(name, "logs", start, _apply_filters, _logs_settled) + + +def journeys(api_key: str) -> tuple[Journey, ...]: + return ( + Journey("logs_open_24h", "logs", _nothing, _goto("/ui/logs"), _logs_settled), + Journey( + "logs_last_7_days", + "logs", + _open_time_range, + lambda page: page.get_by_role("button", name="Last 7 Days", exact=True).click(), + _logs_settled, + ), + Journey( + "logs_next_page", "logs", _open_logs, lambda page: _visible(page, "pagination-next").click(), _logs_settled + ), + _logs_filter( + "logs_filter_key_hash", + lambda page: _filters_drawer(page).get_by_placeholder("Enter key hash…").fill(api_key), + ), + _logs_filter("logs_filter_key_alias", lambda page: _pick_first_option(page, "Search a key alias")), + _logs_filter("logs_filter_team", lambda page: _pick_first_option(page, "Search or select a team")), + _logs_filter("logs_filter_model", lambda page: _pick_first_option(page, "Search a model")), + Journey("usage_open", "usage", _nothing, _goto("/ui/usage"), _usage_settled), + Journey("usage_last_30_days", "usage", _open_usage_range, _pick_usage_preset("Last 30 days"), _usage_settled), + Journey( + "usage_team_view", "usage", _open_usage, lambda page: _select_usage_view(page, "Team Usage"), _usage_settled + ), + Journey( + "usage_customer_view", + "usage", + _open_usage, + lambda page: _select_usage_view(page, "Customer Usage"), + _usage_settled, + ), + Journey( + "usage_model_activity_30_days", + "usage", + _open_model_activity_range, + _pick_usage_preset("Last 30 days"), + _model_activity_settled, + ), + Journey("cache_open", "caching", _nothing, _goto("/ui/caching"), _cache_settled), + Journey("models_open", "models", _nothing, _goto("/ui/models-and-endpoints"), _table_settled(LOADING_MODELS)), + Journey("keys_open", "keys", _nothing, _goto("/ui/api-keys"), _table_settled(LOADING_KEYS)), + ) + + +def database_failures(text: str) -> float: + families: Final = tuple(text_string_to_metric_families(text)) + required: Final = frozenset(("litellm_postgres_failed_requests", "litellm_batch_write_to_db_failed_requests")) + missing: Final = required - frozenset(family.name for family in families) + all_samples: Final = tuple(chain.from_iterable(family.samples for family in families)) + has_activity: Final = any( + sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0 + for sample in all_samples + ) + assert not missing or has_activity, f"Missing PostgreSQL failure counters {missing}" + names: Final = frozenset(f"{name}_total" for name in required) + values: Final = tuple(sample.value for sample in all_samples if sample.name in names) + assert all(math.isfinite(value) and value >= 0 for value in values), "Invalid PostgreSQL failure counter" + return sum(values) diff --git a/tests/e2e/perf/setup/ui_queries.py b/tests/e2e/perf/setup/ui_queries.py deleted file mode 100644 index b6945d79896..00000000000 --- a/tests/e2e/perf/setup/ui_queries.py +++ /dev/null @@ -1,227 +0,0 @@ -from __future__ import annotations - -import math -import statistics -import time -from collections.abc import Callable -from dataclasses import dataclass -from itertools import chain -from typing import Final, Literal -from urllib.parse import urlencode - -from e2e_http import ( - NetworkError, - RateLimitedError, - Result, - Success, - UnauthorizedError, - UnknownApiError, - ValidationError, - without_retries, -) -from e2e_metadata import step -from prometheus_client.parser import text_string_to_metric_families -from pydantic import BaseModel, ConfigDict, JsonValue, RootModel -from transport import HttpTransport - - -class Query(BaseModel): - model_config = ConfigDict(frozen=True) - - -class Document(RootModel[JsonValue]): - model_config = ConfigDict(frozen=True) - - -@dataclass(frozen=True, slots=True) -class Contract: - collection: tuple[str, ...] = () - row_fields: tuple[str, ...] = () - positive: tuple[str, ...] = () - object_fields: tuple[str, ...] = () - excluded_ids: frozenset[str] = frozenset() - - def count(self, document: Document) -> int: - value: Final = descend(document.root, self.collection) - if self.object_fields: - assert isinstance(value, dict), "Expected a JSON object" - assert set(self.object_fields) <= value.keys(), f"Missing fields {self.object_fields}" - return 1 - assert isinstance(value, list) and value, f"Empty/missing collection {self.collection}, check the fixture" - for row in value: - if self.excluded_ids: - assert isinstance(row, dict) and isinstance(row_id := row.get("request_id"), str), "Missing request_id" - assert row_id not in self.excluded_ids, "Repeated page-one row" - if self.row_fields or self.positive: - assert isinstance(row, dict), "Expected object rows" - assert set(self.row_fields) <= row.keys(), f"Missing row fields {self.row_fields}" - for name in self.positive: - assert any(tuple(_positive(row, name) for row in value)), f"Expected positive {name}" - return len(value) - - -def _positive(row: JsonValue, name: str) -> bool: - assert isinstance(row, dict), "Expected object row" - number: Final = row.get(name) - assert isinstance(number, (int, float)) and not isinstance(number, bool), f"Expected numeric {name}" - assert math.isfinite(number), f"Expected finite {name}" - return number > 0 - - -def descend(value: JsonValue, path: tuple[str, ...]) -> JsonValue: - if not path: - return value - assert isinstance(value, dict) and path[0] in value, f"Missing response path {path}" - return descend(value[path[0]], path[1:]) - - -@dataclass(frozen=True, slots=True) -class UiQueryCase: - name: str - path: str - params: BaseModel - contract: Contract - surface: Literal["current"] = "current" - body: BaseModel | None = None - cursor: Literal["none", "session"] = "none" - - -class Sample(BaseModel): - model_config = ConfigDict(frozen=True) - - seconds: float - status: int - rows: int = 0 - error: str = "" - - -class Timings(BaseModel): - model_config = ConfigDict(frozen=True) - - first_seconds: float | None - repeated_samples: int - repeated_p50_seconds: float | None - repeated_p95_seconds: float | None - repeated_p99_seconds: float | None - repeated_max_seconds: float | None - - -def summarize(samples: tuple[Sample, ...]) -> Timings: - assert samples, "No UI requests were measured" - repeated: Final = tuple(sorted(sample.seconds for sample in samples[1:] if not sample.error)) - return Timings( - first_seconds=samples[0].seconds if not samples[0].error else None, - repeated_samples=len(repeated), - repeated_p50_seconds=statistics.median(repeated) if repeated else None, - repeated_p95_seconds=repeated[math.ceil(len(repeated) * 0.95) - 1] if len(repeated) >= 20 else None, - repeated_p99_seconds=repeated[math.ceil(len(repeated) * 0.99) - 1] if len(repeated) >= 100 else None, - repeated_max_seconds=max(repeated) if repeated else None, - ) - - -def error_code(sample: Sample) -> str: - if not sample.error: - return "" - if sample.status > 0 and sample.status != 200: - return f"http_{sample.status}" - if sample.error in ("timeout", "network"): - return sample.error - if sample.error == "Response is not valid JSON": - return "invalid_json" - return "contract" - - -def ui_junit_properties( - samples: tuple[Sample, ...], prerequisite: Sample | None = None -) -> tuple[tuple[str, float], ...]: - requests: Final = (prerequisite, *samples) if prerequisite is not None and not prerequisite.error else samples - codes: Final = tuple(error_code(sample) for sample in requests if sample.error) - timings: Final = summarize(samples) - successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error)) - latency: Final = ( - ("seconds_p50", statistics.median(successful) if successful else None), - ("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None), - ("seconds_max", max(successful) if successful else None), - ("first_seconds", timings.first_seconds), - ("repeated_seconds_p50", timings.repeated_p50_seconds), - ("repeated_seconds_p95", timings.repeated_p95_seconds), - ("repeated_seconds_p99", timings.repeated_p99_seconds), - ("repeated_seconds_max", timings.repeated_max_seconds), - ) - return ( - ("perf_ui_requests", len(requests)), - ("perf_ui_failures", len(codes)), - ("perf_ui_samples", len(successful)), - ("perf_ui_repeated_samples", timings.repeated_samples), - *((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))), - *((f"perf_ui_{name}", value) for name, value in latency if value is not None), - ) - - -def _failure(result: Result[Document], seconds: float) -> Sample: - match result: - case Success(status_code=status): - return Sample(seconds=seconds, status=status, error=f"Unexpected HTTP {status}") - case UnknownApiError(status_code=status): - return Sample(seconds=seconds, status=status, error=f"HTTP {status}") - case UnauthorizedError(): - return Sample(seconds=seconds, status=401, error="Unauthorized") - case RateLimitedError(): - return Sample(seconds=seconds, status=429, error="Rate limited") - case NetworkError(message=message): - return Sample(seconds=seconds, status=-1, error="timeout" if "timed out" in message.lower() else "network") - case ValidationError(): - return Sample(seconds=seconds, status=-1, error="Response is not valid JSON") - - -def measure( - issue: Callable[[], Result[Document]], - contract: Contract, - *, - clock: Callable[[], float] = time.monotonic, -) -> tuple[Sample, Document | None]: - started: Final = clock() - with without_retries(): - result: Final = issue() - seconds: Final = clock() - started - match result: - case Success(status_code=200, data=document): - try: - count: Final = contract.count(document) - except AssertionError as error: - return Sample(seconds=seconds, status=200, error=str(error)), document - return Sample(seconds=seconds, status=200, rows=count), document - case _: - return _failure(result, seconds), None - - -def query_path(case: UiQueryCase) -> str: - query: Final = urlencode(case.params.model_dump(by_alias=True, exclude_none=True), doseq=True) - return f"{case.path}?{query}" if query else case.path - - -@step("Measure UI query {case.name} at {case.path}") -def issue_query(transport: HttpTransport, case: UiQueryCase, *, timeout: float) -> Result[Document]: - if case.body is not None: - return transport.post( - case.path, headers=transport.master, json=case.body, response_type=Document, timeout=timeout - ) - return transport.get( - query_path(case), headers=transport.master, params=Query(), response_type=Document, timeout=timeout - ) - - -def database_failures(text: str) -> float: - families: Final = tuple(text_string_to_metric_families(text)) - required: Final = frozenset(("litellm_postgres_failed_requests", "litellm_batch_write_to_db_failed_requests")) - missing: Final = required - frozenset(family.name for family in families) - all_samples: Final = tuple(chain.from_iterable(family.samples for family in families)) - has_activity: Final = any( - sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0 - for sample in all_samples - ) - assert not missing or has_activity, f"Missing PostgreSQL failure counters {missing}" - names: Final = frozenset(f"{name}_total" for name in required) - values: Final = tuple(sample.value for sample in all_samples if sample.name in names) - assert all(math.isfinite(value) and value >= 0 for value in values), "Invalid PostgreSQL failure counter" - return sum(values) diff --git a/tests/e2e/perf/setup/ui_query_cases.py b/tests/e2e/perf/setup/ui_query_cases.py deleted file mode 100644 index e9b70aad713..00000000000 --- a/tests/e2e/perf/setup/ui_query_cases.py +++ /dev/null @@ -1,206 +0,0 @@ -from __future__ import annotations - -from dataclasses import replace -from datetime import datetime, timedelta -from itertools import chain -from typing import Final, Literal - -from pydantic import ConfigDict - -from setup.ui_queries import Contract, Document, Query, UiQueryCase, descend - - -class Params(Query): - model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) - - start_date: str | None = None - timezone: str | None = None - offset: int | None = None - model_group: str | None = None - by_model_group: str | None = None - end_date: str | None = None - page: int | None = None - page_size: int | None = None - sort_by: str | None = None - sort_order: str | None = None - group_by_session: Literal["true"] | None = None - session_cursor: str | None = None - api_key: str | None = None - team_id: str | None = None - team_ids: str | None = None - user_id: str | None = None - model: str | None = None - model_id: str | None = None - key_alias: str | None = None - search: str | None = None - status_filter: str | None = None - cache_hit_filter: str | None = None - error_code: str | None = None - error_message: str | None = None - span_type: str | None = None - exclude_internal_health_checks: Literal["true"] | None = None - key_aliases: tuple[str, ...] | None = None - models: tuple[str, ...] | None = None - limit: int | None = None - tags: str | None = None - exclude_team_ids: str | None = None - include_current_utc_day: Literal["true"] | None = None - - -def _timestamp(value: datetime) -> str: - return value.strftime("%Y-%m-%d %H:%M:%S") - - -def next_page(case: UiQueryCase, document: Document) -> UiQueryCase: - params: Final = Params.model_validate(case.params.model_dump()) - rows: Final = descend(document.root, case.contract.collection) - assert isinstance(rows, list), "Missing page-one rows" - ids: Final = frozenset(str(row["request_id"]) for row in rows if isinstance(row, dict)) - contract: Final = replace(case.contract, excluded_ids=ids) - cursor: Final = descend(document.root, ("next_session_cursor",)) - assert isinstance(cursor, str) and cursor, "Missing session cursor" - return replace(case, params=params.model_copy(update={"page": 2, "session_cursor": cursor}), contract=contract) - - -_LOGS: Final = Contract(("data",), ("request_id", "startTime", "metadata")) - - -def log_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: - params: Final = Params( - start_date=_timestamp(anchor - timedelta(days=1)), - end_date=_timestamp(anchor), - page=1, - page_size=25, - sort_by="startTime", - sort_order="desc", - group_by_session="true", - ) - base: Final = UiQueryCase("logs_default_24h", "/spend/logs/ui", params, _LOGS) - filtered: Final = params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=30))}) - filters: Final = ( - ("api_key", api_key), - ("team_id", "perf-team-0"), - ("user_id", "perf-user-0"), - ("model", "perf-chat"), - ("model_id", "perf-model-0"), - ("key_alias", "perf-key-0"), - ("status_filter", "failure"), - ("cache_hit_filter", "hit"), - ("error_code", "429"), - ("error_message", "Synthetic"), - ("span_type", "llm"), - ("exclude_internal_health_checks", "true"), - ) - return ( - base, - replace(base, name="logs_cursor_page_2", params=params.model_copy(update={"page": 2}), cursor="session"), - replace( - base, - name="logs_7d", - params=params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=7))}), - ), - replace( - base, - name="logs_30d", - params=params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=30))}), - ), - *( - replace(base, name=f"logs_{name}_30d", params=filtered.model_copy(update={name: value})) - for name, value in filters - ), - *( - replace(base, name=f"logs_sort_{name}", params=params.model_copy(update={"sort_by": name})) - for name in ("spend", "total_tokens", "request_duration_ms", "model", "ttft_ms") - ), - ) - - -def analytics_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: - dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), end_date=anchor.date().isoformat() - ) - cache: Final = UiQueryCase( - "cache_activity_7d", "/global/activity/cache_hits", dates, Contract(("groups",), ("call_type",)) - ) - return ( - UiQueryCase( - "team_users_all_teams", - "/team/spend/by_user", - dates.model_copy(update={"team_ids": ",".join(f"perf-team-{i}" for i in range(100))}), - Contract(("results",), ("team_id", "user_id", "spend"), ("api_requests",)), - ), - UiQueryCase( - "team_users_filtered", - "/team/spend/by_user", - dates.model_copy(update={"team_ids": "perf-team-0"}), - Contract(("results",), ("team_id", "user_id", "spend"), ("api_requests",)), - ), - cache, - replace( - cache, - name="cache_activity_filtered", - params=dates.model_copy(update={"key_aliases": ("perf-key-0", "perf-key-1"), "models": ("perf-chat",)}), - ), - ) - - -def _entity_activity_cases(anchor: datetime, entity: str) -> tuple[UiQueryCase, ...]: - dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), - end_date=anchor.date().isoformat(), - timezone="0", - exclude_team_ids="litellm-dashboard" if entity == "team" else None, - ) - routes: Final = ( - ("aggregated", dates, Contract(("results",), ("date", "breakdown"))), - ( - "aggregated/keys", - dates.model_copy(update={"offset": 0, "limit": 50}), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ( - "aggregated/search", - dates.model_copy(update={"search": "perf-key"}), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ( - "aggregated/model_top_keys", - dates.model_copy(update={"model_group": "perf-chat", "by_model_group": "true"}), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ) - return tuple( - UiQueryCase( - f"usage_{entity}_{route.replace('/', '_')}", - f"/{entity}/daily/activity/{route}", - params, - contract, - ) - for route, params, contract in routes - ) - - -def activity_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: - groups: Final = tuple(_entity_activity_cases(anchor, entity) for entity in ("user", "team", "tag", "customer")) - return ( - *chain.from_iterable(groups), - UiQueryCase( - "cache_leakage_keys", - "/user/daily/activity/aggregated/cache_leakage_keys", - Params( - start_date=(anchor - timedelta(days=30)).date().isoformat(), - end_date=anchor.date().isoformat(), - timezone="0", - include_current_utc_day="true", - ), - Contract(("api_keys",), ("api_key", "metrics", "metadata")), - ), - ) - - -def cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: - return ( - *log_cases(anchor, api_key), - *analytics_cases(anchor), - *activity_cases(anchor), - ) diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py index e3f5236ba47..1cbb5c41ea3 100644 --- a/tests/e2e/perf/setup/ui_query_fixture.py +++ b/tests/e2e/perf/setup/ui_query_fixture.py @@ -3,7 +3,7 @@ from __future__ import annotations import hashlib import os import time -from datetime import datetime +from datetime import datetime, timedelta from typing import Final import psycopg @@ -20,6 +20,45 @@ class UiFixture(BaseModel): api_key: str +SEEDED_TEAMS: Final = 100 +SEEDED_USERS: Final = 2000 +SEEDED_KEYS: Final = 5000 +SEEDED_MODELS: Final = 200 +DAILY_TABLES: Final = ("LiteLLM_DailyUserSpend", "LiteLLM_DailyTeamSpend") +_SEED_SQL: Final = ( + """ + INSERT INTO "LiteLLM_TeamTable" (team_id, team_alias, members_with_roles, metadata, model_spend, model_max_budget) + SELECT 'perf-team-' || n, 'perf-team-' || n, '[]', '{}', '{}', '{}' FROM generate_series(0, %(teams)s - 1) AS n + ON CONFLICT (team_id) DO NOTHING + """, + """ + INSERT INTO "LiteLLM_UserTable" (user_id, user_alias, user_email, user_role, team_id, metadata, model_spend, model_max_budget) + SELECT 'perf-user-' || n, 'perf-user-' || n, 'perf-user-' || n || '@example.com', 'internal_user', + 'perf-team-' || (n %% %(teams)s), '{}', '{}', '{}' + FROM generate_series(0, %(users)s - 1) AS n + ON CONFLICT (user_id) DO NOTHING + """, + """ + INSERT INTO "LiteLLM_VerificationToken" (token, key_name, key_alias, team_id, user_id, aliases, config, permissions, + metadata, model_spend, model_max_budget, created_at, updated_at) + SELECT md5('perf-key:' || n) || md5('perf-key-tail:' || n), 'sk-...perf' || n, 'perf-key-' || n, + 'perf-team-' || (n %% %(teams)s), 'perf-user-' || (n %% %(users)s), '{}', '{}', '{}', '{}', '{}', '{}', + %(anchor)s - make_interval(days => n %% 365), %(anchor)s + FROM generate_series(0, %(keys)s - 1) AS n + ON CONFLICT (token) DO NOTHING + """, + """ + INSERT INTO "LiteLLM_ProxyModelTable" (model_id, model_name, litellm_params, model_info, created_by, updated_by) + SELECT 'perf-model-' || n, 'perf-chat-' || (n %% 20), + jsonb_build_object('model', 'openai/perf-chat', 'api_key', 'perf-seed', + 'api_base', 'os.environ/E2E_PERF_MOCK_OPENAI_URL'), + jsonb_build_object('id', 'perf-model-' || n, 'db_model', true), 'perf-seed', 'perf-seed' + FROM generate_series(0, %(models)s - 1) AS n + ON CONFLICT (model_id) DO NOTHING + """, +) + + def prepare_fixture() -> UiFixture: validate_target(os.environ["DATABASE_URL"]) _finish_upgrade() @@ -27,6 +66,7 @@ def prepare_fixture() -> UiFixture: _ = connection.execute("SET statement_timeout='30s'") row: Final = connection.execute("SELECT spec->>'anchor', rows FROM litellm_perf_seed.checkpoint").fetchone() assert row is not None and TypeAdapter(int).validate_python(row[1]) > 0, "Missing completed seed rows" + seed_ui_tables(connection, anchor=TypeAdapter(datetime).validate_python(row[0])) fixture: Final = UiFixture( anchor=TypeAdapter(datetime).validate_python(row[0]), api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest() @@ -35,6 +75,36 @@ def prepare_fixture() -> UiFixture: return fixture +def seed_ui_tables(connection: psycopg.Connection, anchor: datetime) -> None: + """Give the Keys, Teams, Users and Models pages realistic row counts on the disposable restore. + + Names follow the SpendLogs seed (perf-key-N aliases, perf-team-N, perf-user-N) so the Logs filters and + Team Usage line up with real spend. The Daily* rollups come from the snapshot itself and are only checked. + """ + window: Final = { + "since": (anchor - timedelta(days=7)).strftime("%Y-%m-%d"), + "until": anchor.strftime("%Y-%m-%d"), + } + for table in DAILY_TABLES: + row = connection.execute(f'SELECT count(*) FROM "{table}" WHERE date BETWEEN %(since)s AND %(until)s', window) + assert _count(row.fetchone()) > 0, f"{table} has no rows in the week before the anchor" + params: Final = { + "teams": SEEDED_TEAMS, + "users": SEEDED_USERS, + "keys": SEEDED_KEYS, + "models": SEEDED_MODELS, + "anchor": anchor, + } + for statement in _SEED_SQL: + _ = connection.execute(statement, params) + connection.commit() + + +def _count(row: tuple[object, ...] | None) -> int: + assert row is not None + return TypeAdapter(int).validate_python(row[0]) + + def _finish_upgrade() -> None: if os.environ.get("E2E_DATABASE_UPGRADE") != "1": return diff --git a/tests/e2e/perf/setup/ui_query_isolation.py b/tests/e2e/perf/setup/ui_query_isolation.py index d7f6d725edc..d892c268fa2 100644 --- a/tests/e2e/perf/setup/ui_query_isolation.py +++ b/tests/e2e/perf/setup/ui_query_isolation.py @@ -62,7 +62,9 @@ def wait_idle(connection: psycopg.Connection[dict[str, object]], evidence: Path, for query in queries: if cancel and query.leader_pid is None: _ = connection.execute( - "SELECT pg_cancel_backend(%s)" if query.state == "active" else "SELECT pg_terminate_backend(%s)", + "SELECT pg_cancel_backend(%s)" + if query.state == "active" + else "SELECT pg_terminate_backend(%s)", (query.pid,), ) else: diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py new file mode 100644 index 00000000000..888a44d2e27 --- /dev/null +++ b/tests/e2e/perf/test_ui_journeys.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +import os +from datetime import UTC, datetime +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME +from e2e_http import send_text_external, without_retries +from setup.ui_journeys import ( + assert_within_ceiling, + Dashboard, + Journey, + Sample, + dashboard_page, + database_failures, + error_code, + journeys, + measure, + summarize, + ui_junit_properties, +) +from setup.ui_query_fixture import UiFixture, prepare_fixture +from setup.ui_query_isolation import isolated_database, settled_counter +from transport import HttpTransport + +pytestmark = [ + pytest.mark.perf, + pytest.mark.quiet_stack, + pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"), +] + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "journey_name" in metafunc.fixturenames: + metafunc.parametrize("journey_name", [journey.name for journey in journeys("")]) + + +@pytest.fixture(scope="module") +def ui_fixture() -> UiFixture: + return prepare_fixture() + + +def _metrics(transport: HttpTransport) -> float: + urls: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip()) + try: + assert urls, "Provide a metrics URL for every proxy worker/pod" + readings: Final = tuple(send_text_external("GET", url, headers=transport.master, timeout=10) for url in urls) + assert all(reading.status_code == 200 for reading in readings), "Cannot read database failure metrics" + return sum(database_failures(reading.body) for reading in readings) + except Exception as error: + pytest.exit(f"Database metrics unavailable ({type(error).__name__}). Stopping UI measurements", returncode=2) + + +def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]: + with dashboard_page(dashboard, timeout) as page: + return tuple(measure(page, journey, timeout) for _ in range(repeats + 1)) + + +def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None: + journey: Final = next(journey for journey in journeys(ui_fixture.api_key) if journey.name == journey_name) + transport: Final = HttpTransport(base_url=UI_BASE_URL, master_key=MASTER_KEY) + timeout: Final = float(os.environ.get("E2E_UI_JOURNEY_TIMEOUT_SECONDS", "120")) + repeats: Final = int(os.environ.get("E2E_UI_JOURNEY_REPEATS", "9")) + assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings" + target: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) + assert target.is_dir(), "E2E_ARTIFACT_DIR must exist" + evidence: Final = target / f"ui-isolation-{journey.name}.jsonl" + dashboard: Final = Dashboard( + base_url=UI_BASE_URL, username=UI_USERNAME, password=UI_PASSWORD, anchor=ui_fixture.anchor + ) + with without_retries(): + with isolated_database(journey.name): + before: Final = settled_counter(lambda: _metrics(transport), evidence) + samples: Final = _samples(dashboard, journey, timeout, repeats) + with evidence.open("a", encoding="utf-8") as sample_output: + _ = sample_output.write( + json.dumps({"samples": tuple(sample.model_dump() for sample in samples)}) + "\n" + ) + after_requests: Final = _metrics(transport) + item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped + assert isinstance(item, pytest.Item) + item.user_properties.extend(ui_junit_properties(samples)) + after: Final = settled_counter(lambda: _metrics(transport), evidence) + item.user_properties.extend( + ( + ("perf_ui_database_failures", after_requests - before), + ("perf_ui_cleanup_database_failures", after - after_requests), + ) + ) + artifact: Final = { + "name": journey.name, + "surface": journey.surface, + "measured_at": datetime.now(UTC).isoformat(), + "samples": [sample.model_dump() for sample in samples], + "error_codes": [error_code(sample) for sample in samples], + "timings": summarize(samples).model_dump(), + "timeouts": sum(sample.error == "timeout" for sample in samples), + "failures": sum(bool(sample.error) for sample in samples), + "database_failures_before": before, + "database_failures_after_requests": after_requests, + "database_failures_after_cleanup": after, + } + output: Final = target / f"ui-journey-{journey.name}.json" + _ = output.write_text(json.dumps(artifact, indent=2), encoding="utf-8") + assert_within_ceiling(journey.name, samples) diff --git a/tests/e2e/perf/test_ui_queries.py b/tests/e2e/perf/test_ui_queries.py deleted file mode 100644 index 75ac4d3055f..00000000000 --- a/tests/e2e/perf/test_ui_queries.py +++ /dev/null @@ -1,159 +0,0 @@ -from __future__ import annotations - -import json -import os -from dataclasses import dataclass, replace -from datetime import UTC, datetime -from pathlib import Path -from typing import Final - -import pytest -from e2e_config import CONTROL_PLANE_BASE_URL, MASTER_KEY -from e2e_http import send_text_external, without_retries -from setup.ui_queries import ( - Sample, - UiQueryCase, - database_failures, - error_code, - issue_query, - measure, - summarize, - ui_junit_properties, -) -from setup.ui_query_cases import Params, cases, next_page -from setup.ui_query_fixture import UiFixture, prepare_fixture -from setup.ui_query_isolation import isolated_database, settled_counter -from transport import HttpTransport - -pytestmark = [ - pytest.mark.perf, - pytest.mark.quiet_stack, - pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"), -] - - -def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: - if "ui_case_name" not in metafunc.fixturenames: - return - if os.environ.get("E2E_UI_PERF_DISPOSABLE") == "1" and os.environ.get("E2E_RESET_SPEND_LOGS") == "1": - raise pytest.UsageError("UI performance tests cannot run with E2E_RESET_SPEND_LOGS=1") - names: Final = tuple(case.name for case in cases(datetime(2000, 1, 1, tzinfo=UTC), "collection-only")) - metafunc.parametrize("ui_case_name", names, ids=names) - - -@pytest.fixture(scope="module") -def ui_fixture() -> UiFixture: - return prepare_fixture() - - -def _metrics(transport: HttpTransport) -> float: - urls: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip()) - try: - assert urls, "Provide a metrics URL for every proxy worker/pod" - readings: Final = tuple(send_text_external("GET", url, headers=transport.master, timeout=10) for url in urls) - assert all(reading.status_code == 200 for reading in readings), "Cannot read database failure metrics" - return sum(database_failures(reading.body) for reading in readings) - except Exception as error: - pytest.exit(f"Database metrics unavailable ({type(error).__name__}). Stopping UI measurements", returncode=2) - - -@dataclass(frozen=True, slots=True) -class Prepared: - case: UiQueryCase - prerequisite: Sample | None = None - - -def _page_two(transport: HttpTransport, case: UiQueryCase, timeout: float) -> Prepared: - if case.cursor == "none": - return Prepared(case) - params: Final = Params.model_validate(case.params.model_dump()).model_copy(update={"page": 1}) - first: Final = replace(case, params=params) - sample, document = measure(lambda: issue_query(transport, first, timeout=timeout), case.contract) - if sample.error or document is None: - return Prepared(case, sample) - try: - return Prepared(next_page(case, document), sample) - except AssertionError as error: - return Prepared(case, sample.model_copy(update={"error": str(error)})) - - -def _samples(transport: HttpTransport, case: UiQueryCase, timeout: float, repeats: int) -> tuple[Sample, ...]: - measured, _ = measure(lambda: issue_query(transport, case, timeout=timeout), case.contract) - if measured.error: - return (measured,) - repeated: Final = tuple( - measure(lambda: issue_query(transport, case, timeout=timeout), case.contract)[0] for _ in range(repeats) - ) - return (measured, *repeated) - - -def test_ui_query_performance(ui_case_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None: - case: Final = next(case for case in cases(ui_fixture.anchor, ui_fixture.api_key) if case.name == ui_case_name) - transport: Final = HttpTransport(base_url=CONTROL_PLANE_BASE_URL, master_key=MASTER_KEY) - timeout: Final = float(os.environ.get("E2E_UI_QUERY_TIMEOUT_SECONDS", "120")) - repeats: Final = int(os.environ.get("E2E_UI_QUERY_REPEATS", "9")) - assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings" - target: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) - assert target.is_dir(), "E2E_ARTIFACT_DIR must exist" - evidence: Final = target / f"ui-isolation-{case.name}.jsonl" - with without_retries(): - with isolated_database(case.name): - before: Final = settled_counter(lambda: _metrics(transport), evidence) - paginated: Final = _page_two(transport, case, timeout) - samples: Final = ( - (paginated.prerequisite,) - if paginated.prerequisite is not None and paginated.prerequisite.error - else _samples(transport, paginated.case, timeout, repeats) - ) - with evidence.open("a", encoding="utf-8") as sample_output: - _ = sample_output.write( - json.dumps( - { - "samples": tuple(sample.model_dump() for sample in samples), - "prerequisite": paginated.prerequisite.model_dump() - if paginated.prerequisite is not None - else None, - } - ) - + "\n" - ) - after_requests: Final = _metrics(transport) - item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped - assert isinstance(item, pytest.Item) - item.user_properties.extend(ui_junit_properties(samples, paginated.prerequisite)) - after: Final = settled_counter(lambda: _metrics(transport), evidence) - item.user_properties.extend( - ( - ("perf_ui_database_failures", after_requests - before), - ("perf_ui_cleanup_database_failures", after - after_requests), - ) - ) - timings: Final = summarize(samples) - artifact: Final = { - "name": case.name, - "surface": case.surface, - "path": case.path, - "method": "POST" if case.body is not None else "GET", - "params": paginated.case.params.model_dump(mode="json", by_alias=True, exclude_none=True), - "prerequisite": paginated.prerequisite.model_dump() if paginated.prerequisite is not None else None, - "prerequisite_failed": paginated.prerequisite is not None and bool(paginated.prerequisite.error), - "body": case.body.model_dump(mode="json", exclude_none=True) if case.body is not None else None, - "samples": [sample.model_dump() for sample in samples], - "error_codes": [error_code(sample) for sample in samples], - "timings": timings.model_dump(), - "timeouts": sum(sample.error == "timeout" for sample in samples), - "failures": sum(bool(sample.error) for sample in samples), - "status_counts": { - str(status): sum(sample.status == status for sample in samples) - for status in frozenset(sample.status for sample in samples) - }, - "validation_failures": sum( - bool(sample.error) and (sample.status == 200 or sample.error == "Response is not valid JSON") - for sample in samples - ), - "database_failures_before": before, - "database_failures_after_requests": after_requests, - "database_failures_after_cleanup": after, - } - output: Final = target / f"ui-query-{case.name}.json" - _ = output.write_text(json.dumps(artifact, indent=2), encoding="utf-8") diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.py new file mode 100644 index 00000000000..7d5b0d531ec --- /dev/null +++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from typing import Final + +import pytest +from setup.ui_journeys import ( + UI_JOURNEY_LIMITS, + Sample, + assert_within_ceiling, + database_failures, + journeys, + summarize, + ui_junit_properties, +) + + +def test_first_journey_is_separate_from_repeated_latency() -> None: + samples: Final = (Sample(seconds=80), *(Sample(seconds=i) for i in range(1, 101))) + result: Final = summarize(samples) + assert result.first_seconds == 80 + assert result.repeated_samples == 100 + assert result.repeated_p50_seconds == 50.5 + assert result.repeated_p95_seconds == 95 + assert result.repeated_p99_seconds == 99 + assert result.repeated_max_seconds == 100 + + +def test_failed_repeats_do_not_look_like_fast_successes() -> None: + result: Final = summarize((Sample(seconds=10), Sample(seconds=0.01, error="AssertionError: no rows"))) + assert result.repeated_samples == 0 + assert result.repeated_p50_seconds is None + assert result.repeated_p95_seconds is None + assert result.repeated_p99_seconds is None + assert result.repeated_max_seconds is None + + +def test_small_sample_does_not_report_tail_percentiles() -> None: + result: Final = summarize(tuple(Sample(seconds=1) for _ in range(6))) + assert result.repeated_samples == 5 + assert result.repeated_p50_seconds == 1 + assert result.repeated_p95_seconds is None + assert result.repeated_p99_seconds is None + + +def test_failed_first_journey_emits_error_code_without_latency() -> None: + sample: Final = Sample(seconds=120, error="timeout") + assert summarize((sample,)).first_seconds is None + assert dict(ui_junit_properties((sample,))) == { + "perf_ui_journeys": 1, + "perf_ui_failures": 1, + "perf_ui_samples": 0, + "perf_ui_repeated_samples": 0, + "perf_ui_error_timeout": 1, + } + + +def test_junit_counts_failures_but_only_measures_successful_latency() -> None: + samples: Final = ( + Sample(seconds=2, rows=25), + Sample(seconds=3, rows=25), + Sample(seconds=120, error="timeout"), + Sample(seconds=1, error="AssertionError: Logs table rendered empty"), + Sample(seconds=4, rows=25), + ) + assert dict(ui_junit_properties(samples)) == { + "perf_ui_journeys": 5, + "perf_ui_failures": 2, + "perf_ui_samples": 3, + "perf_ui_repeated_samples": 2, + "perf_ui_error_timeout": 1, + "perf_ui_error_ui": 1, + "perf_ui_seconds_p50": 3, + "perf_ui_seconds_p90": 4, + "perf_ui_seconds_max": 4, + "perf_ui_first_seconds": 2, + "perf_ui_repeated_seconds_p50": 3.5, + "perf_ui_repeated_seconds_max": 4, + } + + +def test_all_journey_percentiles_include_first_and_use_nearest_rank_p90() -> None: + samples: Final = tuple(Sample(seconds=seconds) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5)) + metrics: Final = dict(ui_junit_properties(samples)) + assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == { + "perf_ui_seconds_p50": 5.5, + "perf_ui_seconds_p90": 9, + "perf_ui_seconds_max": 10, + } + assert metrics["perf_ui_samples"] == 10 + + +def test_journeys_have_unique_names_and_cover_every_dashboard_surface() -> None: + names: Final = tuple(journey.name for journey in journeys("key")) + assert len(names) == len(frozenset(names)) + assert frozenset(journey.surface for journey in journeys("key")) == {"logs", "usage", "caching", "models", "keys"} + + +def _metrics(postgres: str, batch: str) -> str: + return ( + "# TYPE litellm_postgres_failed_requests_total counter\n" + f'litellm_postgres_failed_requests_total{{call_type="query_raw"}} {postgres}\n' + "# TYPE litellm_batch_write_to_db_failed_requests_total counter\n" + f'litellm_batch_write_to_db_failed_requests_total{{call_type="batch"}} {batch}\n' + ) + + +def test_postgres_failures_include_background_batch_writes() -> None: + assert database_failures(_metrics("0", "0")) == 0 + assert database_failures(_metrics("2", "3")) == 5 + + +def test_lazy_failure_counters_require_evidence_of_database_activity() -> None: + assert ( + database_failures( + "# TYPE litellm_postgres_total_requests_total counter\nlitellm_postgres_total_requests_total 3\n" + ) + == 0 + ) + + +@pytest.mark.parametrize( + "text", ("", "# TYPE unrelated counter\nunrelated_total 0\n", _metrics("NaN", "0"), _metrics("-1", "0")) +) +def test_missing_or_invalid_metrics_cannot_pass_as_zero(text: str) -> None: + with pytest.raises(AssertionError): + _ = database_failures(text) + + +def test_every_journey_has_a_latency_ceiling() -> None: + assert frozenset(journey.name for journey in journeys("key")) == frozenset(UI_JOURNEY_LIMITS) + + +def test_failed_sample_fails_the_journey_before_latency_is_judged() -> None: + with pytest.raises(AssertionError, match="1 of 2 logs_open_24h samples failed: timeout"): + assert_within_ceiling("logs_open_24h", (Sample(seconds=0.2, rows=25), Sample(seconds=120, error="timeout"))) + + +def test_slow_journey_fails_its_ceiling() -> None: + fast: Final = tuple(Sample(seconds=1, rows=25) for _ in range(9)) + assert_within_ceiling("logs_open_24h", fast) + with pytest.raises(AssertionError, match=r"logs_open_24h p90 25\.0s over 20s"): + assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25))) + with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"): + assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3))) diff --git a/tests/e2e_harness/perf/setup/test_ui_queries.py b/tests/e2e_harness/perf/setup/test_ui_queries.py deleted file mode 100644 index f24d03c6bba..00000000000 --- a/tests/e2e_harness/perf/setup/test_ui_queries.py +++ /dev/null @@ -1,257 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Final -from urllib.parse import parse_qs, urlsplit - -import pytest -from e2e_http import NetworkError, Result, Success, UnknownApiError -from setup.ui_queries import ( - Contract, - Document, - Sample, - database_failures, - measure, - query_path, - summarize, - ui_junit_properties, -) -from setup.ui_query_cases import Params, cases, next_page - - -def test_query_string_preserves_ui_arrays_and_boolean_strings() -> None: - requests: Final = cases(datetime(2026, 1, 1, tzinfo=UTC), "key") - cache: Final = next(case for case in requests if case.name == "cache_activity_filtered") - assert parse_qs(urlsplit(query_path(cache)).query) == { - "start_date": ["2025-12-25"], - "end_date": ["2026-01-01"], - "key_aliases": ["perf-key-0", "perf-key-1"], - "models": ["perf-chat"], - } - health: Final = next(case for case in requests if case.name == "logs_exclude_internal_health_checks_30d") - query: Final = parse_qs(urlsplit(query_path(health)).query) - assert query["exclude_internal_health_checks"] == ["true"] - assert query["group_by_session"] == ["true"] - - -@pytest.mark.parametrize("entity", ("user", "team", "tag", "customer")) -@pytest.mark.parametrize( - ("route", "extra"), - ( - ("aggregated", {}), - ("aggregated/keys", {"offset": 0, "limit": 50}), - ("aggregated/search", {"search": "perf-key"}), - ("aggregated/model_top_keys", {"model_group": "perf-chat", "by_model_group": "true"}), - ), -) -def test_usage_requests_list_keys_without_a_single_key_filter( - entity: str, route: str, extra: dict[str, str | int] -) -> None: - case: Final = next( - case - for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") - if case.path == f"/{entity}/daily/activity/{route}" - ) - assert case.params.model_dump(exclude_none=True) == { - "start_date": "2025-12-25", - "end_date": "2026-01-01", - "timezone": "0", - **({"exclude_team_ids": "litellm-dashboard"} if entity == "team" else {}), - **extra, - } - - -@pytest.mark.parametrize(("name", "value"), (("api_key", "key"), ("key_alias", "perf-key-0"))) -def test_key_filters_request_30_days_of_logs(name: str, value: str) -> None: - case: Final = next( - case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.name == f"logs_{name}_30d" - ) - assert parse_qs(urlsplit(query_path(case)).query) == { - "start_date": ["2025-12-02 00:00:00"], - "end_date": ["2026-01-01 00:00:00"], - "page_size": ["25"], - "page": ["1"], - "sort_by": ["startTime"], - "sort_order": ["desc"], - "group_by_session": ["true"], - name: [value], - } - - -@pytest.mark.parametrize("invalid", ("not-a-number", True, float("nan"), float("inf"))) -def test_positive_row_does_not_hide_a_malformed_later_row(invalid: str | bool | float) -> None: - with pytest.raises(AssertionError, match=r"Expected (numeric|finite) spend"): - Contract(positive=("spend",)).count(Document([{"spend": 1}, {"spend": invalid}])) - - -def test_session_cursor_is_reused_and_repeated_rows_rejected() -> None: - case: Final = next(case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.cursor == "session") - first: Final = Document.model_validate_json('{"data":[{"request_id":"one"}],"next_session_cursor":"opaque"}') - second: Final = next_page(case, first) - assert Params.model_validate(second.params).session_cursor == "opaque" - assert Params.model_validate(second.params).page == 2 - with pytest.raises(AssertionError, match="Repeated page-one row"): - _ = second.contract.count(first) - - -def test_first_query_is_separate_from_repeated_latency() -> None: - samples: Final = (Sample(seconds=80, status=200), *(Sample(seconds=i, status=200) for i in range(1, 101))) - result: Final = summarize(samples) - assert result.first_seconds == 80 - assert result.repeated_samples == 100 - assert result.repeated_p50_seconds == 50.5 - assert result.repeated_p95_seconds == 95 - assert result.repeated_p99_seconds == 99 - assert result.repeated_max_seconds == 100 - - -def test_failed_repeats_do_not_look_like_fast_successes() -> None: - result: Final = summarize((Sample(seconds=10, status=200), Sample(seconds=0.01, status=503, error="HTTP 503"))) - assert result.repeated_samples == 0 - assert result.repeated_p50_seconds is None - assert result.repeated_p95_seconds is None - assert result.repeated_p99_seconds is None - assert result.repeated_max_seconds is None - - -def test_small_sample_does_not_report_tail_percentiles() -> None: - result: Final = summarize(tuple(Sample(seconds=1, status=200) for _ in range(6))) - assert result.repeated_samples == 5 - assert result.repeated_p50_seconds == 1 - assert result.repeated_p95_seconds is None - assert result.repeated_p99_seconds is None - - -def test_failed_first_request_emits_error_code_without_latency() -> None: - sample: Final = Sample(seconds=30, status=500, error="HTTP 500") - assert summarize((sample,)).first_seconds is None - assert dict(ui_junit_properties((sample,))) == { - "perf_ui_requests": 1, - "perf_ui_failures": 1, - "perf_ui_samples": 0, - "perf_ui_repeated_samples": 0, - "perf_ui_error_http_500": 1, - } - - -def test_junit_counts_failures_but_only_measures_successful_latency() -> None: - samples: Final = ( - Sample(seconds=2, status=200), - Sample(seconds=3, status=200), - Sample(seconds=30, status=-1, error="timeout"), - Sample(seconds=30, status=503, error="HTTP 503"), - Sample(seconds=1, status=200, error="Empty collection"), - ) - assert dict(ui_junit_properties(samples, Sample(seconds=4, status=200))) == { - "perf_ui_requests": 6, - "perf_ui_failures": 3, - "perf_ui_samples": 2, - "perf_ui_seconds_p50": 2.5, - "perf_ui_seconds_p90": 3, - "perf_ui_seconds_max": 3, - "perf_ui_repeated_samples": 1, - "perf_ui_error_timeout": 1, - "perf_ui_error_http_503": 1, - "perf_ui_error_contract": 1, - "perf_ui_first_seconds": 2, - "perf_ui_repeated_seconds_p50": 3, - "perf_ui_repeated_seconds_max": 3, - } - - -def test_failed_cursor_prerequisite_is_counted_once() -> None: - sample: Final = Sample(seconds=30, status=503, error="HTTP 503") - assert dict(ui_junit_properties((sample,), sample)) == dict(ui_junit_properties((sample,))) - - -def test_all_request_percentiles_include_first_and_use_nearest_rank_p90() -> None: - samples: Final = tuple(Sample(seconds=seconds, status=200) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5)) - metrics: Final = dict(ui_junit_properties(samples)) - assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == { - "perf_ui_seconds_p50": 5.5, - "perf_ui_seconds_p90": 9, - "perf_ui_seconds_max": 10, - } - assert metrics["perf_ui_samples"] == 10 - - -@pytest.mark.parametrize("body", ("{}", '{"data": []}', '{"data": [{"spend": 1}]}', '{"data": [null]}')) -def test_empty_or_malformed_rows_fail_even_with_http_200(body: str) -> None: - ticks: Final = iter((10.0, 12.5)) - result, _ = measure( - lambda: Success(status_code=200, data=Document.model_validate_json(body)), - Contract(("data",), ("request_id",)), - clock=lambda: next(ticks), - ) - assert result.status == 200 - assert result.seconds == 2.5 - assert result.rows == 0 - assert result.error - - -def test_valid_response_records_elapsed_seconds_and_rows() -> None: - ticks: Final = iter((5.0, 5.125)) - result, document = measure( - lambda: Success(status_code=200, data=Document.model_validate_json('{"data":[{"request_id":"one"}]}')), - Contract(("data",), ("request_id",)), - clock=lambda: next(ticks), - ) - assert result == Sample(seconds=0.125, status=200, rows=1) - assert document is not None - - -def test_zero_spend_rows_are_allowed_alongside_nonzero_fixture_data() -> None: - contract: Final = Contract(positive=("spend",)) - assert contract.count(Document.model_validate_json('[{"spend":0},{"spend":2}]')) == 2 - with pytest.raises(AssertionError, match="positive spend"): - _ = contract.count(Document.model_validate_json('[{"spend":0}]')) - - -@pytest.mark.parametrize( - "failure,error,status", - ( - (UnknownApiError(status_code=500, body="sensitive error detail"), "HTTP 500", 500), - (NetworkError(message="Read timed out"), "timeout", -1), - (NetworkError(message="connection refused"), "network", -1), - ), -) -def test_failed_requests_are_recorded_without_response_bodies( - failure: Result[Document], - error: str, - status: int, -) -> None: - ticks: Final = iter((1.0, 2.0)) - result, document = measure(lambda: failure, Contract(), clock=lambda: next(ticks)) - assert result == Sample(seconds=1, status=status, error=error) - assert document is None - - -def _metrics(postgres: str, batch: str) -> str: - return ( - "# TYPE litellm_postgres_failed_requests_total counter\n" - f'litellm_postgres_failed_requests_total{{call_type="query_raw"}} {postgres}\n' - "# TYPE litellm_batch_write_to_db_failed_requests_total counter\n" - f'litellm_batch_write_to_db_failed_requests_total{{call_type="batch"}} {batch}\n' - ) - - -def test_postgres_failures_include_background_batch_writes() -> None: - assert database_failures(_metrics("0", "0")) == 0 - assert database_failures(_metrics("2", "3")) == 5 - - -def test_lazy_failure_counters_require_evidence_of_database_activity() -> None: - assert ( - database_failures( - "# TYPE litellm_postgres_total_requests_total counter\nlitellm_postgres_total_requests_total 3\n" - ) - == 0 - ) - - -@pytest.mark.parametrize( - "text", ("", "# TYPE unrelated counter\nunrelated_total 0\n", _metrics("NaN", "0"), _metrics("-1", "0")) -) -def test_missing_or_invalid_metrics_cannot_pass_as_zero(text: str) -> None: - with pytest.raises(AssertionError): - _ = database_failures(text) diff --git a/tests/e2e_harness/perf/test_upgrade.py b/tests/e2e_harness/perf/test_upgrade.py index 0904b7a7b5a..89e3a620f48 100644 --- a/tests/e2e_harness/perf/test_upgrade.py +++ b/tests/e2e_harness/perf/test_upgrade.py @@ -308,9 +308,7 @@ def test_invalid_200_is_not_an_allowed_non_200(failed: WireRecord) -> None: def test_upgrade_verdict_uses_regular_limits_without_cpu_or_datastore_samples( failures: int, overhead: float, ttft: float, growth: float, expected: tuple[str, ...] ) -> None: - nonstream: Final = tuple( - record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100) - ) + nonstream: Final = tuple(record(i).model_copy(update={"total_s": overhead, "mock_delay_s": 0}) for i in range(100)) stream: Final = record(100).model_copy(update={"stream": True, "first_token_s": ttft}) observed: Final = Observation( records=(*nonstream, stream, *(failed_record(i) for i in range(101, 101 + failures))),