From 17db64ef78aa2a3d8517422b4a2a1636c0cdd409 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 17:10:39 -0700 Subject: [PATCH] test(perf): record the row count and size of each table the Admin UI journeys query (#47683) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/setup/ui_query_fixture.py | 54 +++++++++++++++++++ tests/e2e/perf/test_ui_journeys.py | 5 +- .../perf/setup/test_ui_journeys.py | 20 ++++++- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py index cfffa0822e8..ed941a096bd 100644 --- a/tests/e2e/perf/setup/ui_query_fixture.py +++ b/tests/e2e/perf/setup/ui_query_fixture.py @@ -3,7 +3,9 @@ from __future__ import annotations import hashlib import os import time +from collections.abc import Iterable from datetime import datetime, timedelta +from itertools import chain from typing import Final import psycopg @@ -18,6 +20,33 @@ class UiFixture(BaseModel): anchor: AwareDatetime api_key: str + """`perf_ui_db_*` JUnit properties naming how much data the journeys ran against.""" + volume: tuple[tuple[str, int], ...] = () + + +class TableVolume(BaseModel): + model_config = ConfigDict(frozen=True) + + table: str + rows: int + bytes: int + + +QUERIED_TABLES: Final = ( + "LiteLLM_SpendLogs", + "LiteLLM_DailyUserSpend", + "LiteLLM_DailyTeamSpend", + "LiteLLM_DailyTagSpend", + "LiteLLM_VerificationToken", + "LiteLLM_TeamTable", + "LiteLLM_UserTable", + "LiteLLM_ProxyModelTable", +) +_VOLUME_SQL: Final = """ + SELECT c.relname, GREATEST(c.reltuples, 0)::bigint, pg_total_relation_size(c.oid) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = ANY(%(tables)s) +""" SEEDED_TEAMS: Final = 100 @@ -59,6 +88,30 @@ _SEED_SQL: Final = ( ) +def volume_properties(volumes: Iterable[TableVolume]) -> tuple[tuple[str, int], ...]: + """One `perf_ui_db_rows_` and one `perf_ui_db_bytes_
` property per table the journeys read, + so the dashboard can say what the latencies were measured against.""" + return tuple( + chain.from_iterable( + ((f"perf_ui_db_rows_{v.table}", v.rows), (f"perf_ui_db_bytes_{v.table}", v.bytes)) for v in volumes + ) + ) + + +def table_volumes(connection: psycopg.Connection, spend_log_rows: int) -> tuple[TableVolume, ...]: + """Planner row estimates and on-disk size (indexes included) of the queried tables. The restore's + `LiteLLM_SpendLogs` count comes from the seed checkpoint, exact where the estimate can be stale.""" + adapter: Final = TypeAdapter(tuple[str, int, int]) + found: Final = tuple( + adapter.validate_python(row) + for row in connection.execute(_VOLUME_SQL, {"tables": list(QUERIED_TABLES)}).fetchall() + ) + return tuple( + TableVolume(table=table, rows=spend_log_rows if table == "LiteLLM_SpendLogs" else rows, bytes=size) + for table, rows, size in sorted(found) + ) + + def prepare_fixture() -> UiFixture: validate_target(os.environ["DATABASE_URL"]) _finish_upgrade() @@ -71,6 +124,7 @@ def prepare_fixture() -> UiFixture: anchor=TypeAdapter(datetime).validate_python(row[0]), api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest() + hashlib.md5(b"perf-key-tail:0", usedforsecurity=False).hexdigest(), + volume=volume_properties(table_volumes(connection, TypeAdapter(int).validate_python(row[1]))), ) return fixture diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py index db25608a00d..63eefedec68 100644 --- a/tests/e2e/perf/test_ui_journeys.py +++ b/tests/e2e/perf/test_ui_journeys.py @@ -11,14 +11,14 @@ from types import FrameType from typing import Final import pytest -from playwright.sync_api import Page from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME from e2e_http import send_text_external, without_retries +from playwright.sync_api import Page from setup.ui_journeys import ( - assert_within_ceiling, Dashboard, Journey, Sample, + assert_within_ceiling, dashboard_page, database_failures, error_code, @@ -111,6 +111,7 @@ def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: p item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped assert isinstance(item, pytest.Item) item.user_properties.extend(ui_junit_properties(samples)) + item.user_properties.extend(ui_fixture.volume) after: Final = settled_counter(lambda: _metrics(transport), evidence) item.user_properties.extend( ( diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.py index dd2c1fc0301..d008368052c 100644 --- a/tests/e2e_harness/perf/setup/test_ui_journeys.py +++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py @@ -6,9 +6,9 @@ import pytest from setup.ui_journeys import ( UI_JOURNEY_LIMITS, Sample, - error_code, assert_within_ceiling, database_failures, + error_code, journeys, summarize, ui_junit_properties, @@ -153,3 +153,21 @@ def test_failed_data_request_status_becomes_the_error_code() -> None: metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung))) assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1 assert "perf_ui_error_ui" not in metrics + + +def test_volume_properties_name_rows_and_bytes_per_queried_table() -> None: + from setup.ui_query_fixture import TableVolume, volume_properties + + properties: Final = volume_properties( + ( + TableVolume(table="LiteLLM_SpendLogs", rows=850_000_000, bytes=1_000_000_000_000), + TableVolume(table="LiteLLM_VerificationToken", rows=5_000, bytes=4_194_304), + ) + ) + + assert properties == ( + ("perf_ui_db_rows_LiteLLM_SpendLogs", 850_000_000), + ("perf_ui_db_bytes_LiteLLM_SpendLogs", 1_000_000_000_000), + ("perf_ui_db_rows_LiteLLM_VerificationToken", 5_000), + ("perf_ui_db_bytes_LiteLLM_VerificationToken", 4_194_304), + )