From 280496b9dd2dbb7245ed3a5da22477074f7b518b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 12:49:58 -0700 Subject: [PATCH] test(perf): let the UI journey fixture run on a restore upgraded by a released migrations image (#47628) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/UI_QUERIES.md | 2 ++ tests/e2e/perf/setup/ui_query_fixture.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md index 9bf82c8c07e..d2cbcd368fc 100644 --- a/tests/e2e/perf/UI_QUERIES.md +++ b/tests/e2e/perf/UI_QUERIES.md @@ -54,4 +54,6 @@ Each JUnit testcase `test_ui_journey_latency[]` carries `perf_ui_journeys` The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot +To measure a released version on the same snapshot, create the litellm-perf build with `perf_gateway_version=` (for example `v1.106.1`) next to `litellm_sha`. The gateway, backend and the restore's schema upgrade then come from `ghcr.io/berriai/litellm-{gateway,backend,migrations}:`, where the migrations image runs `prisma migrate deploy` and builds the request-log indexes the way a customer upgrade does, while the runner and these journeys come from `litellm_sha`. The fixture skips its upgrade-worker wait when `_LiteLLMPerfUpgrade` is absent. The snapshot's schema is v1.105.1 and the restore only migrates forward, so older tags cannot be measured + Each sample in `ui-journey-.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane. diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py index 1cbb5c41ea3..cfffa0822e8 100644 --- a/tests/e2e/perf/setup/ui_query_fixture.py +++ b/tests/e2e/perf/setup/ui_query_fixture.py @@ -105,10 +105,19 @@ def _count(row: tuple[object, ...] | None) -> int: return TypeAdapter(int).validate_python(row[0]) +def _released_schema(connection: psycopg.Connection) -> bool: + """A released tag's migrations image upgrades the restore (schema and request-log indexes) before the + stack starts, so there is no upgrade worker state to wait on.""" + row: Final = connection.execute("""SELECT to_regclass('public."_LiteLLMPerfUpgrade"') IS NULL""").fetchone() + return row is None or TypeAdapter(bool).validate_python(row[0]) + + def _finish_upgrade() -> None: if os.environ.get("E2E_DATABASE_UPGRADE") != "1": return with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True, connect_timeout=10) as connection: + if _released_schema(connection): + return release_indexes(connection) deadline: Final = time.monotonic() + 6 * 3600 while time.monotonic() < deadline: