diff --git a/tests/e2e/perf/UI_QUERIES.md b/tests/e2e/perf/UI_QUERIES.md index eabd1f04813..552f1b953fd 100644 --- a/tests/e2e/perf/UI_QUERIES.md +++ b/tests/e2e/perf/UI_QUERIES.md @@ -2,18 +2,47 @@ Run `test_ui_queries.py` against a dedicated proxy and a disposable database restored from the metadata-only performance seed. These tests do not call an LLM or measure browser rendering -The suite measures 87 request shapes across 48 routes. Cases follow callers in `ui/litellm-dashboard/src/components/networking.tsx` and the mounted Logs, Usage, Old Usage, Caching, Prompt Caching, Guardrails, Tool Policies and LiteAdmin components +The suite measures 42 request shapes across 20 routes used by current dashboard screens. It replays direct HTTP requests against the existing bulk seed, with the seed anchor replacing the wall clock and a UTC browser timezone. It does not run a browser or claim to cover every dashboard request -| Artifact surface | Requests | +## UI callers + +All paths below are relative to `ui/litellm-dashboard/src/`. Each case requires an authenticated admin session with access to that screen, and selected resources must exist in the fixture + +| Cases | UI action and source | | --- | --- | -| `current` | Logs list, request detail, session, user/end-user facets, team usage by user, cache activity, prompt-caching requests, guardrail logs and tool logs | -| `old_usage` | Global activity/model activity, provider spend, tags/tag names, logs, keys, teams, models and end-user spend | -| `liteadmin` | Agent-selected global, key and team spend reports, not automatic page-load traffic | -| `key_recovery` | Aggregated, keys, search and model-top-keys routes for user, team, tag, organization, customer and agent, plus user cache-leakage keys | +| `logs_*` | Logs > Request Logs, change date range, filter, sort or next page. `components/logs/request/useLogFilterLogic.ts` and `RequestLogsFilters.tsx` | +| `usage_*` | Usage overview or entity tab, open key activity, search keys, expand a model group. `app/(dashboard)/usage/_components/components/UsagePageView.tsx`, `EntityUsage/EntityUsage.tsx`, `EntityUsage/entityFetchFns.ts`, `components/UsagePage/components/KeyActivityPanel.tsx` | +| `team_users_*` | Team Usage. `EntityUsage/EntityUsage.tsx` passes selected teams or all available non-dashboard teams to `TeamUserSpendCard.tsx` | +| `cache_activity_*` | Caching > Cache Analytics, optionally filter aliases/models. `app/(dashboard)/hooks/caching/useCacheActivity.ts` | +| `cache_leakage_keys` | Cost Optimization, switch cache leakage to Keys. `app/(dashboard)/cost-optimization/_components/useCacheLeakageKeys.ts` | -The ordinary daily-activity queries use daily tables. The `key_recovery` fixture deliberately omits token metadata and verifies the alias recovered from SpendLogs. Daily exports and the user pagination route do not perform this recovery and are excluded. Historical model-metrics routes with no mounted UI caller are also excluded +Request serialization lives in `components/networking.tsx`. Logs use page size 25 and session grouping, including the next-session cursor returned by page one. Individual Logs filters use a 30-day window to exercise larger matching datasets. Their case names end in `_30d` so the measurements do not share history with the old 24-hour filters. The synthetic cursorless page-three request is excluded -Logs cover the default 24-hour window, 7/30-day windows, UI filters/sorts, offset pagination and session cursors. Prompt-caching pagination uses the returned timestamp and request ID. Cursor page two must not repeat page-one IDs +Usage list, search and model drilldown requests do not inject an `api_key` filter. Team activity excludes `litellm-dashboard`. Cache leakage supplies `include_current_utc_day=true`. Dates and filter values are deterministic fixture values, not captured production traffic + +Old Usage is excluded because its expensive-query gate normally hides those calls at this database size. LiteAdmin's agent-selected reports are excluded from this deterministic dashboard set. Missing-token alias recovery is backend functional coverage, not a separate UI action, and its artificial requests and orphan fixtures are removed + +This mapping is source-verified, not browser-recorded. Recheck the named callers when changing cases. Neither browser capture nor an automatic UI-drift checker is installed by this suite + +## Dataset scope + +Logs, team-user spend and cache activity query the bulk SpendLogs data. User, team, tag and customer usage, plus cache leakage, query daily tables aggregated from that dataset. These are sequential large-dataset queries, not concurrent traffic or saturation tests + +A read-only restore of `db-perf-v1-105-1tb` gave exact request totals from daily aggregates: 9,442,606 across 24 hours, 66,077,435 across seven days and 283,189,684 across 30 days. The retained 30-day key filter matches 1,134,559 requests, user filter 11,316,454, team filter 113,298,119 and model filter 94,395,473. These describe matching requests, not rows scanned or returned by the API, and are documentation rather than performance assertions + +PostgreSQL table-wide estimates (`pg_class.reltuples`) are approximately 2.70 million daily user rows, 2.70 million daily team rows, 5.40 million daily tag rows and 78.27 million daily end-user rows. These are not exact counts within each Usage case's seven-day window + +The individual end-user filter is excluded because it matched only 104 requests in 24 hours and 8,616 across all dates. User/end-user dropdown requests are also excluded because their queries cap the input at 10,000 matching SpendLogs before extracting distinct values. Neither is a broad matching-data workload in this seed + +Excluded because they only had tiny matching fixtures or individual-record lookups: organization/agent usage, guardrail/tool logs, request-ID lookup, payload/session drawers, session/search filters for `perf-ui-*`, the OAuth-token filter, and all prompt-caching request cases. The seed has response-cache hits but no prompt-cache usage or injection metadata, so the prompt-caching cases only matched the 320 added rows. No UI-specific rows are inserted anymore + +| Remaining cases | Count | Variations | +| --- | ---: | --- | +| Logs | 21 | 24h/7d/30d, page 2 with session cursor, 12 individual filters over 30 days, five additional descending sorts | +| User/team/tag/customer usage | 16 | Aggregated totals, keys, key search, model top keys for each | +| Team-user spend | 2 | All 100 seeded teams or `perf-team-0` | +| Cache activity | 2 | All or selected aliases/model | +| Cache leakage keys | 1 | 30-day window | ## Deployment prerequisites @@ -21,13 +50,13 @@ Use a fresh proxy against the disposable restore, never the active seed source o Configure `litellm_settings.service_callback: [prometheus_system]` and expose metrics for every proxy worker/pod. Set `E2E_DATABASE_METRICS_URLS` to their comma-separated URLs, not a load-balanced endpoint that can miss a pod. Absent failure counters count as zero only when the endpoint exposes a positive PostgreSQL request counter. Missing instrumentation stops the run -The fixture creates missing legacy views using the proxy's view definitions and `LIMIT 0` existence probes, without reading through existing views. Enable expensive queries on this dedicated proxy with `DISABLE_EXPENSIVE_DB_QUERIES=false` and a `MAX_SPENDLOG_ROWS_TO_QUERY` above the seed's row count. These routes are normally hidden by the UI at this scale +No expensive-query override or legacy view creation is needed for these cases. Keep the normal large-database UI restrictions -The cases use explicit dates from the seed anchor where the UI supplies dates. Some legacy views always use the database clock's last 30 days. An old snapshot with no matching rows records a contract failure, rather than reporting fast empty queries as success. Refresh that fixture explicitly instead of rewriting the baseline timestamps +The cases use explicit dates from the seed anchor where the UI supplies dates. A missing nonempty fixture result records a contract failure rather than reporting a fast empty query as success ## Run -Set `DATABASE_URL` and `LITELLM_MASTER_KEY` securely for this disposable deployment. The URL must name the same database used by the proxy, with hostname `db-perf-perf-.litellm.svc`. The explicit disposable confirmation permits bounded fixture inserts and cancellation of leftover queries in this database. The fixtures add 320 specialized logs, 25 orphan-key logs, daily rows and guardrail/tool index rows. Destroy the disposable database after the run +Set `DATABASE_URL` and `LITELLM_MASTER_KEY` securely for this disposable deployment. The URL must name the same database used by the proxy, with hostname `db-perf-perf-.litellm.svc`. The explicit disposable confirmation permits cancellation of leftover queries in this database. Setup reads the existing seed checkpoint without adding UI-specific rows. Destroy the disposable database after the run ```bash export LITELLM_PROXY_URL=http://localhost:4000 @@ -42,15 +71,17 @@ uv run pytest tests/e2e/perf/test_ui_queries.py --reruns 0 --junitxml="$E2E_ARTI Do not enable SpendLog retention or run other traffic on this stack. The `quiet_stack` marker coordinates local pytest callers but cannot stop external traffic. Use one pytest worker for comparable timings -`E2E_UI_QUERY_REPEATS` defaults to 5 and `E2E_UI_QUERY_TIMEOUT_SECONDS` to 120. Every request is issued once without transport retries. A failed first request stops that case's repeated samples. Cursor prerequisites are recorded separately, and failed prerequisites stop that case's measurements +`E2E_UI_QUERY_REPEATS` defaults to 9, giving ten measured requests including the first, and `E2E_UI_QUERY_TIMEOUT_SECONDS` defaults to 120. Every request is issued once without transport retries. A failed first request stops that case's repeated samples. Cursor prerequisites are recorded separately, and failed prerequisites stop that case's measurements ## Reading results Each case writes `ui-query-.json` with its exact method, path, parameters and body, response status counts, elapsed times, row counts, errors and before/after PostgreSQL failure counters. Response bodies and authentication headers are not saved -The first request is reported separately from repeated requests. This is not a cold-cache claim: previous cases, fixture creation and previous runs can warm database buffers or application caches. Restart the disposable proxy between independent runs when measuring process-cache recovery +The dashboard's `perf_ui_seconds_p50`, `_p90` and `_max` include all successful measured requests, including the first. `perf_ui_samples` is their sample count. p50 is the median and p90 uses nearest rank, the ninth sorted value with ten successes. Failures and cursor prerequisites contribute no latency. Pass percentage is `(perf_ui_requests - perf_ui_failures) / perf_ui_requests * 100`, including cursor prerequisites in the request counts -Repeated p50 is reported for successful samples, p95 only with at least 20 and p99 only with at least 100. First-request latency is omitted when that request fails. HTTP, JSON, response-shape, nonempty-result, cursor and alias-recovery failures are recorded, never asserted. Endpoint failures, database counter increases and latency do not fail CI +The first-request and repeated-request metrics are retained for existing consumers. This is not a cold-cache claim: previous cases and previous runs can warm database buffers or application caches. Restart the disposable proxy between independent runs when comparing independent runs + +Repeated p50 is reported for successful samples, p95 only with at least 20 and p99 only with at least 100. First-request latency is omitted when that request fails. HTTP, JSON, response-shape, nonempty-result, cursor failures are recorded, never asserted. Endpoint failures, database counter increases and latency do not fail CI Each JUnit testcase `test_ui_query_performance[]` carries `perf_ui_requests`, `perf_ui_failures`, `perf_ui_repeated_samples`, and nonzero `perf_ui_error_` counts. Codes distinguish `http_400`, `http_500`, `http_503`, other HTTP statuses, `timeout`, `network`, `invalid_json` and `contract`. Successful latency properties are `perf_ui_first_seconds` and `perf_ui_repeated_seconds_p50`, `_p95`, `_p99`, `_max`. A failed cursor prerequisite is counted once and contributes no latency diff --git a/tests/e2e/perf/setup/ui_queries.py b/tests/e2e/perf/setup/ui_queries.py index 0155b69c2cf..b6945d79896 100644 --- a/tests/e2e/perf/setup/ui_queries.py +++ b/tests/e2e/perf/setup/ui_queries.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import math import statistics import time @@ -8,6 +7,7 @@ from collections.abc import Callable from dataclasses import dataclass from itertools import chain from typing import Final, Literal +from urllib.parse import urlencode from e2e_http import ( NetworkError, @@ -39,12 +39,9 @@ class Contract: row_fields: tuple[str, ...] = () positive: tuple[str, ...] = () object_fields: tuple[str, ...] = () - recovered_alias: str | None = None excluded_ids: frozenset[str] = frozenset() def count(self, document: Document) -> int: - if self.recovered_alias is not None: - assert contains_alias(document.root, self.recovered_alias), "SpendLog key metadata was not recovered" value: Final = descend(document.root, self.collection) if self.object_fields: assert isinstance(value, dict), "Expected a JSON object" @@ -71,16 +68,6 @@ def _positive(row: JsonValue, name: str) -> bool: return number > 0 -def contains_alias(value: JsonValue, alias: str) -> bool: - if isinstance(value, dict): - return value.get("key_alias") == alias or any(contains_alias(child, alias) for child in value.values()) - return isinstance(value, list) and any(contains_alias(child, alias) for child in value) - - -def orphan_key(name: str) -> str: - return hashlib.sha256(f"ui-perf-orphan-{name}".encode()).hexdigest() - - def descend(value: JsonValue, path: tuple[str, ...]) -> JsonValue: if not path: return value @@ -94,9 +81,9 @@ class UiQueryCase: path: str params: BaseModel contract: Contract - surface: Literal["current", "old_usage", "liteadmin", "key_recovery"] = "current" + surface: Literal["current"] = "current" body: BaseModel | None = None - cursor: Literal["none", "session", "prompt_cache"] = "none" + cursor: Literal["none", "session"] = "none" class Sample(BaseModel): @@ -150,7 +137,11 @@ def ui_junit_properties( requests: Final = (prerequisite, *samples) if prerequisite is not None and not prerequisite.error else samples codes: Final = tuple(error_code(sample) for sample in requests if sample.error) timings: Final = summarize(samples) + successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error)) latency: Final = ( + ("seconds_p50", statistics.median(successful) if successful else None), + ("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None), + ("seconds_max", max(successful) if successful else None), ("first_seconds", timings.first_seconds), ("repeated_seconds_p50", timings.repeated_p50_seconds), ("repeated_seconds_p95", timings.repeated_p95_seconds), @@ -160,6 +151,7 @@ def ui_junit_properties( return ( ("perf_ui_requests", len(requests)), ("perf_ui_failures", len(codes)), + ("perf_ui_samples", len(successful)), ("perf_ui_repeated_samples", timings.repeated_samples), *((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))), *((f"perf_ui_{name}", value) for name, value in latency if value is not None), @@ -203,6 +195,11 @@ def measure( return _failure(result, seconds), None +def query_path(case: UiQueryCase) -> str: + query: Final = urlencode(case.params.model_dump(by_alias=True, exclude_none=True), doseq=True) + return f"{case.path}?{query}" if query else case.path + + @step("Measure UI query {case.name} at {case.path}") def issue_query(transport: HttpTransport, case: UiQueryCase, *, timeout: float) -> Result[Document]: if case.body is not None: @@ -210,7 +207,7 @@ def issue_query(transport: HttpTransport, case: UiQueryCase, *, timeout: float) case.path, headers=transport.master, json=case.body, response_type=Document, timeout=timeout ) return transport.get( - case.path, headers=transport.master, params=case.params, response_type=Document, timeout=timeout + query_path(case), headers=transport.master, params=Query(), response_type=Document, timeout=timeout ) diff --git a/tests/e2e/perf/setup/ui_query_cases.py b/tests/e2e/perf/setup/ui_query_cases.py index 11fc3e8db1e..e9b70aad713 100644 --- a/tests/e2e/perf/setup/ui_query_cases.py +++ b/tests/e2e/perf/setup/ui_query_cases.py @@ -5,9 +5,9 @@ from datetime import datetime, timedelta from itertools import chain from typing import Final, Literal -from pydantic import ConfigDict, Field +from pydantic import ConfigDict -from setup.ui_queries import Contract, Document, Query, UiQueryCase, descend, orphan_key +from setup.ui_queries import Contract, Document, Query, UiQueryCase, descend class Params(Query): @@ -23,15 +23,12 @@ class Params(Query): page_size: int | None = None sort_by: str | None = None sort_order: str | None = None - group_by_session: bool | None = None + group_by_session: Literal["true"] | None = None session_cursor: str | None = None - session_id: str | None = None - request_id: str | None = None api_key: str | None = None team_id: str | None = None team_ids: str | None = None user_id: str | None = None - end_user: str | None = None model: str | None = None model_id: str | None = None key_alias: str | None = None @@ -41,26 +38,13 @@ class Params(Query): error_code: str | None = None error_message: str | None = None span_type: str | None = None - exclude_internal_health_checks: bool | None = None - used_client_oauth_token: bool | None = None - lower: str | None = Field(default=None, alias="filter[startTime][gte]") - upper: str | None = Field(default=None, alias="filter[startTime][lte]") - q: str | None = None + exclude_internal_health_checks: Literal["true"] | None = None key_aliases: tuple[str, ...] | None = None models: tuple[str, ...] | None = None - filter: Literal["all", "injected", "hits"] | None = None - cursor_start_time: str | None = None - cursor_request_id: str | None = None - guardrail_id: str | None = None limit: int | None = None tags: str | None = None - group_by: str | None = None - - -class EndUserBody(Query): - startTime: str | None = None - endTime: str | None = None - api_key: str | None = None + exclude_team_ids: str | None = None + include_current_utc_day: Literal["true"] | None = None def _timestamp(value: datetime) -> str: @@ -73,25 +57,12 @@ def next_page(case: UiQueryCase, document: Document) -> UiQueryCase: assert isinstance(rows, list), "Missing page-one rows" ids: Final = frozenset(str(row["request_id"]) for row in rows if isinstance(row, dict)) contract: Final = replace(case.contract, excluded_ids=ids) - if case.cursor == "session": - cursor: Final = descend(document.root, ("next_session_cursor",)) - assert isinstance(cursor, str) and cursor, "Missing session cursor" - return replace(case, params=params.model_copy(update={"page": 2, "session_cursor": cursor}), contract=contract) - cursor_data: Final = descend(document.root, ("next_cursor",)) - assert isinstance(cursor_data, dict), "Missing prompt-cache cursor" - start: Final = cursor_data.get("start_time") - request_id: Final = cursor_data.get("request_id") - assert isinstance(start, str) and isinstance(request_id, str), "Invalid prompt-cache cursor" - return replace( - case, - contract=contract, - params=params.model_copy(update={"page": None, "cursor_start_time": start, "cursor_request_id": request_id}), - ) + cursor: Final = descend(document.root, ("next_session_cursor",)) + assert isinstance(cursor, str) and cursor, "Missing session cursor" + return replace(case, params=params.model_copy(update={"page": 2, "session_cursor": cursor}), contract=contract) _LOGS: Final = Contract(("data",), ("request_id", "startTime", "metadata")) -_REQUESTS: Final = Contract(("requests",), ("request_id", "start_time", "cache_read_tokens", "gateway_injected")) -_ROWS: Final = Contract(("logs",), ("id", "timestamp", "model")) def log_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: @@ -102,31 +73,27 @@ def log_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: page_size=25, sort_by="startTime", sort_order="desc", - group_by_session=True, + group_by_session="true", ) base: Final = UiQueryCase("logs_default_24h", "/spend/logs/ui", params, _LOGS) + filtered: Final = params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=30))}) filters: Final = ( ("api_key", api_key), ("team_id", "perf-team-0"), ("user_id", "perf-user-0"), - ("end_user", "perf-end-user-0"), ("model", "perf-chat"), ("model_id", "perf-model-0"), ("key_alias", "perf-key-0"), - ("session_id", "perf-ui-session"), ("status_filter", "failure"), ("cache_hit_filter", "hit"), ("error_code", "429"), ("error_message", "Synthetic"), ("span_type", "llm"), - ("search", "perf-ui-1"), - ("exclude_internal_health_checks", True), - ("used_client_oauth_token", False), + ("exclude_internal_health_checks", "true"), ) return ( base, replace(base, name="logs_cursor_page_2", params=params.model_copy(update={"page": 2}), cursor="session"), - replace(base, name="logs_offset_page_3", params=params.model_copy(update={"page": 3})), replace( base, name="logs_7d", @@ -138,49 +105,13 @@ def log_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: params=params.model_copy(update={"start_date": _timestamp(anchor - timedelta(days=30))}), ), *( - replace(base, name=f"logs_{name}", params=params.model_copy(update={name: value})) + replace(base, name=f"logs_{name}_30d", params=filtered.model_copy(update={name: value})) for name, value in filters ), *( replace(base, name=f"logs_sort_{name}", params=params.model_copy(update={"sort_by": name})) for name in ("spend", "total_tokens", "request_duration_ms", "model", "ttft_ms") ), - UiQueryCase("log_id_lookup", "/spend/logs/ui", Params(request_id="perf-ui-1", page_size=1), _LOGS), - UiQueryCase( - "log_payload", - "/spend/logs/ui/perf-ui-1", - Params(start_date=params.start_date), - Contract(object_fields=("messages", "response", "proxy_server_request")), - ), - UiQueryCase( - "session_logs", "/spend/logs/session/ui", Params(session_id="perf-ui-session", page=1, page_size=100), _LOGS - ), - ) - - -def facet_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: - params: Final = Params.model_validate( - {"lower": _timestamp(anchor - timedelta(days=1)), "upper": _timestamp(anchor), "page_size": 50, "page": 1} - ) - return tuple( - chain.from_iterable( - ( - UiQueryCase(f"facet_{entity}", f"/management/v1/spend_logs/{entity}", params, Contract(("data",))), - UiQueryCase( - f"facet_{entity}_page_2", - f"/management/v1/spend_logs/{entity}", - params.model_copy(update={"page": 2}), - Contract(("data",)), - ), - UiQueryCase( - f"facet_{entity}_search", - f"/management/v1/spend_logs/{entity}", - params.model_copy(update={"q": "perf-"}), - Contract(("data",)), - ), - ) - for entity in ("users", "end_users") - ) ) @@ -188,9 +119,6 @@ def analytics_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: dates: Final = Params( start_date=(anchor - timedelta(days=7)).date().isoformat(), end_date=anchor.date().isoformat() ) - prompt_dates: Final = Params( - start_date=(anchor - timedelta(days=30)).isoformat(), end_date=anchor.isoformat(), filter="all", page_size=10 - ) cache: Final = UiQueryCase( "cache_activity_7d", "/global/activity/cache_hits", dates, Contract(("groups",), ("call_type",)) ) @@ -213,179 +141,15 @@ def analytics_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: name="cache_activity_filtered", params=dates.model_copy(update={"key_aliases": ("perf-key-0", "perf-key-1"), "models": ("perf-chat",)}), ), - *( - UiQueryCase( - f"prompt_cache_{filter}", - "/cost_optimization/prompt_caching/requests", - prompt_dates.model_copy(update={"filter": filter}), - _REQUESTS, - ) - for filter in ("all", "injected", "hits") - ), - UiQueryCase( - "prompt_cache_page_2", - "/cost_optimization/prompt_caching/requests", - prompt_dates, - _REQUESTS, - cursor="prompt_cache", - ), - UiQueryCase( - "guardrail_logs", - "/guardrails/usage/logs", - dates.model_copy(update={"guardrail_id": "perf-ui-guardrail", "page": 1, "page_size": 50}), - _ROWS, - ), - UiQueryCase( - "tool_logs", - "/v1/tool/perf-ui-tool/logs", - Params( - start_date=_timestamp(anchor - timedelta(days=90)), end_date=_timestamp(anchor), page=1, page_size=50 - ), - _ROWS, - ), ) -def old_usage_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: - month_start: Final = anchor.replace(day=1) - next_month: Final = (month_start + timedelta(days=32)).replace(day=1) - month: Final = Params( - start_date=month_start.date().isoformat(), end_date=(next_month - timedelta(days=1)).date().isoformat() - ) - week: Final = Params( - start_date=(anchor - timedelta(days=7)).isoformat(), end_date=anchor.isoformat(), tags="all-tags" - ) - return ( - UiQueryCase( - "old_activity", - "/global/activity", - month, - Contract(("daily_data",), ("date",), ("api_requests",)), - "old_usage", - ), - UiQueryCase( - "old_activity_models", - "/global/activity/model", - month, - Contract(row_fields=("model", "daily_data"), positive=("sum_api_requests",)), - "old_usage", - ), - UiQueryCase( - "old_spend_provider", - "/global/spend/provider", - month, - Contract(row_fields=("provider", "spend")), - "old_usage", - ), - UiQueryCase( - "old_tags", - "/global/spend/tags", - week, - Contract(("spend_per_tag",), ("name", "spend"), ("log_count",)), - "old_usage", - ), - UiQueryCase( - "old_tags_filtered", - "/global/spend/tags", - week.model_copy(update={"tags": "project-0,production"}), - Contract(("spend_per_tag",), ("name", "spend"), ("log_count",)), - "old_usage", - ), - UiQueryCase("old_tag_names", "/global/spend/all_tag_names", Params(), Contract(("tag_names",)), "old_usage"), - UiQueryCase( - "old_spend_logs", "/global/spend/logs", Params(), Contract(row_fields=("date", "spend")), "old_usage" - ), - UiQueryCase( - "old_keys", - "/global/spend/keys", - Params(limit=5), - Contract(row_fields=("api_key", "key_alias"), positive=("total_spend",)), - "old_usage", - ), - UiQueryCase("old_teams", "/global/spend/teams", Params(), Contract(("daily_spend",)), "old_usage"), - UiQueryCase( - "old_models", - "/global/spend/models", - Params(limit=5), - Contract(row_fields=("model",), positive=("total_spend",)), - "old_usage", - ), - UiQueryCase( - "old_end_users", - "/global/spend/end_users", - Params(), - Contract(row_fields=("end_user",), positive=("total_count",)), - "old_usage", - body=Query(), - ), - UiQueryCase( - "old_end_users_filtered", - "/global/spend/end_users", - Params(), - Contract(row_fields=("end_user",), positive=("total_count",)), - "old_usage", - body=EndUserBody(startTime=week.start_date, endTime=week.end_date, api_key=api_key), - ), - ) - - -def report_cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: +def _entity_activity_cases(anchor: datetime, entity: str) -> tuple[UiQueryCase, ...]: dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), end_date=anchor.date().isoformat() - ) - return ( - UiQueryCase( - "agent_global_report_teams", - "/global/spend/report", - dates.model_copy(update={"group_by": "team"}), - Contract(row_fields=("group_by_day", "teams")), - "liteadmin", - ), - UiQueryCase( - "agent_global_report_customers", - "/global/spend/report", - dates.model_copy(update={"group_by": "customer"}), - Contract(row_fields=("group_by_day", "customers")), - "liteadmin", - ), - UiQueryCase( - "agent_global_report_keys", - "/global/spend/report", - dates.model_copy(update={"group_by": "api_key"}), - Contract(row_fields=("api_key", "model_details"), positive=("total_cost",)), - "liteadmin", - ), - UiQueryCase( - "agent_key_report", - "/key/spend/report", - dates.model_copy(update={"api_key": api_key}), - Contract(row_fields=("api_key", "model_details"), positive=("total_cost",)), - "liteadmin", - ), - UiQueryCase( - "agent_team_report", - "/team/spend/report", - dates.model_copy(update={"team_id": "perf-team-0"}), - Contract(row_fields=("api_key", "model_details"), positive=("total_cost",)), - "liteadmin", - ), - ) - - -def cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: - return ( - *log_cases(anchor, api_key), - *facet_cases(anchor), - *analytics_cases(anchor), - *old_usage_cases(anchor, api_key), - *report_cases(anchor, api_key), - *recovery_cases(anchor), - ) - - -def _entity_recovery_cases(anchor: datetime, entity: str) -> tuple[UiQueryCase, ...]: - dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), end_date=anchor.date().isoformat(), timezone="0" + start_date=(anchor - timedelta(days=7)).date().isoformat(), + end_date=anchor.date().isoformat(), + timezone="0", + exclude_team_ids="litellm-dashboard" if entity == "team" else None, ) routes: Final = ( ("aggregated", dates, Contract(("results",), ("date", "breakdown"))), @@ -394,37 +158,49 @@ def _entity_recovery_cases(anchor: datetime, entity: str) -> tuple[UiQueryCase, dates.model_copy(update={"offset": 0, "limit": 50}), Contract(("api_keys",), ("api_key", "metrics", "metadata")), ), - ("aggregated/search", dates, Contract(("api_keys",), ("api_key", "metrics", "metadata"))), + ( + "aggregated/search", + dates.model_copy(update={"search": "perf-key"}), + Contract(("api_keys",), ("api_key", "metrics", "metadata")), + ), ( "aggregated/model_top_keys", dates.model_copy(update={"model_group": "perf-chat", "by_model_group": "true"}), Contract(("api_keys",), ("api_key", "metrics", "metadata")), ), ) - return tuple(_recovery_case(entity, route, params, contract) for route, params, contract in routes) - - -def _recovery_case(entity: str, route: str, params: Params, contract: Contract) -> UiQueryCase: - name: Final = f"recovery_{entity}_{route.replace('/', '_')}" - key: Final = orphan_key(name) - return UiQueryCase( - name, - f"/{entity}/daily/activity/{route}", - params.model_copy(update={"api_key": key, **({"search": key} if route.endswith("search") else {})}), - replace(contract, recovered_alias="perf-ui-recovered"), - "key_recovery", + return tuple( + UiQueryCase( + f"usage_{entity}_{route.replace('/', '_')}", + f"/{entity}/daily/activity/{route}", + params, + contract, + ) + for route, params, contract in routes ) -def recovery_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: - entities: Final = ("user", "team", "tag", "organization", "customer", "agent") - groups: Final = tuple(_entity_recovery_cases(anchor, entity) for entity in entities) - dates: Final = Params( - start_date=(anchor - timedelta(days=7)).date().isoformat(), end_date=anchor.date().isoformat(), timezone="0" - ) +def activity_cases(anchor: datetime) -> tuple[UiQueryCase, ...]: + groups: Final = tuple(_entity_activity_cases(anchor, entity) for entity in ("user", "team", "tag", "customer")) return ( - *sum(groups, ()), - _recovery_case( - "user", "aggregated/cache_leakage_keys", dates, Contract(("api_keys",), ("api_key", "metrics", "metadata")) + *chain.from_iterable(groups), + UiQueryCase( + "cache_leakage_keys", + "/user/daily/activity/aggregated/cache_leakage_keys", + Params( + start_date=(anchor - timedelta(days=30)).date().isoformat(), + end_date=anchor.date().isoformat(), + timezone="0", + include_current_utc_day="true", + ), + Contract(("api_keys",), ("api_key", "metrics", "metadata")), ), ) + + +def cases(anchor: datetime, api_key: str) -> tuple[UiQueryCase, ...]: + return ( + *log_cases(anchor, api_key), + *analytics_cases(anchor), + *activity_cases(anchor), + ) diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py index fe98665e12b..e3f5236ba47 100644 --- a/tests/e2e/perf/setup/ui_query_fixture.py +++ b/tests/e2e/perf/setup/ui_query_fixture.py @@ -1,21 +1,14 @@ from __future__ import annotations -import asyncio import hashlib import os import time -from collections.abc import Mapping, Sequence -from dataclasses import dataclass from datetime import datetime -from pathlib import Path from typing import Final import psycopg -from psycopg import sql from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter -from setup.ui_queries import orphan_key -from setup.ui_query_cases import recovery_cases from setup.ui_query_isolation import validate_target from setup.upgrade_worker import read_state, release_indexes @@ -32,7 +25,6 @@ def prepare_fixture() -> UiFixture: _finish_upgrade() with psycopg.connect(os.environ["DATABASE_URL"], connect_timeout=10) as connection: _ = connection.execute("SET statement_timeout='30s'") - asyncio.run(_create_views(connection)) row: Final = connection.execute("SELECT spec->>'anchor', rows FROM litellm_perf_seed.checkpoint").fetchone() assert row is not None and TypeAdapter(int).validate_python(row[1]) > 0, "Missing completed seed rows" fixture: Final = UiFixture( @@ -40,23 +32,6 @@ def prepare_fixture() -> UiFixture: api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest() + hashlib.md5(b"perf-key-tail:0", usedforsecurity=False).hexdigest(), ) - for statement in Path(__file__).with_suffix(".sql").read_text().split(";"): - if statement.strip(): - _ = connection.execute( - statement.encode(), - { - "anchor": fixture.anchor, - "api_key": fixture.api_key, - "gateway_marker": "litellm_gateway_injected_cache", - "request_ids": [f"perf-ui-{i}" for i in range(1, 321)], - }, - ) - count: Final = connection.execute( - 'SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s)', - ([f"perf-ui-{i}" for i in range(1, 321)],), - ).fetchone() - assert count is not None and count[0] == 320, "Incomplete UI fixture" - _recovery_rows(connection, fixture) return fixture @@ -76,54 +51,3 @@ def _finish_upgrade() -> None: pass time.sleep(5) raise AssertionError("Snapshot indexes did not finish before UI measurements") - - -@dataclass(frozen=True, slots=True) -class _Views: - connection: psycopg.Connection[tuple[object, ...]] - - async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: - assert not args and "LIMIT 1" in query, "Expected a bounded view existence probe" - with self.connection.transaction(): - _ = self.connection.execute(query.replace("LIMIT 1", "LIMIT 0").encode()) - return () - - async def execute_raw(self, query: str, *args: object) -> int: - assert not args - _ = self.connection.execute(query.encode()) - return 0 - - -async def _create_views(connection: psycopg.Connection[tuple[object, ...]]) -> None: - from litellm.proxy.db.create_views import create_missing_views - - await create_missing_views(_Views(connection)) - - -def _recovery_rows(connection: psycopg.Connection[tuple[object, ...]], fixture: UiFixture) -> None: - tables: Final = ( - ("LiteLLM_DailyUserSpend", "user_id", "perf-user-0"), - ("LiteLLM_DailyTeamSpend", "team_id", "perf-team-0"), - ("LiteLLM_DailyTagSpend", "tag", "production"), - ("LiteLLM_DailyOrganizationSpend", "organization_id", "perf-ui-organization"), - ("LiteLLM_DailyEndUserSpend", "end_user_id", "perf-end-user-0"), - ("LiteLLM_DailyAgentSpend", "agent_id", "perf-ui-agent"), - ) - for case, key in ((case, orphan_key(case.name)) for case in recovery_cases(fixture.anchor)): - _ = connection.execute( - """INSERT INTO "LiteLLM_SpendLogs" - SELECT (jsonb_populate_record(NULL::"LiteLLM_SpendLogs", to_jsonb(s) || jsonb_build_object( - 'request_id', %s::text, 'api_key', %s::text, - 'metadata', jsonb_build_object('user_api_key_alias', 'perf-ui-recovered') - ))).* FROM "LiteLLM_SpendLogs" s WHERE request_id = 'perf-ui-1' - ON CONFLICT DO NOTHING""", - (f"perf-ui-{case.name}", key), - ) - for table, field, entity in tables: - _ = connection.execute( - sql.SQL("""INSERT INTO {} (id, {}, date, api_key, model, model_group, custom_llm_provider, - spend, prompt_tokens, completion_tokens, api_requests, successful_requests, updated_at) - VALUES (%s, %s, %s, %s, 'perf-chat', 'perf-chat', 'openai', 1, 1000, 100, 1, 1, now()) - ON CONFLICT DO NOTHING""").format(sql.Identifier(table), sql.Identifier(field)), - (f"perf-ui-{table}-{case.name}", entity, fixture.anchor.date().isoformat(), key), - ) diff --git a/tests/e2e/perf/setup/ui_query_fixture.sql b/tests/e2e/perf/setup/ui_query_fixture.sql deleted file mode 100644 index 863c8aa42b1..00000000000 --- a/tests/e2e/perf/setup/ui_query_fixture.sql +++ /dev/null @@ -1,48 +0,0 @@ -INSERT INTO "LiteLLM_SpendLogs" -SELECT (jsonb_populate_record(NULL::"LiteLLM_SpendLogs", to_jsonb(template) || jsonb_build_object( - 'request_id', 'perf-ui-' || g, - 'litellm_call_id', md5('perf-ui-call-' || g), - 'api_key', %(api_key)s::text, - 'team_id', 'perf-team-0', - 'user', 'perf-user-' || g %% 200, - 'end_user', 'perf-end-user-' || g %% 200, - 'model', 'perf-chat', 'model_id', 'perf-model-0', 'model_group', 'perf-chat', - 'custom_llm_provider', 'openai', - 'startTime', %(anchor)s::timestamp - g * interval '10 seconds', - 'endTime', %(anchor)s::timestamp - g * interval '10 seconds' + interval '1 second', - 'completionStartTime', %(anchor)s::timestamp - g * interval '10 seconds' + interval '100 milliseconds', - 'session_id', CASE WHEN g %% 2 = 0 THEN 'perf-ui-session' ELSE 'perf-ui-session-' || g END, - 'status', CASE WHEN g %% 10 = 0 THEN 'failure' ELSE 'success' END, - 'cache_hit', CASE WHEN g %% 10 = 1 THEN 'True' ELSE 'False' END, - 'spend', 0.01, 'prompt_tokens', 1000, 'completion_tokens', 100, 'total_tokens', 1100, - 'messages', '{}', 'response', '{}', 'proxy_server_request', '{}', - 'metadata', jsonb_build_object( - 'used_client_oauth_token', false, - 'user_api_key_alias', 'perf-key-0', 'user_api_key_team_alias', 'perf-team-0', - 'usage_object', jsonb_build_object('prompt_tokens', 1000, 'completion_tokens', 100, - 'cache_read_input_tokens', 500, 'cache_creation_input_tokens', 100), - %(gateway_marker)s::text, 'perf-model-0', - 'error_information', CASE WHEN g %% 10 = 0 THEN - jsonb_build_object('error_code', '429', 'error_message', 'Synthetic upstream rate limit') ELSE '{}'::jsonb END, - 'guardrail_information', jsonb_build_array(jsonb_build_object( - 'guardrail_name', 'perf-ui-guardrail', 'guardrail_status', 'success', 'duration', 0.01)) - ) -))).* -FROM "LiteLLM_SpendLogs" template CROSS JOIN generate_series(1, 320) g -WHERE template.request_id = 'perf-seed-1' -ON CONFLICT (request_id) DO NOTHING; - -INSERT INTO "LiteLLM_GuardrailsTable" (guardrail_id, guardrail_name, litellm_params, updated_at) -VALUES ('perf-ui-guardrail', 'perf-ui-guardrail', - '{"guardrail":"generic_guardrail_api","mode":"pre_call","api_base":"http://127.0.0.1:1"}', now()) -ON CONFLICT (guardrail_id) DO NOTHING; - -INSERT INTO "LiteLLM_SpendLogGuardrailIndex" (request_id, guardrail_id, start_time) -SELECT request_id, 'perf-ui-guardrail', "startTime" FROM "LiteLLM_SpendLogs" -WHERE request_id = ANY(%(request_ids)s) -ON CONFLICT DO NOTHING; - -INSERT INTO "LiteLLM_SpendLogToolIndex" (request_id, tool_name, start_time) -SELECT request_id, 'perf-ui-tool', "startTime" FROM "LiteLLM_SpendLogs" -WHERE request_id = ANY(%(request_ids)s) -ON CONFLICT DO NOTHING; diff --git a/tests/e2e/perf/test_ui_queries.py b/tests/e2e/perf/test_ui_queries.py index 33a80e3a542..75ac4d3055f 100644 --- a/tests/e2e/perf/test_ui_queries.py +++ b/tests/e2e/perf/test_ui_queries.py @@ -66,9 +66,7 @@ class Prepared: def _page_two(transport: HttpTransport, case: UiQueryCase, timeout: float) -> Prepared: if case.cursor == "none": return Prepared(case) - params: Final = Params.model_validate(case.params.model_dump()).model_copy( - update={"page": 1 if case.cursor == "session" else None} - ) + params: Final = Params.model_validate(case.params.model_dump()).model_copy(update={"page": 1}) first: Final = replace(case, params=params) sample, document = measure(lambda: issue_query(transport, first, timeout=timeout), case.contract) if sample.error or document is None: @@ -93,7 +91,7 @@ def test_ui_query_performance(ui_case_name: str, ui_fixture: UiFixture, request: case: Final = next(case for case in cases(ui_fixture.anchor, ui_fixture.api_key) if case.name == ui_case_name) transport: Final = HttpTransport(base_url=CONTROL_PLANE_BASE_URL, master_key=MASTER_KEY) timeout: Final = float(os.environ.get("E2E_UI_QUERY_TIMEOUT_SECONDS", "120")) - repeats: Final = int(os.environ.get("E2E_UI_QUERY_REPEATS", "5")) + repeats: Final = int(os.environ.get("E2E_UI_QUERY_REPEATS", "9")) assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings" target: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) assert target.is_dir(), "E2E_ARTIFACT_DIR must exist" diff --git a/tests/e2e_harness/perf/setup/test_ui_queries.py b/tests/e2e_harness/perf/setup/test_ui_queries.py index 341a3573b13..f24d03c6bba 100644 --- a/tests/e2e_harness/perf/setup/test_ui_queries.py +++ b/tests/e2e_harness/perf/setup/test_ui_queries.py @@ -2,20 +2,79 @@ from __future__ import annotations from datetime import UTC, datetime from typing import Final +from urllib.parse import parse_qs, urlsplit import pytest from e2e_http import NetworkError, Result, Success, UnknownApiError -from setup.ui_queries import Contract, Document, Sample, database_failures, measure, summarize, ui_junit_properties +from setup.ui_queries import ( + Contract, + Document, + Sample, + database_failures, + measure, + query_path, + summarize, + ui_junit_properties, +) from setup.ui_query_cases import Params, cases, next_page -def test_facet_dates_use_the_ui_bracketed_filter_names() -> None: - case: Final = next(case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.name == "facet_users") - assert case.params.model_dump(by_alias=True, exclude_none=True) == { - "filter[startTime][gte]": "2025-12-31 00:00:00", - "filter[startTime][lte]": "2026-01-01 00:00:00", - "page_size": 50, - "page": 1, +def test_query_string_preserves_ui_arrays_and_boolean_strings() -> None: + requests: Final = cases(datetime(2026, 1, 1, tzinfo=UTC), "key") + cache: Final = next(case for case in requests if case.name == "cache_activity_filtered") + assert parse_qs(urlsplit(query_path(cache)).query) == { + "start_date": ["2025-12-25"], + "end_date": ["2026-01-01"], + "key_aliases": ["perf-key-0", "perf-key-1"], + "models": ["perf-chat"], + } + health: Final = next(case for case in requests if case.name == "logs_exclude_internal_health_checks_30d") + query: Final = parse_qs(urlsplit(query_path(health)).query) + assert query["exclude_internal_health_checks"] == ["true"] + assert query["group_by_session"] == ["true"] + + +@pytest.mark.parametrize("entity", ("user", "team", "tag", "customer")) +@pytest.mark.parametrize( + ("route", "extra"), + ( + ("aggregated", {}), + ("aggregated/keys", {"offset": 0, "limit": 50}), + ("aggregated/search", {"search": "perf-key"}), + ("aggregated/model_top_keys", {"model_group": "perf-chat", "by_model_group": "true"}), + ), +) +def test_usage_requests_list_keys_without_a_single_key_filter( + entity: str, route: str, extra: dict[str, str | int] +) -> None: + case: Final = next( + case + for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") + if case.path == f"/{entity}/daily/activity/{route}" + ) + assert case.params.model_dump(exclude_none=True) == { + "start_date": "2025-12-25", + "end_date": "2026-01-01", + "timezone": "0", + **({"exclude_team_ids": "litellm-dashboard"} if entity == "team" else {}), + **extra, + } + + +@pytest.mark.parametrize(("name", "value"), (("api_key", "key"), ("key_alias", "perf-key-0"))) +def test_key_filters_request_30_days_of_logs(name: str, value: str) -> None: + case: Final = next( + case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.name == f"logs_{name}_30d" + ) + assert parse_qs(urlsplit(query_path(case)).query) == { + "start_date": ["2025-12-02 00:00:00"], + "end_date": ["2026-01-01 00:00:00"], + "page_size": ["25"], + "page": ["1"], + "sort_by": ["startTime"], + "sort_order": ["desc"], + "group_by_session": ["true"], + name: [value], } @@ -35,19 +94,6 @@ def test_session_cursor_is_reused_and_repeated_rows_rejected() -> None: _ = second.contract.count(first) -def test_prompt_cache_cursor_preserves_both_fields() -> None: - case: Final = next(case for case in cases(datetime(2026, 1, 1, tzinfo=UTC), "key") if case.cursor == "prompt_cache") - first: Final = Document.model_validate_json( - '{"requests":[{"request_id":"one"}],"next_cursor":{"start_time":"2026-01-01T00:00:00Z","request_id":"one"}}' - ) - params: Final = Params.model_validate(next_page(case, first).params) - assert params.cursor_start_time == "2026-01-01T00:00:00Z" - assert params.cursor_request_id == "one" - assert params.page is None - with pytest.raises(AssertionError, match="Missing prompt-cache cursor"): - _ = next_page(case, Document.model_validate_json('{"requests":[{"request_id":"one"}],"next_cursor":null}')) - - def test_first_query_is_separate_from_repeated_latency() -> None: samples: Final = (Sample(seconds=80, status=200), *(Sample(seconds=i, status=200) for i in range(1, 101))) result: Final = summarize(samples) @@ -82,6 +128,7 @@ def test_failed_first_request_emits_error_code_without_latency() -> None: assert dict(ui_junit_properties((sample,))) == { "perf_ui_requests": 1, "perf_ui_failures": 1, + "perf_ui_samples": 0, "perf_ui_repeated_samples": 0, "perf_ui_error_http_500": 1, } @@ -98,6 +145,10 @@ def test_junit_counts_failures_but_only_measures_successful_latency() -> None: assert dict(ui_junit_properties(samples, Sample(seconds=4, status=200))) == { "perf_ui_requests": 6, "perf_ui_failures": 3, + "perf_ui_samples": 2, + "perf_ui_seconds_p50": 2.5, + "perf_ui_seconds_p90": 3, + "perf_ui_seconds_max": 3, "perf_ui_repeated_samples": 1, "perf_ui_error_timeout": 1, "perf_ui_error_http_503": 1, @@ -113,6 +164,17 @@ def test_failed_cursor_prerequisite_is_counted_once() -> None: assert dict(ui_junit_properties((sample,), sample)) == dict(ui_junit_properties((sample,))) +def test_all_request_percentiles_include_first_and_use_nearest_rank_p90() -> None: + samples: Final = tuple(Sample(seconds=seconds, status=200) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5)) + metrics: Final = dict(ui_junit_properties(samples)) + assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == { + "perf_ui_seconds_p50": 5.5, + "perf_ui_seconds_p90": 9, + "perf_ui_seconds_max": 10, + } + assert metrics["perf_ui_samples"] == 10 + + @pytest.mark.parametrize("body", ("{}", '{"data": []}', '{"data": [{"spend": 1}]}', '{"data": [null]}')) def test_empty_or_malformed_rows_fail_even_with_http_200(body: str) -> None: ticks: Final = iter((10.0, 12.5)) @@ -145,13 +207,6 @@ def test_zero_spend_rows_are_allowed_alongside_nonzero_fixture_data() -> None: _ = contract.count(Document.model_validate_json('[{"spend":0}]')) -def test_daily_aggregate_must_return_recovered_spendlog_alias() -> None: - contract: Final = Contract(row_fields=("metadata",), recovered_alias="perf-ui-recovered") - assert contract.count(Document.model_validate_json('[{"metadata":{"key_alias":"perf-ui-recovered"}}]')) == 1 - with pytest.raises(AssertionError, match="not recovered"): - _ = contract.count(Document.model_validate_json('[{"metadata":{"key_alias":null}}]')) - - @pytest.mark.parametrize( "failure,error,status", (