diff --git a/tests/e2e/perf/cases/mixed_size_requests_2rps.py b/tests/e2e/perf/cases/mixed_size_requests_2rps.py index f95fb0845c4..89263977fb0 100644 --- a/tests/e2e/perf/cases/mixed_size_requests_2rps.py +++ b/tests/e2e/perf/cases/mixed_size_requests_2rps.py @@ -70,6 +70,8 @@ CASE: Final = PerformanceTestCase( Limit("cpu_seconds_per_request", max=1.26), Limit("postgres_calls_per_request", max=3.5), Limit("postgres_max_statement_per_request", max=0.5), + Limit("postgres_max_background_statement_per_request", max=0.9), Limit("redis_calls_per_request", max=21.0), ), + background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}), ) diff --git a/tests/e2e/perf/cases/small_bodies_5rps.py b/tests/e2e/perf/cases/small_bodies_5rps.py index 9d3484de05d..73772465492 100644 --- a/tests/e2e/perf/cases/small_bodies_5rps.py +++ b/tests/e2e/perf/cases/small_bodies_5rps.py @@ -69,6 +69,7 @@ CASE: Final = PerformanceTestCase( Limit("memory_growth_megabytes", max=100), Limit("postgres_calls_per_request", max=1.8), Limit("postgres_max_statement_per_request", max=0.5), + Limit("postgres_max_background_statement_per_request", max=0.9), Limit("redis_calls_per_request", max=18), Limit("cpu_seconds_per_request", max=0.29), ), diff --git a/tests/e2e/perf/setup/datastore.py b/tests/e2e/perf/setup/datastore.py index b4ad10b5a5a..54c6943a054 100644 --- a/tests/e2e/perf/setup/datastore.py +++ b/tests/e2e/perf/setup/datastore.py @@ -76,6 +76,11 @@ class StatementCalls: return "plumbing" return "awaited" if verb == "SELECT" and tables[0] in _AWAITED_TABLES else "background" + @property + def table(self) -> str: + tables: Final = _TABLE.findall(self.query) + return tables[0] if tables else "" + @property def summary(self) -> str: return re.sub(r"\s+", " ", self.query)[:110] diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.py index be64679c9da..f634df7de84 100644 --- a/tests/e2e/perf/setup/observe.py +++ b/tests/e2e/perf/setup/observe.py @@ -135,6 +135,14 @@ class Observation: awaited: Final = tuple(item for item in self.statements or () if item.bucket == "awaited") return max(awaited, key=_item_count, default=None) + def busiest_background_statement(self, exempt_tables: frozenset[str]) -> StatementCalls | None: + """The background statement with the most calls, skipping the case's per-flush exemptions: a rollup + written once per request reads 1.0 here while batched writers stay below it.""" + background: Final = tuple( + item for item in self.statements or () if item.bucket == "background" and item.table not in exempt_tables + ) + return max(background, key=_item_count, default=None) + @property def retried_markers(self) -> frozenset[str]: counts: Final = Counter(seen.marker for seen in self.mock_requests if seen.marker is not None) @@ -202,7 +210,9 @@ def _text_sha1(text: str, *, intervened: bool) -> str: return hashlib.sha1(original.encode()).hexdigest() -def observe(metric: Metric, percentile_rank: int | None, observation: Observation) -> float | None: +def observe( + metric: Metric, percentile_rank: int | None, observation: Observation, exempt_tables: frozenset[str] = frozenset() +) -> float | None: """The observed value of one metric, at a percentile when the metric is per request.""" per_request: dict[str, list[float]] = { "gateway_added_seconds": observation.gateway_added_seconds(), @@ -221,6 +231,11 @@ def observe(metric: Metric, percentile_rank: int | None, observation: Observatio return None busiest: Final = observation.busiest_awaited_statement return observation.count_per_request(0 if busiest is None else busiest.calls) + if metric == "postgres_max_background_statement_per_request": + if observation.statements is None: + return None + busiest_background: Final = observation.busiest_background_statement(exempt_tables) + return observation.count_per_request(0 if busiest_background is None else busiest_background.calls) if metric == "redis_calls_per_request": return observation.count_per_request(calls_in("awaited", observation.redis_commands)) if metric == "redis_background_calls_per_request": @@ -266,6 +281,7 @@ REPORTED: Final[tuple[tuple[Metric, int | None], ...]] = ( ("postgres_calls_per_request", None), ("postgres_background_calls_per_request", None), ("postgres_max_statement_per_request", None), + ("postgres_max_background_statement_per_request", None), ("redis_calls_per_request", None), ("redis_background_calls_per_request", None), ("cpu_seconds_per_request", None), @@ -287,9 +303,9 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[ Row( metric=metric, percentile=rank, - observed=observe(metric, rank, observation), + observed=observe(metric, rank, observation, case.background_cap_exempt_tables), limit=limit.max if limit else None, - detail=_detail(metric, observation), + detail=_detail(metric, observation, case.background_cap_exempt_tables), ) ) exceeded: Final = [ @@ -302,11 +318,14 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[ return rows, exceeded -def _detail(metric: Metric, observation: Observation) -> str: - if metric != "postgres_max_statement_per_request": - return "" - busiest: Final = observation.busiest_awaited_statement - return "" if busiest is None else busiest.summary +def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[str]) -> str: + if metric == "postgres_max_statement_per_request": + busiest: Final = observation.busiest_awaited_statement + return "" if busiest is None else busiest.summary + if metric == "postgres_max_background_statement_per_request": + busiest_background: Final = observation.busiest_background_statement(exempt_tables) + return "" if busiest_background is None else busiest_background.summary + return "" def _row_properties(row: Row) -> tuple[tuple[str, float], ...]: diff --git a/tests/e2e/perf/setup/performance_test_case.py b/tests/e2e/perf/setup/performance_test_case.py index 6a734e2923f..37ad640db62 100644 --- a/tests/e2e/perf/setup/performance_test_case.py +++ b/tests/e2e/perf/setup/performance_test_case.py @@ -24,6 +24,7 @@ Metric = Literal[ "postgres_calls_per_request", "postgres_background_calls_per_request", "postgres_max_statement_per_request", + "postgres_max_background_statement_per_request", "cpu_seconds_per_request", "redis_calls_per_request", "redis_background_calls_per_request", @@ -149,6 +150,9 @@ class PerformanceTestCase: traffic: Traffic limits: tuple[Limit, ...] mock_latency: MockLatency = field(default_factory=lambda: MockLatency()) + background_cap_exempt_tables: frozenset[str] = frozenset() + """Tables whose background write is one upsert per writer flush and, at this case's request rate, lands + above the `postgres_max_background_statement_per_request` cap without being per request.""" def team(self, name: str) -> Team: return next(team for team in self.teams if team.name == name) diff --git a/tests/e2e_harness/perf/setup/test_observe.py b/tests/e2e_harness/perf/setup/test_observe.py index 4b7e2d64a14..ad54c61aed0 100644 --- a/tests/e2e_harness/perf/setup/test_observe.py +++ b/tests/e2e_harness/perf/setup/test_observe.py @@ -70,6 +70,7 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un "perf_postgres_calls_per_request": 10.0, "perf_postgres_background_calls_per_request": 0.0, "perf_postgres_max_statement_per_request": 10.0, + "perf_postgres_max_background_statement_per_request": 0.0, "perf_redis_calls_per_request_limit": 35.0, } @@ -93,6 +94,13 @@ def _observation_with(statements: tuple[StatementCalls, ...]) -> Observation: MAX_STATEMENT_CASE: Final = replace(CASE, limits=(Limit("postgres_max_statement_per_request", max=0.5),)) +BACKGROUND_CAP_CASE: Final = replace( + CASE, + limits=(Limit("postgres_max_background_statement_per_request", max=0.9),), + background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}), +) +GUARDRAIL_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyGuardrailMetrics" ("guardrail_id","date") VALUES ($1,$2)' +MODEL_USAGE_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyModelUsage" ("date","model") VALUES ($1,$2)' TEAM_REFRESH: Final = 'SELECT * FROM "LiteLLM_TeamMembership" WHERE user_id = $1' USER_LOOKUP: Final = 'SELECT * FROM "LiteLLM_UserTable" WHERE user_id = $1' @@ -125,3 +133,37 @@ def test_per_statement_limit_without_db_counters_reports_nothing_observed() -> N _, exceeded = evaluate(MAX_STATEMENT_CASE, observation) assert exceeded == ["postgres_max_statement_per_request: limit set but nothing observed"] + + +def test_per_flush_rollups_pass_the_background_cap_and_the_exempt_table_is_skipped() -> None: + statements: Final = ( + _statement(1, GUARDRAIL_UPSERT, calls=1150), + _statement(2, MODEL_USAGE_UPSERT, calls=810), + _statement(3, USER_LOOKUP, calls=1000), + ) + + rows, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements)) + + assert exceeded == [] + assert dict(junit_properties(rows))["perf_postgres_max_background_statement_per_request"] == 0.81 + + +def test_a_rollup_written_on_every_request_fails_the_background_cap_and_names_the_statement() -> None: + statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1050), _statement(2, MODEL_USAGE_UPSERT, calls=1000)) + + _, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements)) + + assert exceeded == [ + f"postgres_max_background_statement_per_request: observed 1.0000 > limit 0.9000 ({MODEL_USAGE_UPSERT})" + ], exceeded + + +def test_background_cap_without_exemptions_counts_the_guardrail_upsert() -> None: + case: Final = replace(BACKGROUND_CAP_CASE, background_cap_exempt_tables=frozenset()) + statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1150), _statement(2, MODEL_USAGE_UPSERT, calls=810)) + + _, exceeded = evaluate(case, _observation_with(statements)) + + assert exceeded == [ + f"postgres_max_background_statement_per_request: observed 1.1500 > limit 0.9000 ({GUARDRAIL_UPSERT})" + ], exceeded