test(perf): cap every background Postgres statement at 0.9 calls per request (#46957)

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/e2e/perf/cases/mixed_size_requests_2rps.py

@@ -70,6 +70,8 @@ CASE: Final = PerformanceTestCase(
         Limit("cpu_seconds_per_request", max=1.26),
         Limit("postgres_calls_per_request", max=3.5),
         Limit("postgres_max_statement_per_request", max=0.5),
+        Limit("postgres_max_background_statement_per_request", max=0.9),
         Limit("redis_calls_per_request", max=21.0),
     ),
+    background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
 )

tests/e2e/perf/cases/small_bodies_5rps.py

@@ -69,6 +69,7 @@ CASE: Final = PerformanceTestCase(
         Limit("memory_growth_megabytes", max=100),
         Limit("postgres_calls_per_request", max=1.8),
         Limit("postgres_max_statement_per_request", max=0.5),
+        Limit("postgres_max_background_statement_per_request", max=0.9),
         Limit("redis_calls_per_request", max=18),
         Limit("cpu_seconds_per_request", max=0.29),
     ),

tests/e2e/perf/setup/datastore.py

@@ -76,6 +76,11 @@ class StatementCalls:
             return "plumbing"
         return "awaited" if verb == "SELECT" and tables[0] in _AWAITED_TABLES else "background"
 
+    @property
+    def table(self) -> str:
+        tables: Final = _TABLE.findall(self.query)
+        return tables[0] if tables else ""
+
     @property
     def summary(self) -> str:
         return re.sub(r"\s+", " ", self.query)[:110]

tests/e2e/perf/setup/observe.py

@@ -135,6 +135,14 @@ class Observation:
         awaited: Final = tuple(item for item in self.statements or () if item.bucket == "awaited")
         return max(awaited, key=_item_count, default=None)
 
+    def busiest_background_statement(self, exempt_tables: frozenset[str]) -> StatementCalls | None:
+        """The background statement with the most calls, skipping the case's per-flush exemptions: a rollup
+        written once per request reads 1.0 here while batched writers stay below it."""
+        background: Final = tuple(
+            item for item in self.statements or () if item.bucket == "background" and item.table not in exempt_tables
+        )
+        return max(background, key=_item_count, default=None)
+
     @property
     def retried_markers(self) -> frozenset[str]:
         counts: Final = Counter(seen.marker for seen in self.mock_requests if seen.marker is not None)
@@ -202,7 +210,9 @@ def _text_sha1(text: str, *, intervened: bool) -> str:
     return hashlib.sha1(original.encode()).hexdigest()
 
 
-def observe(metric: Metric, percentile_rank: int | None, observation: Observation) -> float | None:
+def observe(
+    metric: Metric, percentile_rank: int | None, observation: Observation, exempt_tables: frozenset[str] = frozenset()
+) -> float | None:
     """The observed value of one metric, at a percentile when the metric is per request."""
     per_request: dict[str, list[float]] = {
         "gateway_added_seconds": observation.gateway_added_seconds(),
@@ -221,6 +231,11 @@ def observe(metric: Metric, percentile_rank: int | None, observation: Observatio
             return None
         busiest: Final = observation.busiest_awaited_statement
         return observation.count_per_request(0 if busiest is None else busiest.calls)
+    if metric == "postgres_max_background_statement_per_request":
+        if observation.statements is None:
+            return None
+        busiest_background: Final = observation.busiest_background_statement(exempt_tables)
+        return observation.count_per_request(0 if busiest_background is None else busiest_background.calls)
     if metric == "redis_calls_per_request":
         return observation.count_per_request(calls_in("awaited", observation.redis_commands))
     if metric == "redis_background_calls_per_request":
@@ -266,6 +281,7 @@ REPORTED: Final[tuple[tuple[Metric, int | None], ...]] = (
     ("postgres_calls_per_request", None),
     ("postgres_background_calls_per_request", None),
     ("postgres_max_statement_per_request", None),
+    ("postgres_max_background_statement_per_request", None),
     ("redis_calls_per_request", None),
     ("redis_background_calls_per_request", None),
     ("cpu_seconds_per_request", None),
@@ -287,9 +303,9 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[
             Row(
                 metric=metric,
                 percentile=rank,
-                observed=observe(metric, rank, observation),
+                observed=observe(metric, rank, observation, case.background_cap_exempt_tables),
                 limit=limit.max if limit else None,
-                detail=_detail(metric, observation),
+                detail=_detail(metric, observation, case.background_cap_exempt_tables),
             )
         )
     exceeded: Final = [
@@ -302,11 +318,14 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[
     return rows, exceeded
 
 
-def _detail(metric: Metric, observation: Observation) -> str:
-    if metric != "postgres_max_statement_per_request":
-        return ""
-    busiest: Final = observation.busiest_awaited_statement
-    return "" if busiest is None else busiest.summary
+def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[str]) -> str:
+    if metric == "postgres_max_statement_per_request":
+        busiest: Final = observation.busiest_awaited_statement
+        return "" if busiest is None else busiest.summary
+    if metric == "postgres_max_background_statement_per_request":
+        busiest_background: Final = observation.busiest_background_statement(exempt_tables)
+        return "" if busiest_background is None else busiest_background.summary
+    return ""
 
 
 def _row_properties(row: Row) -> tuple[tuple[str, float], ...]:

tests/e2e/perf/setup/performance_test_case.py

@@ -24,6 +24,7 @@ Metric = Literal[
     "postgres_calls_per_request",
     "postgres_background_calls_per_request",
     "postgres_max_statement_per_request",
+    "postgres_max_background_statement_per_request",
     "cpu_seconds_per_request",
     "redis_calls_per_request",
     "redis_background_calls_per_request",
@@ -149,6 +150,9 @@ class PerformanceTestCase:
     traffic: Traffic
     limits: tuple[Limit, ...]
     mock_latency: MockLatency = field(default_factory=lambda: MockLatency())
+    background_cap_exempt_tables: frozenset[str] = frozenset()
+    """Tables whose background write is one upsert per writer flush and, at this case's request rate, lands
+    above the `postgres_max_background_statement_per_request` cap without being per request."""
 
     def team(self, name: str) -> Team:
         return next(team for team in self.teams if team.name == name)

tests/e2e_harness/perf/setup/test_observe.py

@@ -70,6 +70,7 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un
         "perf_postgres_calls_per_request": 10.0,
         "perf_postgres_background_calls_per_request": 0.0,
         "perf_postgres_max_statement_per_request": 10.0,
+        "perf_postgres_max_background_statement_per_request": 0.0,
         "perf_redis_calls_per_request_limit": 35.0,
     }
 
@@ -93,6 +94,13 @@ def _observation_with(statements: tuple[StatementCalls, ...]) -> Observation:
 
 
 MAX_STATEMENT_CASE: Final = replace(CASE, limits=(Limit("postgres_max_statement_per_request", max=0.5),))
+BACKGROUND_CAP_CASE: Final = replace(
+    CASE,
+    limits=(Limit("postgres_max_background_statement_per_request", max=0.9),),
+    background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
+)
+GUARDRAIL_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyGuardrailMetrics" ("guardrail_id","date") VALUES ($1,$2)'
+MODEL_USAGE_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyModelUsage" ("date","model") VALUES ($1,$2)'
 TEAM_REFRESH: Final = 'SELECT * FROM "LiteLLM_TeamMembership" WHERE user_id = $1'
 USER_LOOKUP: Final = 'SELECT * FROM "LiteLLM_UserTable" WHERE user_id = $1'
 
@@ -125,3 +133,37 @@ def test_per_statement_limit_without_db_counters_reports_nothing_observed() -> N
     _, exceeded = evaluate(MAX_STATEMENT_CASE, observation)
 
     assert exceeded == ["postgres_max_statement_per_request: limit set but nothing observed"]
+
+
+def test_per_flush_rollups_pass_the_background_cap_and_the_exempt_table_is_skipped() -> None:
+    statements: Final = (
+        _statement(1, GUARDRAIL_UPSERT, calls=1150),
+        _statement(2, MODEL_USAGE_UPSERT, calls=810),
+        _statement(3, USER_LOOKUP, calls=1000),
+    )
+
+    rows, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements))
+
+    assert exceeded == []
+    assert dict(junit_properties(rows))["perf_postgres_max_background_statement_per_request"] == 0.81
+
+
+def test_a_rollup_written_on_every_request_fails_the_background_cap_and_names_the_statement() -> None:
+    statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1050), _statement(2, MODEL_USAGE_UPSERT, calls=1000))
+
+    _, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements))
+
+    assert exceeded == [
+        f"postgres_max_background_statement_per_request: observed 1.0000 > limit 0.9000 ({MODEL_USAGE_UPSERT})"
+    ], exceeded
+
+
+def test_background_cap_without_exemptions_counts_the_guardrail_upsert() -> None:
+    case: Final = replace(BACKGROUND_CAP_CASE, background_cap_exempt_tables=frozenset())
+    statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1150), _statement(2, MODEL_USAGE_UPSERT, calls=810))
+
+    _, exceeded = evaluate(case, _observation_with(statements))
+
+    assert exceeded == [
+        f"postgres_max_background_statement_per_request: observed 1.1500 > limit 0.9000 ({GUARDRAIL_UPSERT})"
+    ], exceeded