test(e2e): mixed_size_requests_2rps perf case (coding-agent bodies 5k to 1m tokens with tools, JWT and key callers, two guardrails) (#46717)

* test(e2e): mixed_size_requests_2rps perf case (coding-agent bodies, JWT and key callers, two guardrails)

* test(e2e_harness): test_observe builds DbCounters with statements and expects the background Postgres line

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>

tests/e2e/perf/cases/mixed_size_requests_2rps.py

@@ -0,0 +1,74 @@
+"""Mixed-size requests at 2 rps on the small gateway: three real model groups over the mock providers,
+two teams plus users in no team, JWT and key callers, a pre and
+a post guardrail, half the requests streaming, short chats, long contexts and tool calls. Budgets
+are real money at cost-map prices (a 335k-token `claude-sonnet-5` request is about $0.70 and the
+run spends roughly $600), so they are sized so no caller runs dry mid-run."""
+
+from __future__ import annotations
+
+from typing import Final
+
+from setup.performance_test_case import (
+    LONG,
+    SHORT,
+    Guardrail,
+    Limit,
+    MockLatency,
+    PerformanceTestCase,
+    Team,
+    Traffic,
+    User,
+    VirtualKey,
+)
+
+CASE: Final = PerformanceTestCase(
+    name="mixed_size_requests_2rps",
+    teams=(
+        Team("team-a", models=("gpt-6-luna", "claude-sonnet-5"), max_budget=5000.0, rpm_limit=6000),
+        Team("team-b", models=("claude-sonnet-5-bedrock", "claude-sonnet-5"), max_budget=5000.0),
+    ),
+    users=(
+        User("alice", team="team-a"),
+        User("bob", team="team-a"),
+        User("carol", team="team-a"),
+        User("dave", team="team-b"),
+        User("erin", team="team-b"),
+        User("frank", team="team-b"),
+        User("grace"),
+        User("heidi"),
+        User("ivan"),
+        User("judy", max_budget=1000.0),
+    ),
+    virtual_keys=(
+        VirtualKey("key-a1", team="team-a", models=("gpt-6-luna",), rpm_limit=1200),
+        VirtualKey("key-a2", team="team-a", user="alice"),
+        VirtualKey("key-b1", team="team-b"),
+        VirtualKey("key-b2", team="team-b", user="dave"),
+        VirtualKey("key-plain"),
+    ),
+    model_groups=("gpt-6-luna", "claude-sonnet-5", "claude-sonnet-5-bedrock"),
+    mock_latency=MockLatency(seconds=1.0, jitter_seconds=0.25),
+    guardrails=(Guardrail("perf-pre", mode="pre_call"), Guardrail("perf-post", mode="post_call")),
+    traffic=Traffic(
+        seed=7,
+        rps=2.0,
+        total_requests=1200,
+        endpoints={"chat_completions": 0.5, "responses": 0.25, "messages": 0.25},
+        shapes={SHORT: 0.75, LONG: 0.25},
+        stream_share=0.5,
+        intervene_share=0.0,
+        block_share=0.0,
+        concurrency=100,
+    ),
+    limits=(
+        Limit("unexpected_status_count", max=3),
+        Limit("gateway_added_seconds", percentile=50, max=0.58),
+        Limit("gateway_added_seconds", percentile=99, max=6.1),
+        Limit("time_to_first_token_seconds", percentile=50, max=2.7),
+        Limit("time_to_first_token_seconds", percentile=99, max=8.8),
+        Limit("memory_growth_megabytes", max=600),
+        Limit("cpu_seconds_per_request", max=1.26),
+        Limit("postgres_calls_per_request", max=3.5),
+        Limit("redis_calls_per_request", max=21.0),
+    ),
+)

tests/e2e_harness/perf/setup/test_observe.py

@@ -5,7 +5,7 @@ from typing import Final
 
 from cases.small_bodies_5rps import CASE
 from memory_readings import RssCapture
-from setup.datastore import DbCounters
+from setup.datastore import DbCounters, StatementCalls
 from setup.observe import Observation, evaluate, junit_properties
 from setup.performance_test_case import Limit
 from setup.wire import WireRecord
@@ -35,6 +35,10 @@ def served(seq: int, total_s: float) -> WireRecord:
     )
 
 
+def _key_lookup(calls: int) -> StatementCalls:
+    return StatementCalls(queryid=1, query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1', calls=calls)
+
+
 def test_junit_properties_carry_each_observed_metric_and_each_limit_including_unobserved_ones() -> None:
     case: Final = replace(
         CASE,
@@ -50,8 +54,8 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un
         rss_before=NO_RSS,
         rss_after=NO_RSS,
         wall_seconds=3.0,
-        db_before=DbCounters(commits=100),
-        db_after=DbCounters(commits=130),
+        db_before=DbCounters(commits=100, statements=(_key_lookup(calls=100),)),
+        db_after=DbCounters(commits=130, statements=(_key_lookup(calls=130),)),
     )
     rows, _ = evaluate(case, observation)
 
@@ -64,5 +68,6 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un
         "perf_gateway_added_seconds_p50_limit": 0.625,
         "perf_gateway_added_seconds_p99": 0.75,
         "perf_postgres_calls_per_request": 10.0,
+        "perf_postgres_background_calls_per_request": 0.0,
         "perf_redis_calls_per_request_limit": 35.0,
     }