From 7e39e95bbb9a1d3d46da75be5a5ed423d9c2f9bb Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 14:01:54 +0000 Subject: [PATCH] test(perf): give the direct vs bridge cases their own ceilings at 1.2x the lane median --- tests/e2e/perf/cases/chat_completions_2rps.py | 23 +++++++++++++++++++ tests/e2e/perf/cases/direct_vs_bridge.py | 12 ++++++---- tests/e2e/perf/cases/messages_2rps.py | 23 +++++++++++++++++++ tests/e2e/perf/cases/responses_2rps.py | 23 +++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/tests/e2e/perf/cases/chat_completions_2rps.py b/tests/e2e/perf/cases/chat_completions_2rps.py index 9792d99c14f..4e48e584534 100644 --- a/tests/e2e/perf/cases/chat_completions_2rps.py +++ b/tests/e2e/perf/cases/chat_completions_2rps.py @@ -5,6 +5,8 @@ from __future__ import annotations from typing import Final +from setup.performance_test_case import Limit + from .direct_vs_bridge import direct_vs_bridge CASE: Final = direct_vs_bridge( @@ -12,4 +14,25 @@ CASE: Final = direct_vs_bridge( "chat_completions", direct="gpt-6-luna", bridge="gpt-6-luna-responses", + limits=( + Limit("unexpected_status_count", max=3), + Limit("gateway_added_seconds", percentile=50, max=0.326), + Limit("gateway_added_seconds", percentile=99, max=2.23), + Limit("litellm_overhead_seconds", percentile=50, max=0.0372), + Limit("litellm_overhead_seconds", percentile=99, max=1.53), + Limit("litellm_response_seconds", percentile=50, max=1.05), + Limit("litellm_response_seconds", percentile=99, max=3.06), + Limit("prometheus_redis_latency_mean_seconds", max=0.00372), + Limit("postgres_active_sessions", max=0.00636), + Limit("postgres_slowest_statement_seconds", max=0.042), + Limit("postgres_peak_connections", max=24), + Limit("time_to_first_token_seconds", percentile=50, max=1.62), + Limit("time_to_first_token_seconds", percentile=99, max=3.66), + Limit("memory_growth_megabytes", max=600), + Limit("cpu_seconds_per_request", max=0.577), + Limit("postgres_calls_per_request", max=2.6), + Limit("postgres_max_statement_per_request", max=0.427), + Limit("postgres_max_background_statement_per_request", max=1.08), + Limit("redis_calls_per_request", max=15.0), + ), ) diff --git a/tests/e2e/perf/cases/direct_vs_bridge.py b/tests/e2e/perf/cases/direct_vs_bridge.py index 13f609c8299..3482c6986a4 100644 --- a/tests/e2e/perf/cases/direct_vs_bridge.py +++ b/tests/e2e/perf/cases/direct_vs_bridge.py @@ -1,14 +1,15 @@ """The mixed case's traffic on one public API, half of it on a deployment the gateway serves on that provider's own API (direct) and half on a deployment it has to translate to another provider -API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as +API (bridge). Same 2 rps, request count, shapes, stream share, callers and guardrails as `mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the -translation's cost.""" +translation's cost. Each case carries its own limits, 1.2x the median of ten lane runs on every +stable tag from v1.103.0 to v1.106.0, the same rule as the other perf cases.""" from __future__ import annotations from dataclasses import replace -from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey +from setup.performance_test_case import Endpoint, Limit, PerformanceTestCase, VirtualKey from .mixed_size_requests_2rps import CASE as MIXED @@ -17,7 +18,9 @@ def _scoped(key: VirtualKey, direct: str) -> VirtualKey: return replace(key, models=(direct,)) if key.models else key -def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase: +def direct_vs_bridge( + name: str, endpoint: Endpoint, *, direct: str, bridge: str, limits: tuple[Limit, ...] +) -> PerformanceTestCase: return replace( MIXED, name=name, @@ -28,4 +31,5 @@ def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys), model_groups=(direct, bridge), traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}), + limits=limits, ) diff --git a/tests/e2e/perf/cases/messages_2rps.py b/tests/e2e/perf/cases/messages_2rps.py index df5d221d430..527608d6e62 100644 --- a/tests/e2e/perf/cases/messages_2rps.py +++ b/tests/e2e/perf/cases/messages_2rps.py @@ -6,6 +6,8 @@ from __future__ import annotations from typing import Final +from setup.performance_test_case import Limit + from .direct_vs_bridge import direct_vs_bridge CASE: Final = direct_vs_bridge( @@ -13,4 +15,25 @@ CASE: Final = direct_vs_bridge( "messages", direct="claude-sonnet-5", bridge="claude-sonnet-5-bedrock-converse", + limits=( + Limit("unexpected_status_count", max=3), + Limit("gateway_added_seconds", percentile=50, max=0.296), + Limit("gateway_added_seconds", percentile=99, max=3.36), + Limit("litellm_overhead_seconds", percentile=50, max=0.214), + Limit("litellm_overhead_seconds", percentile=99, max=3.15), + Limit("litellm_response_seconds", percentile=50, max=1.18), + Limit("litellm_response_seconds", percentile=99, max=3.31), + Limit("prometheus_redis_latency_mean_seconds", max=0.00216), + Limit("postgres_active_sessions", max=0.00816), + Limit("postgres_slowest_statement_seconds", max=0.042), + Limit("postgres_peak_connections", max=24), + Limit("time_to_first_token_seconds", percentile=50, max=1.56), + Limit("time_to_first_token_seconds", percentile=99, max=5.07), + Limit("memory_growth_megabytes", max=600), + Limit("cpu_seconds_per_request", max=0.74), + Limit("postgres_calls_per_request", max=2.59), + Limit("postgres_max_statement_per_request", max=0.427), + Limit("postgres_max_background_statement_per_request", max=0.856), + Limit("redis_calls_per_request", max=18.1), + ), ) diff --git a/tests/e2e/perf/cases/responses_2rps.py b/tests/e2e/perf/cases/responses_2rps.py index dd55d536eb6..2781cf5e450 100644 --- a/tests/e2e/perf/cases/responses_2rps.py +++ b/tests/e2e/perf/cases/responses_2rps.py @@ -5,6 +5,8 @@ from __future__ import annotations from typing import Final +from setup.performance_test_case import Limit + from .direct_vs_bridge import direct_vs_bridge CASE: Final = direct_vs_bridge( @@ -12,4 +14,25 @@ CASE: Final = direct_vs_bridge( "responses", direct="gpt-6-luna", bridge="claude-sonnet-5", + limits=( + Limit("unexpected_status_count", max=3), + Limit("gateway_added_seconds", percentile=50, max=0.24), + Limit("gateway_added_seconds", percentile=99, max=2.05), + Limit("litellm_overhead_seconds", percentile=50, max=0.153), + Limit("litellm_overhead_seconds", percentile=99, max=1.8), + Limit("litellm_response_seconds", percentile=50, max=1.11), + Limit("litellm_response_seconds", percentile=99, max=3.27), + Limit("prometheus_redis_latency_mean_seconds", max=0.00228), + Limit("postgres_active_sessions", max=0.011), + Limit("postgres_slowest_statement_seconds", max=0.042), + Limit("postgres_peak_connections", max=26), + Limit("time_to_first_token_seconds", percentile=50, max=1.43), + Limit("time_to_first_token_seconds", percentile=99, max=2.69), + Limit("memory_growth_megabytes", max=600), + Limit("cpu_seconds_per_request", max=0.382), + Limit("postgres_calls_per_request", max=2.59), + Limit("postgres_max_statement_per_request", max=0.426), + Limit("postgres_max_background_statement_per_request", max=1.51), + Limit("redis_calls_per_request", max=15.7), + ), )