From 8d9b9926d1658df27fd560264f017a1497b81e7a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 19:32:03 -0700 Subject: [PATCH] test(perf): add chat_completions_2rps, responses_2rps and messages_2rps direct vs bridge cases (#47752) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/gateway/perf_ci_config.yml | 12 +++++++ tests/e2e/perf/cases/chat_completions_2rps.py | 15 +++++++++ tests/e2e/perf/cases/direct_vs_bridge.py | 31 +++++++++++++++++++ tests/e2e/perf/cases/messages_2rps.py | 16 ++++++++++ tests/e2e/perf/cases/responses_2rps.py | 15 +++++++++ tests/e2e/perf/setup/observe.py | 7 +++++ 6 files changed, 96 insertions(+) create mode 100644 tests/e2e/perf/cases/chat_completions_2rps.py create mode 100644 tests/e2e/perf/cases/direct_vs_bridge.py create mode 100644 tests/e2e/perf/cases/messages_2rps.py create mode 100644 tests/e2e/perf/cases/responses_2rps.py diff --git a/tests/e2e/gateway/perf_ci_config.yml b/tests/e2e/gateway/perf_ci_config.yml index 7ab6637e985..767db313847 100644 --- a/tests/e2e/gateway/perf_ci_config.yml +++ b/tests/e2e/gateway/perf_ci_config.yml @@ -46,6 +46,11 @@ model_list: model: openai/gpt-6-luna api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL api_key: perf-mock-not-checked + - model_name: gpt-6-luna-responses + litellm_params: + model: openai/responses/gpt-6-luna + api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL + api_key: perf-mock-not-checked - model_name: claude-sonnet-5 litellm_params: model: anthropic/claude-sonnet-5 @@ -58,6 +63,13 @@ model_list: aws_region_name: us-east-1 aws_access_key_id: perf-mock-not-checked aws_secret_access_key: perf-mock-not-checked + - model_name: claude-sonnet-5-bedrock-converse + litellm_params: + model: bedrock/converse/anthropic.claude-sonnet-5 + aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL + aws_region_name: us-east-1 + aws_access_key_id: perf-mock-not-checked + aws_secret_access_key: perf-mock-not-checked guardrails: - guardrail_name: perf-pre litellm_params: diff --git a/tests/e2e/perf/cases/chat_completions_2rps.py b/tests/e2e/perf/cases/chat_completions_2rps.py new file mode 100644 index 00000000000..9792d99c14f --- /dev/null +++ b/tests/e2e/perf/cases/chat_completions_2rps.py @@ -0,0 +1,15 @@ +"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions +route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route.""" + +from __future__ import annotations + +from typing import Final + +from .direct_vs_bridge import direct_vs_bridge + +CASE: Final = direct_vs_bridge( + "chat_completions_2rps", + "chat_completions", + direct="gpt-6-luna", + bridge="gpt-6-luna-responses", +) diff --git a/tests/e2e/perf/cases/direct_vs_bridge.py b/tests/e2e/perf/cases/direct_vs_bridge.py new file mode 100644 index 00000000000..13f609c8299 --- /dev/null +++ b/tests/e2e/perf/cases/direct_vs_bridge.py @@ -0,0 +1,31 @@ +"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on +that provider's own API (direct) and half on a deployment it has to translate to another provider +API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as +`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the +translation's cost.""" + +from __future__ import annotations + +from dataclasses import replace + +from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey + +from .mixed_size_requests_2rps import CASE as MIXED + + +def _scoped(key: VirtualKey, direct: str) -> VirtualKey: + return replace(key, models=(direct,)) if key.models else key + + +def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase: + return replace( + MIXED, + name=name, + teams=( + replace(MIXED.team("team-a"), models=(direct, bridge)), + replace(MIXED.team("team-b"), models=(bridge,)), + ), + virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys), + model_groups=(direct, bridge), + traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}), + ) diff --git a/tests/e2e/perf/cases/messages_2rps.py b/tests/e2e/perf/cases/messages_2rps.py new file mode 100644 index 00000000000..df5d221d430 --- /dev/null +++ b/tests/e2e/perf/cases/messages_2rps.py @@ -0,0 +1,16 @@ +"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route, +`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the +Bedrock mock's Converse route.""" + +from __future__ import annotations + +from typing import Final + +from .direct_vs_bridge import direct_vs_bridge + +CASE: Final = direct_vs_bridge( + "messages_2rps", + "messages", + direct="claude-sonnet-5", + bridge="claude-sonnet-5-bedrock-converse", +) diff --git a/tests/e2e/perf/cases/responses_2rps.py b/tests/e2e/perf/cases/responses_2rps.py new file mode 100644 index 00000000000..dd55d536eb6 --- /dev/null +++ b/tests/e2e/perf/cases/responses_2rps.py @@ -0,0 +1,15 @@ +"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route, +`claude-sonnet-5` is bridged to the Anthropic mock's Messages route.""" + +from __future__ import annotations + +from typing import Final + +from .direct_vs_bridge import direct_vs_bridge + +CASE: Final = direct_vs_bridge( + "responses_2rps", + "responses", + direct="gpt-6-luna", + bridge="claude-sonnet-5", +) diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.py index ccdae6398a9..1532a78606e 100644 --- a/tests/e2e/perf/setup/observe.py +++ b/tests/e2e/perf/setup/observe.py @@ -565,6 +565,7 @@ def _diagnostics(observation: Observation) -> list[str]: return [ " diagnostics (not limits):", f" bodies off reply {observation.bodies_off_reply()}", + f" mock routes {_mock_routes(observation)}", f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}", f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}", f" retried share {len(observation.retried_markers) / count:.4f}", @@ -575,6 +576,12 @@ def _diagnostics(observation: Observation) -> list[str]: ] +def _mock_routes(observation: Observation) -> str: + """Which provider route each request landed on, so a direct-vs-bridge case shows its split.""" + counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests) + return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a" + + def _datastore_lines(observation: Observation) -> list[str]: """What the per-request Postgres and Redis counts are made of, so a red run names the statement or command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""