test(e2e): sync main suite into rc 1.108.0

Copy the canonical E2E suite and harness from main at f91774e2f9. Preserve replacement integration coverage for cases moved out of E2E. Keep the older integration harness compatible by omitting its unsupported shares_worker_state marker. Application code is unchanged.
This commit is contained in:
Yuneng Jiang 2026-11-02 11:17:47 -08:00
parent 61fe5be5b8
commit 98deebe78c
No known key found for this signature in database
107 changed files with 6874 additions and 3524 deletions

View file

@ -172,12 +172,7 @@ def open_consent() -> None:
raise RuntimeError("consent page timed out")
@pytest.mark.mcp_oauth_live
def test_oauth_dies_on_consent() -> None:
open_consent()
def test_plain_dies_on_consent() -> None:
def test_dies_on_consent() -> None:
open_consent()
"""
@ -353,16 +348,13 @@ class TestStepsReachTheReport:
def test_a_failed_phase_s_own_report_carries_the_steps(tmp_path: Path) -> None:
"""Plugins that read the failed setup or call report, not the teardown one
junitxml writes from, see where the test died too, oauth-live or not."""
junitxml writes from, see where the test died too."""
write_suite(tmp_path, {"test_consent.py": FAILED_PHASE_SUITE, "report_spy.py": REPORT_SPY_PLUGIN})
child: Final = run_child_pytest(tmp_path, "-p", "report_spy", env={"E2E_MCP_OAUTH_LIVE": "1"})
child: Final = run_child_pytest(tmp_path, "-p", "report_spy")
seen_path: Final = tmp_path / "failed-reports.jsonl"
assert seen_path.exists(), f"no failed report reached the spy:\n{child.stdout}\n{child.stderr}"
seen: Final = dict(map(FailedReport.validate_json, seen_path.read_text().splitlines()))
assert seen == {
"test_oauth_dies_on_consent": ("open the consent page",),
"test_plain_dies_on_consent": ("open the consent page",),
}, child.stdout
assert seen == {"test_dies_on_consent": ("open the consent page",)}, child.stdout
class TestDeclaredPropertiesReachTheReport:

File diff suppressed because one or more lines are too long

View file

@ -90,18 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell
They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds)
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it:
```yaml
general_settings:
store_prompts_in_spend_logs: true
```
Alternatively, start the proxy with `STORE_PROMPTS_IN_SPEND_LOGS=true litellm --config <your-e2e-config>.yml --port 4000`. Setting the variable only on the pytest process does not configure the proxy. With prompt storage disabled, the proxy correctly redacts `guardrail_response`, so that configuration cannot exercise this test's entity-detail assertions. Enable this only on a test stack using synthetic prompts
A couple of logging destinations are configured on the proxy rather than by the test. The Weave tests scope their callback to the key they create, but litellm builds the `weave_otel` logger from `WANDB_API_KEY` and `WANDB_PROJECT_ID` before it applies the per-key vars, so the proxy needs both in its own environment or the key-scoped callback never initializes and nothing ships
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and start the proxy with `LITELLM_OTEL_V2=true`, the `otel` callback, and `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy.
### Secret manager lanes
@ -261,55 +250,3 @@ The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthr
Excluded transport and telemetry headers are `host`, `content-length`, `connection`, `accept-encoding`, `user-agent`, `traceparent`, `tracestate`, `x-request-id`, `x-client-request-id` and `x-stainless-*`. Inbound transfer-encoding is unsupported; send JSON with content-length framing. The destination represents host identity and the relay carries original body bytes. Replay does not verify credentials, SDK timeout/retry behavior, transport performance, model availability or stateful remote IDs. Live relay uses original request bytes and header values, never the stored identity
Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The `python` job in `.github/workflows/test-linting.yml` runs them alongside the provider-edge and fixture tests at the root of `tests/e2e_harness/` with `--noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures`; they need only synthetic HTTP providers and temporary fixture storage
## MCP OAuth happy path
`test_mcp_oauth_happy_path_e2e.py` runs one shared scenario with four variants:
aggregate gateway SSO and explicitly configured per-server JWT, each directly
against Linear and through the live provider edge. The edge forwards to real
Linear without replay and compares the forwarded bearer to the encrypted
canonical user/server credential. This observes the forwarding boundary, not
Linear's internal logs. Direct variants independently exercise discovery
Use the existing database preparation, Prisma generation and Keycloak setup.
Build and stage the dashboard from the tested checkout as in the UI runner.
Provide `DATABASE_URL`, `LITELLM_MASTER_KEY`, `LITELLM_SALT_KEY`, `LITELLM_LICENSE`,
and the `E2E_KEYCLOAK_*` settings. Capture a test-account Linear login using
`mcp/linear_session_capture.py` and set `E2E_LINEAR_STORAGE_STATE` to that private
file. The test workspace must contain a team. Do not publish browser state or
raw test/proxy output
```bash
E2E_MCP_OAUTH_LIVE=1 E2E_FIXTURE_MODE=live E2E_PROVIDER_CACHE=0 \
uv run --no-sync pytest tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
--rootdir=. --reruns 0
```
The test starts and restarts its own source-built proxy on a free loopback port,
retaining its database and SSO client but no Redis or process-local cache. It
does not restart an existing proxy or clear shared databases. Gateway login,
consent, immediate list/call and post-restart reconnect must all succeed. The
aggregate client never injects a gateway header; the explicitly labeled JWT
variant configures `x-litellm-api-key` for the first consent and reconnects with
only its gateway JWT after restart
`.github/workflows/test-mcp-oauth-e2e.yml` automatically requests a run for
same-repository pull requests changing MCP, gateway authentication/SSO, consent
UI, dependencies or the relevant E2E harness/workflow paths. It retains manual
`workflow_dispatch` for targeted verification. The four cases run in the
protected `e2e-changed` environment after its normal deployment approval;
reviewers should approve and inspect this separate OAuth check when it appears.
Fork pull requests do not run this credentialed job; use a reviewed
same-repository branch for their verification. The workflow's path-filtered
check is not configured here as a globally required branch-protection check.
Provision `E2E_LINEAR_STORAGE_STATE_B64` as a secret there and retain the existing E2E license/AWS role configuration. A missing or
expired session fails the job; collection, deselection and skips are not passes.
No LLM call is needed
Coverage remains limited to authorization-code OAuth over HTTP. M2M, OBO,
PKCE passthrough, static/BYOK, ID-JAG, forwarding, SigV4 and stdio are outside this
scenario; consult the registry and LIT-3559 for their existing coverage and gaps.
LIT-4506 owns broader isolation/failure regressions. LIT-7737 retains ownership
of dependency/Python compatibility and its matrix; this test reuses its delivered
environment and does not change dependency constraints or compatibility gates

View file

@ -18,7 +18,7 @@ from dataclasses import dataclass
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import settle_propagation
from e2e_http import NoBody, Result, Success, get_external, is_ok
from e2e_http import NoBody, Result, Success, is_ok
from e2e_metadata import STEP_FRAMES, step
from proxy_client import ProxyClient
@ -36,11 +36,6 @@ class A2ASkill(BaseModel):
examples: list[str] | None = None
class A2AProvider(BaseModel):
organization: str
url: str
class AgentCardParams(BaseModel):
"""The upstream agent card an admin registers. `protocolVersion` is the field the
proxy validates against SUPPORTED_A2A_PROTOCOL_VERSIONS on registration."""
@ -57,29 +52,6 @@ class AgentCardParams(BaseModel):
preferred_transport: str | None = Field(default=None, serialization_alias="preferredTransport")
class UpstreamAgentCard(BaseModel):
"""A real published agent card parsed from a public /.well-known endpoint. Keys on
the A2A wire aliases so `model_validate_json` reads the served JSON and
`model_dump(by_alias=True)` re-emits it unchanged for verbatim registration; it is
only ever fetched-and-validated, never hand-constructed, so aliasing on the wire
names does not affect any call site."""
model_config = ConfigDict(populate_by_name=True)
protocol_version: str = Field(alias="protocolVersion")
name: str
description: str
version: str
url: str
provider: A2AProvider | None = None
documentation_url: str | None = Field(default=None, alias="documentationUrl")
capabilities: A2ACapabilities = A2ACapabilities()
skills: list[A2ASkill]
default_input_modes: list[str] = Field(default=["text"], alias="defaultInputModes")
default_output_modes: list[str] = Field(default=["text"], alias="defaultOutputModes")
preferred_transport: str | None = Field(default=None, alias="preferredTransport")
class A2ABridgeParams(BaseModel):
"""litellm_params that route the agent through the completion bridge: an A2A
message/send is transformed into a litellm.acompletion against this provider."""
@ -93,7 +65,7 @@ class A2ABridgeParams(BaseModel):
class AgentRegisterBody(BaseModel):
agent_name: str
agent_card_params: AgentCardParams | UpstreamAgentCard
agent_card_params: AgentCardParams
litellm_params: A2ABridgeParams | None = None
@ -136,32 +108,9 @@ class A2ATextPart(BaseModel):
text: str
class A2ASearchPropertiesParams(BaseModel):
"""The strict param schema of the published property agent's `search_properties`
skill (unknown keys are rejected upstream), so a natural-language query like
"properties for sale in SF under $2M" is expressed as typed fields."""
un_locode: str | None = None
service_type: str | None = None
property_type: str | None = None
bedrooms_min: int | None = None
asking_price_max: float | None = None
limit: int | None = None
class A2ASkillInvocation(BaseModel):
skill: str
params: A2ASearchPropertiesParams
class A2ADataPart(BaseModel):
kind: str = "data"
data: A2ASkillInvocation
class A2AOutboundMessage(BaseModel):
role: str = "user"
parts: list[A2ATextPart | A2ADataPart]
parts: list[A2ATextPart]
message_id: str = Field(serialization_alias="messageId")
@ -189,57 +138,10 @@ class A2AResponseMessage(BaseModel):
parts: list[A2AResponsePart] = []
class A2ATaskStatus(BaseModel):
state: str | None = None
message: A2AResponseMessage | None = None
class A2AListingLocation(BaseModel):
"""Only the location fields a test reads back off a returned listing."""
un_locode: str | None = None
class A2AListing(BaseModel):
"""A single property card from the agent's `search_results` artifact; only the
identity/location fields a test asserts on are modelled."""
raia_id: str
property_type: str | None = None
service_type: str | None = None
location: A2AListingLocation = A2AListingLocation()
class A2ASearchResults(BaseModel):
"""The DataPart payload the property agent's `search_properties` skill returns:
the run count plus the listing cards themselves. Proof the tool actually ran and
matched, not just that the task completed with some text."""
total: int
count: int
listings: list[A2AListing] = []
class A2AArtifactPart(BaseModel):
kind: str | None = None
data: A2ASearchResults | None = None
class A2AArtifact(BaseModel):
model_config = ConfigDict(populate_by_name=True)
artifact_id: str | None = Field(default=None, alias="artifactId")
name: str | None = None
parts: list[A2AArtifactPart] = []
class A2AResult(BaseModel):
"""A message/send result. In 0.3 the message fields sit directly on the result
(`kind`/`role`/`parts`); in 1.0 they are nested under `message`; a real agent that
runs a task replies with a `task` whose agent text lives on `status.message` and
whose tool output lives on `artifacts`. `text` reads the agent's reply from
whichever shape the served version produced; `search_results` reads the tool's
structured output when the agent ran a skill."""
(`kind`/`role`/`parts`); in 1.0 they are nested under `message`. `text` reads the
agent's reply from whichever shape the served version produced."""
model_config = ConfigDict(populate_by_name=True)
@ -248,8 +150,6 @@ class A2AResult(BaseModel):
message_id: str | None = Field(default=None, alias="messageId")
parts: list[A2AResponsePart] = []
message: A2AResponseMessage | None = None
status: A2ATaskStatus | None = None
artifacts: list[A2AArtifact] = []
@property
def text(self) -> str:
@ -257,8 +157,6 @@ class A2AResult(BaseModel):
parts = self.message.parts
elif self.parts:
parts = self.parts
elif self.status is not None and self.status.message is not None:
parts = self.status.message.parts
else:
parts = []
return "".join(part.text or "" for part in parts)
@ -267,14 +165,6 @@ class A2AResult(BaseModel):
def is_nested_v1_shape(self) -> bool:
return self.message is not None
@property
def search_results(self) -> A2ASearchResults | None:
for artifact in self.artifacts:
for part in artifact.parts:
if part.data is not None:
return part.data
return None
class A2AError(BaseModel):
code: int
@ -380,11 +270,3 @@ class A2AClient:
def build_a2a_client(proxy: ProxyClient) -> A2AClient:
return A2AClient(proxy=proxy)
@step("Fetch a published A2A agent card from its /.well-known endpoint")
def fetch_agent_card(url: str, *, timeout: float = 20.0) -> Result[UpstreamAgentCard]:
"""Fetch a live A2A agent card from its /.well-known endpoint and parse it into the
registration model, so a test can register a real published card verbatim rather
than a hand-rolled one."""
return get_external(url, response_type=UpstreamAgentCard, timeout=timeout)

View file

@ -17,18 +17,14 @@ import pytest
from a2a_client import (
A2ABridgeParams,
A2AClient,
A2ADataPart,
A2AJsonRpcRequest,
A2AMessageSendParams,
A2AOutboundMessage,
A2ASearchPropertiesParams,
A2ASkill,
A2ASkillInvocation,
A2ATextPart,
AgentCardParams,
AgentRegisterBody,
AgentResponse,
fetch_agent_card,
)
from e2e_config import unique_marker
from e2e_http import Result, UnknownApiError, unwrap
@ -50,9 +46,6 @@ BRIDGE = A2ABridgeParams(
model=BRIDGE_MODEL,
)
MOVEHOME_AGENT_CARD_URL = "https://movehome.org/.well-known/agent.json"
MOVEHOME_ORIGIN = "https://movehome.org"
pytestmark = pytest.mark.e2e
@ -135,48 +128,6 @@ class TestA2AAgentLifecycle:
assert result is not None
assert result.text != ""
@pytest.mark.covers("other.a2a.message_send.real_world_agent_replies")
@meta(
Subject(
domain=Domain.AGENTS_API,
route=Route.A2A,
)
)
def test_real_world_agent_replies_to_property_query(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
upstream = unwrap(fetch_agent_card(MOVEHOME_AGENT_CARD_URL)).model_copy(update={"url": MOVEHOME_ORIGIN})
assert upstream.protocol_version == "0.3.0"
marker = unique_marker()
body = AgentRegisterBody(agent_name=f"e2e-a2a-real-{marker}", agent_card_params=upstream)
agent = unwrap(client.register_agent(body))
resources.defer(lambda: client.delete_agent(agent.agent_id))
assert agent.agent_card_params.protocol_version == "0.3"
location = "GBLON"
request = A2AJsonRpcRequest(
id=f"e2e-{unique_marker()}",
params=A2AMessageSendParams(
message=A2AOutboundMessage(
parts=[
A2ADataPart(
data=A2ASkillInvocation(
skill="search_properties",
params=A2ASearchPropertiesParams(un_locode=location, service_type="long_term", limit=3),
)
)
],
message_id=unique_marker(),
)
),
)
response = unwrap(client.send_message(agent.agent_id, scoped_key, request))
assert response.error is None
assert response.result is not None
results = response.result.search_results
assert results is not None, "agent returned no search_results artifact; skill did not run"
assert results.total > 0
assert results.listings, "search_properties matched nothing; agent returned no property cards"
assert all(listing.raia_id for listing in results.listings)
assert all(listing.location.un_locode == location for listing in results.listings)
@pytest.mark.covers("other.a2a.discovery.proxy_fronted_card")
@meta(
Subject(

View file

@ -15,7 +15,7 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Final, Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, ConfigDict, Field
from e2e_metadata import step
from proxy_client import ProxyClient
@ -77,6 +77,16 @@ class BatchErrorList(BaseModel):
data: list[BatchErrorItem] = []
class BatchUsage(BaseModel):
model_config = ConfigDict(frozen=True)
prompt_tokens: int | None = None
completion_tokens: int | None = None
total_tokens: int | None = None
input_tokens: int | None = None
output_tokens: int | None = None
class BatchObject(BaseModel):
id: str
object: str | None = None
@ -90,6 +100,7 @@ class BatchObject(BaseModel):
completion_window: str | None = None
created_at: int | None = None
model: str | None = None
usage: BatchUsage | None = None
class BatchList(BaseModel):

View file

@ -1893,6 +1893,13 @@ class TestBatchTerminalState:
f"listed-completed marker retrieved as {fetched.status!r}"
)
assert fetched.output_file_id, "completed batch has no output_file_id"
usage: Final = fetched.usage
assert usage is not None and usage.input_tokens and usage.output_tokens, (
f"completed batch has no batch token counts: {usage!r}"
)
assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), (
f"chat token counts disagree with the batch token counts: {usage!r}"
)
downloaded = client.proxy.transport.download(
f"/v1/files/{fetched.output_file_id}/content",

View file

@ -17,7 +17,6 @@ import functools
import os
from collections.abc import Generator, Iterator
from datetime import datetime, timezone
from pathlib import Path
from types import MappingProxyType
from typing import Final
@ -29,7 +28,6 @@ from e2e_config import (
FIXTURE_DIR,
FIXTURE_MODE_RAW,
MANAGED_FILES_OPT_IN_ENV,
MCP_OAUTH_LIVE_OPT_IN_ENV,
OTEL_TLS_OPT_IN_ENV,
OTEL_V2_OPT_IN_ENV,
OWNED_GATEWAY_OPT_IN_ENV,
@ -72,7 +70,6 @@ OPT_IN_MARKERS: Final = MappingProxyType(
"redis_chaos": PERF_TESTS_OPT_IN_ENV,
"perf": PERF_TESTS_OPT_IN_ENV,
"cli_determinism": CLI_DETERMINISM_OPT_IN_ENV,
"mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV,
"provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV,
"owned_gateway": OWNED_GATEWAY_OPT_IN_ENV,
"otel_v2": OTEL_V2_OPT_IN_ENV,
@ -170,11 +167,6 @@ def pytest_configure(config: pytest.Config) -> None:
"quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; "
"every other test waits for it to finish",
)
config.addinivalue_line(
"markers",
"mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless "
"E2E_MCP_OAUTH_LIVE is set",
)
config.addinivalue_line(
"markers",
"provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the "
@ -331,8 +323,6 @@ def pytest_runtest_setup(item: pytest.Item) -> None:
item.user_properties.extend(item.config.stash[_IDLE_RSS].junit_properties)
if not _reaches_proxy(item):
return
if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames:
return
reason = _proxy_fail_reason()
if reason is not None:
pytest.fail(reason)
@ -376,12 +366,6 @@ def pytest_runtest_makereport(
report = yield
if report.when in ("setup", "call"):
attach_step_properties(item)
if item.get_closest_marker("mcp_oauth_live") is not None and call.excinfo is not None:
# Publish code locations only, never exception messages, source text or locals.
item.user_properties.append(("oauth_failure_phase", report.when))
item.user_properties.append(("oauth_exception_type", call.excinfo.type.__name__))
for entry in call.excinfo.traceback:
item.user_properties.append(("oauth_frame", f"{Path(entry.path).name}:{entry.lineno + 1}:{entry.name}"))
if report.when == "call":
item.stash[_CALL_PASSED] = report.passed
report.user_properties = list(item.user_properties)

View file

@ -1,11 +1,6 @@
# Guardrail enforcement (behavior features). Grounded in litellm/proxy/guardrails/guardrail_hooks/.
# Rolls up into the "Logging & Guardrails" dashboard module together with logging.*
- {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"}
- {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"}
- {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"}
- {id: guardrail.presidio.post_call.spend_log_stores_masked_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, messages, anthropic_messages_stream, responses], source: "guardrail_hooks/presidio.py", fail_before_fix: proven, rationale: "When an output guardrail masks the response, the spend log stores the masked text the caller received rather than the raw model output, on every endpoint and both stream modes (LIT-8325)"}
- {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"}
- {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"}
- {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"}
- {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"}
- {id: guardrail.litellm_content_filter.pre_call.blocks_video, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [videos], source: "test_key_guardrail_video_e2e.py", fail_before_fix: proven, rationale: "A content-filter guardrail attached to a key (metadata.guardrails) blocks a banned prompt on POST /v1/videos before the provider is called; before the fix the route's call type was unknown to the unified guardrail hook and the prompt went to the provider unscanned (LIT-6685)"}

View file

@ -13,6 +13,8 @@
- {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"}
- {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"}
- {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"}
- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention sent through the proxy, litellm.completion or Router.completion reaches the body sent to OpenAI /v1/chat/completions, so a caller asking for 24h prompt caching gets it instead of a silent drop"}
- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention on an openai/responses/ model (chat completions in, Responses out) reaches the body sent to OpenAI /v1/responses"}
- {id: llm.chat_completions.openai.service_tier.stream.echoes_served_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: stream, assertions: [works], source: "litellm_core_utils/streaming_handler.py", fail_before_fix: proven, rationale: "Every relayed stream chunk carries the service_tier OpenAI stamped on it, so a streaming caller can see which tier served the request"}
- {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"}
- {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"}
@ -43,6 +45,9 @@
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}
- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the AgentCore tool-use id shape"}
- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the AgentCore tool-use id shape"}
- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A short Gemini tool-call id still round-trips when the client drops provider fields"}
- {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"}
- {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"}
@ -68,13 +73,14 @@
- {id: llm.messages.anthropic.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Messages API"}
- {id: llm.messages.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching via Messages API"}
- {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"}
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven}
- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven}
- {id: llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works_for_kimi_k3_max_tokens_1], source: "llm_translation/test_bedrock_kimi_k3_messages_e2e.py", rationale: "Bedrock Converse accepts max_tokens=1 on the Anthropic Messages surface after LiteLLM applies the model minimum", fail_before_fix: proven}
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven}
- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven}
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
- {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
- {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input and missing model are rejected"}
- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
@ -133,11 +139,6 @@
- {id: llm.responses.ollama.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses streams text deltas, usage and a terminal event"}
- {id: llm.responses.ollama.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses returns one addressable get_weather call"}
- {id: llm.responses.ollama.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses tool result round trip reaches the model"}
- {id: llm.chat_completions.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "service_tier balanced and auto map to Sail completion windows and bill the matching price columns; Sail serves flex only to background responses and Batch"}
- {id: llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [rejects_unknown_tier], source: "llm_translation/test_sail_e2e.py", rationale: "A service_tier Sail has no completion window for is a 400 without drop_params"}
- {id: llm.chat_completions.sail.service_tier.nonstream.drops_unknown_tier_and_bills_asap, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [drops_unknown_tier_and_bills_asap], source: "llm_translation/test_sail_e2e.py", rationale: "An unknown service_tier under drop_params is dropped and billed at asap in both the cost header and spend log"}
- {id: llm.responses.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "A caller metadata.completion_window of balanced on /v1/responses bills Sail balanced rates"}
- {id: llm.messages.sail.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: sail, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_sail_e2e.py", rationale: "Sail over /v1/messages"}
- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"}
- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"}
- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"}
@ -152,3 +153,4 @@
- {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"}
- {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"}
- {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"}
- {id: llm.responses.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "llm_translation/test_responses_codex_tool_search_e2e.py", rationale: "Codex client tool_search over /v1/responses finds and loads a deferred tool on Claude"}

View file

@ -2,6 +2,7 @@
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings, refuse chat with a gateway 400, and route chat through a live fallback; zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}

View file

@ -1,4 +1,4 @@
# MCP module. Grounded in litellm/proxy/_experimental/mcp_server/. See tests/e2e/AGENTS.md for the grammar.
# MCP module. Grounded in litellm/proxy/mcp_server/. See tests/e2e/AGENTS.md for the grammar.
- id: mcp.list_tools.api_key.succeeds
module: mcp
tier: P0
@ -79,14 +79,6 @@
assertions: [succeeds]
source: "db.py user_oauth_credential lookup"
rationale: OAuth2 token passthrough; per-user credential storage
- id: mcp.call_tool.oauth.persists_across_processes
module: mcp
tier: P1
operation: call_tool
auth_family: oauth
assertions: [persists_across_processes]
source: "outbound_credentials/per_user_oauth_store.py V2PerUserTokenStore"
rationale: Stored per-user token survives a verified restart of an owned gateway with no Redis cache
- id: mcp.list_tools.none.succeeds
module: mcp
tier: P1

View file

@ -55,7 +55,6 @@
- {id: other.a2a.register.persists, module: other, tier: P1, area: a2a, assertions: [persists], source: "agent_endpoints/endpoints.py:325-443", rationale: "POST /v1/agents registers an agent card; GET /v1/agents/{id} reads it back"}
- {id: other.a2a.register.unsupported_version_rejected, module: other, tier: P1, area: a2a, assertions: [unsupported_version_rejected], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a protocolVersion outside SUPPORTED_A2A_PROTOCOL_VERSIONS is refused with 400"}
- {id: other.a2a.register.semver_version_accepted, module: other, tier: P1, area: a2a, assertions: [semver_version_accepted], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a patch-level semver like 0.3.0 (what the Google A2A SDK emits) registers, stores and serves the canonical 0.3 rather than 400ing; regression guard for the v1.92 report"}
- {id: other.a2a.message_send.real_world_agent_replies, module: other, tier: P1, area: a2a, assertions: [real_world_agent_replies], source: "agent_endpoints/a2a_endpoints.py asend_message", rationale: "A real published a2a agent fetched live from a public /.well-known endpoint (pinning the full semver 0.3.0 the a2a-sdk emits) registers, serves the canonical 0.3, and a message/send skill invocation proxies to the live upstream and returns the agent's reply"}
- {id: other.a2a.register.malformed_version_rejected, module: other, tier: P1, area: a2a, assertions: [malformed_version_rejected], source: "a2a/agent_card.py normalize_protocol_version", rationale: "A malformed protocolVersion like 0.3.garbage fails full-string semver validation and is refused with 400 instead of truncating to a supported family"}
- {id: other.a2a.discovery.proxy_fronted_card, module: other, tier: P1, area: a2a, assertions: [proxy_fronted_card], source: "agent_endpoints/a2a_endpoints.py get_agent_card", rationale: "/.well-known/agent-card.json serves the proxy url + supportedInterfaces and the LiteLLM virtual-key bearer scheme, not the upstream"}
- {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"}

View file

@ -41,7 +41,7 @@
- {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"}
- {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"}
- {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"}
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
- {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"}
- {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"}
- {id: quota_management.spend_tracking.discount_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: discount_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider discount changes the real request's spend row"}
@ -62,11 +62,11 @@
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"}
- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"}
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
- {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"}
- {id: quota_management.spend_tracking.service_tier_stream.records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", fail_before_fix: proven, rationale: "A streamed call with no service_tier requested bills at the rates of the tier OpenAI stamps on its chunks and records that served tier on the row; the reassembled stream dropped the provider tier so the row recorded none and priced at the default rates"}
- {id: quota_management.spend_tracking.service_tier_stream.responses_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [responses], source: "responses/streaming_iterator.py", rationale: "A streamed /v1/responses call bills at the tier carried on the response.completed event's inner response and records that served tier on the spend row"}
- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/pass_through/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"}
- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/messages/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"}
- {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"}
- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"}
- {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"}

View file

@ -81,6 +81,7 @@ LlmCapability = Literal[
"pdf_input",
"logprobs",
"prompt_cache_1h",
"prompt_cache_24h",
"prompt_cache_5m",
"response_headers",
"service_tier",

View file

@ -111,10 +111,6 @@ CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haik
CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5")
S3_PARTITION_GRANULARITY = os.environ.get("E2E_S3_PARTITION_GRANULARITY", "day")
LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp")
LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "")
LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.linear.app when PR #33787 landed
# Jaeger query API of the compose stack's OTEL trace destination (the `jaeger`
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
# read exported spans back through it.
@ -207,7 +203,6 @@ MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK"
PERF_TESTS_OPT_IN_ENV: Final = "E2E_PERF_TESTS"
CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE"
OWNED_GATEWAY_OPT_IN_ENV: Final = "E2E_OWNED_GATEWAY"
OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2"

View file

@ -499,6 +499,30 @@ def get_external[R: BaseModel](
return classify(resp, response_type)
def get_text_external(
url: str,
*,
headers: BaseModel | None = None,
timeout: float = 30.0,
ca_bundle: str | None = None,
) -> str | NetworkError:
"""GET an absolute URL outside the proxy whose body is plain text (e.g. a pod log), not JSON.
A non-2xx status is reported as a NetworkError so the caller can fail loudly. JSON is accepted too so an API
that answers an error as a JSON document (the Kubernetes Status on a pod log) returns that error instead of a 406."""
try:
resp = requests.get(
url,
headers={"Accept": "text/plain, application/json", **(_headers(headers) if headers is not None else {})},
timeout=timeout,
verify=ca_bundle if ca_bundle is not None else True,
)
except requests.RequestException as exc:
return NetworkError(message=str(exc))
if not (200 <= resp.status_code < 300):
return NetworkError(message=f"{resp.status_code} from {url}: {resp.text[:300]}")
return resp.text
def post_external[R: BaseModel](
url: str,
*,

View file

@ -27,6 +27,8 @@ litellm_settings:
callbacks:
- prometheus
- otel
service_callback:
- prometheus_system
require_auth_for_metrics_endpoint: false
drop_params: true
request_timeout: 120
@ -44,6 +46,11 @@ model_list:
model: openai/gpt-6-luna
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
api_key: perf-mock-not-checked
- model_name: gpt-6-luna-responses
litellm_params:
model: openai/responses/gpt-6-luna
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
api_key: perf-mock-not-checked
- model_name: claude-sonnet-5
litellm_params:
model: anthropic/claude-sonnet-5
@ -56,6 +63,13 @@ model_list:
aws_region_name: us-east-1
aws_access_key_id: perf-mock-not-checked
aws_secret_access_key: perf-mock-not-checked
- model_name: claude-sonnet-5-bedrock-converse
litellm_params:
model: bedrock/converse/anthropic.claude-sonnet-5
aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL
aws_region_name: us-east-1
aws_access_key_id: perf-mock-not-checked
aws_secret_access_key: perf-mock-not-checked
guardrails:
- guardrail_name: perf-pre
litellm_params:

View file

@ -32,7 +32,7 @@ litellm_settings:
- host: 127.0.0.1
port: 6379
ssl: true
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
callbacks: ["datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: e2e-search
@ -97,9 +97,3 @@ files_settings:
finetune_settings:
- custom_llm_provider: openai
api_key: os.environ/OPENAI_API_KEY
mcp_servers:
devin:
url: "https://mcp.devin.ai/mcp"
auth_type: api_key
auth_value: os.environ/DEVIN_API_KEY

View file

@ -41,8 +41,6 @@ from pydantic import BaseModel, Field
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"]
PiiAction = Literal["MASK", "BLOCK"]
BlockedWordAction = Literal["BLOCK", "MASK"]
@ -80,22 +78,6 @@ class BlockCodeExecutionParamsBody(GuardrailParamsBase):
guardrail: Literal["block_code_execution"] = "block_code_execution"
class PresidioParamsBody(GuardrailParamsBase):
"""Presidio PII guardrail params. `presidio_filter_scope="input"` keeps the
registration to a single callback on the configured mode; the default
("both") also registers a second post_call output-masking callback, which a
pre_call- or logging_only-scoped test must not drag in. `output_parse_pii`
stays unset/False: True would unmask the response back to the caller."""
guardrail: Literal["presidio"] = "presidio"
presidio_analyzer_api_base: str
presidio_anonymizer_api_base: str
presidio_filter_scope: Literal["input", "output", "both"] | None = None
presidio_language: str | None = None
output_parse_pii: bool | None = None
pii_entities_config: dict[PiiEntity, PiiAction] | None = None
class ToolPermissionRuleBody(BaseModel):
"""One tool_permission rule: a decision for the tool named by `tool_name`."""
@ -130,7 +112,6 @@ GuardrailParamsBody = (
| BedrockGuardrailParamsBody
| OpenAIModerationParamsBody
| BlockCodeExecutionParamsBody
| PresidioParamsBody
| ToolPermissionParamsBody
| CustomCodeParamsBody
)

View file

@ -1,857 +0,0 @@
"""Live e2e: the Presidio PII guardrail masks, per its configured hook point.
pre_call: the guardrail calls the Presidio analyzer/anonymizer on the request
messages BEFORE the model runs, so the model only ever sees placeholders like
<EMAIL_ADDRESS>. A prompt asking the model to repeat a fake email + phone back
must come back with the placeholders echoed and the raw PII absent, on
/chat/completions and on /v1/messages (Anthropic format).
post_call: the mirror hook. The request reaches the model unmasked and the
MODEL OUTPUT is what gets anonymized, so the caller never receives raw PII the
model repeated back. The two hooks are told apart behaviorally rather than by
configuration: the post_call prompt asks for a value derived from the raw email
(its local part, which is not itself an entity Presidio masks) alongside the
address itself, so the answer proves the model saw the raw address while the
address in the same response comes back as <EMAIL_ADDRESS>.
The analyzer/anonymizer endpoints come from PRESIDIO_ANALYZER_API_BASE /
PRESIDIO_ANONYMIZER_API_BASE; missing env is a hard failure, never a skip.
Each guardrail registers with an explicit presidio_filter_scope so only the
configured hook's callback exists (the default "both" registers input masking
AND a post_call output masker), and is deleted on teardown.
The spend-log audit test requires general_settings.store_prompts_in_spend_logs:
true in the proxy config, or STORE_PROMPTS_IN_SPEND_LOGS=true in the proxy
process environment before startup. Setting it only on pytest has no effect.
Without this opt-in, redacting guardrail_response is expected proxy behavior;
this suite deliberately requires the detected-entity details to remain visible.
"""
from __future__ import annotations
import os
import re
import time
from collections.abc import Callable, Iterator
from typing import Final, Literal
import pytest
from pydantic import BaseModel, JsonValue, TypeAdapter
from e2e_config import unique_marker
from e2e_http import Result, StreamingResponse, Success
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from guardrails_client import GuardrailMode, GuardrailsClient, PiiAction, PiiEntity, PresidioParamsBody
from lifecycle import ResourceManager
from models import (
AnthropicMessagesResponse,
ChatResponse,
GuardrailEntityMatch,
GuardrailRunRecord,
SpendLogRow,
)
pytestmark = pytest.mark.e2e
MODEL = "gemini-2.5-flash"
# A guardrail created via POST /guardrails reaches the worker that served the
# create immediately, but every other worker only picks it up on its next
# periodic DB sync (~30s), so the first requests can be served unguarded.
GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0
GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0
# Presidio's anonymizer replaces a detected entity with its unnumbered type
# placeholder, e.g. <EMAIL_ADDRESS>. The pre_call assertions match on the bare
# token because the model is echoing the masked prompt and may not preserve the
# angle brackets; the logged payload keeps the placeholder verbatim.
MASKED_EMAIL_TOKEN = "EMAIL_ADDRESS"
MASKED_PHONE_TOKEN = "PHONE_NUMBER"
MASKED_CREDIT_CARD_TOKEN = "CREDIT_CARD"
# Fictional NANP 555 number; a standard format Presidio's phone recognizer detects.
FAKE_PHONE = "+1 415-555-0134"
FAKE_VISA_TEST_CARD = "4111 1111 1111 1111"
_CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}")
_CONTENT_KEYS: Final = frozenset({"content", "text"})
def _presidio_bases() -> tuple[str, str]:
analyzer = os.environ.get("PRESIDIO_ANALYZER_API_BASE", "").strip()
anonymizer = os.environ.get("PRESIDIO_ANONYMIZER_API_BASE", "").strip()
if not analyzer or not anonymizer:
pytest.fail(
"Presidio e2e requires PRESIDIO_ANALYZER_API_BASE and PRESIDIO_ANONYMIZER_API_BASE "
"(the running Presidio analyzer/anonymizer services); missing env is a hard failure, not a skip"
)
return analyzer, anonymizer
def _register_presidio(
client: GuardrailsClient,
resources: ResourceManager,
*,
name: str,
mode: GuardrailMode | list[GuardrailMode] = "pre_call",
filter_scope: Literal["input", "output", "both"] | None = "input",
entities: dict[PiiEntity, PiiAction] | None = None,
) -> None:
analyzer, anonymizer = _presidio_bases()
guardrail_id = client.register(
name,
PresidioParamsBody(
mode=mode,
default_on=False,
presidio_analyzer_api_base=analyzer,
presidio_anonymizer_api_base=anonymizer,
presidio_filter_scope=filter_scope,
pii_entities_config=entities,
),
)
resources.defer(lambda: client.delete_guardrail(guardrail_id))
def _fake_email() -> str:
return f"jane.doe.{unique_marker()}@example.com"
def _pii_prompt(marker: str, email: str) -> str:
return (
f"{marker} Repeat this sentence back to me exactly, word for word: "
f"My email address is {email} and my phone number is {FAKE_PHONE}."
)
def _first_content(response: ChatResponse) -> str:
if not response.choices:
return ""
message = response.choices[0].message
return (message.content if message else None) or ""
class _StreamDelta(BaseModel):
content: str | None = None
class _StreamChoice(BaseModel):
delta: _StreamDelta
class _StreamChunk(BaseModel):
choices: tuple[_StreamChoice, ...] = ()
class _AnthropicStreamDelta(BaseModel):
type: str | None = None
text: str | None = None
class _AnthropicStreamEvent(BaseModel):
type: str
delta: _AnthropicStreamDelta | None = None
def _credit_card_prompt(marker: str) -> str:
return (
f"{marker} Reply with only the well known Visa sandbox test card number that starts with 4111, "
"the 16 digits grouped in fours separated by spaces, and nothing else."
)
def _passes_luhn(digits: str) -> bool:
checksum = sum(
digit if position % 2 == 0 else (digit * 2 - 9 if digit * 2 > 9 else digit * 2)
for position, digit in enumerate(int(char) for char in reversed(digits))
)
return checksum % 10 == 0
def _contains_card_number(text: str) -> bool:
"""Presidio's CREDIT_CARD recognizer only reports Luhn-valid digit runs, so a
Luhn-invalid number the model hallucinates is not something masking can catch."""
return any(
13 <= len(digits) <= 19 and _passes_luhn(digits)
for digits in (re.sub(r"[ -]", "", match.group()) for match in _CARD_DIGIT_RUN.finditer(text))
)
def _stream_content(result: StreamingResponse) -> str:
return "".join(
choice.delta.content
for event in result.stream_events
if event != "[DONE]"
for choice in _StreamChunk.model_validate_json(event).choices[:1]
if choice.delta.content
)
def _anthropic_stream_content(result: StreamingResponse) -> str:
return "".join(
event.delta.text
for payload in result.stream_events
for event in [_AnthropicStreamEvent.model_validate_json(payload)]
if event.type == "content_block_delta"
and event.delta is not None
and event.delta.type == "text_delta"
and event.delta.text
)
def _messages_text(response: AnthropicMessagesResponse) -> str:
"""The text of a /v1/messages answer, whichever shape the proxy produced
(Anthropic-native content blocks or OpenAI-normalized choices)."""
parts: list[str] = []
for block in response.content or []:
if block.text:
parts.append(block.text)
for choice in response.choices or []:
if choice.message and choice.message.content:
parts.append(choice.message.content)
return "\n".join(parts)
def _assert_eventually_masked[R: BaseModel](
fetch: Callable[[], Result[R]], extract: Callable[[R], str], *, email: str
) -> None:
"""Retry the call until the response comes back masked, to the propagation
deadline. An unmasked early response is in-flight guardrail propagation, not
a failure, and neither is a transient non-Success (a replica that has not
reloaded the guardrail answers 404, the live model can rate-limit) - only a
response that still carries the raw PII at the deadline is."""
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
last: str = "<no successful response yet>"
while True:
result = fetch()
match result:
case Success(data=data):
content = extract(data)
last = content
masked = MASKED_EMAIL_TOKEN in content and MASKED_PHONE_TOKEN in content and email not in content
if masked:
assert FAKE_PHONE not in content, (
f"the raw phone number must be masked before the model sees it, but the "
f"response echoed it: {content[:300]!r}"
)
return
case _:
last = f"<non-Success result: {result}>"
if time.monotonic() >= deadline:
pytest.fail(
f"presidio pre_call guardrail never masked the PII within "
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
)
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
class TestPresidioPreCallMasking:
@pytest.mark.covers(
"guardrail.presidio.pre_call.masks",
exercised_on=["chat_completions"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_pre_call_masks_pii_on_chat_completions(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = f"e2e-presidio-pre-chat-{unique_marker()}"
_register_presidio(client, resources, name=name)
email = _fake_email()
prompt = _pii_prompt(unique_marker(), email)
_assert_eventually_masked(
lambda: client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128),
_first_content,
email=email,
)
@pytest.mark.covers(
"guardrail.presidio.pre_call.masks",
exercised_on=["messages"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.MESSAGES,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_pre_call_masks_pii_on_messages(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = f"e2e-presidio-pre-msg-{unique_marker()}"
_register_presidio(client, resources, name=name)
email = _fake_email()
prompt = _pii_prompt(unique_marker(), email)
_assert_eventually_masked(
lambda: client.messages(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128),
_messages_text,
email=email,
)
#: Room for the model's reasoning tokens plus the three-line answer; a lower cap
#: truncates the response before the address it is supposed to mask.
_POST_CALL_MAX_TOKENS = 512
#: The post_call scenario masks these two entities and nothing else. Left
#: unscoped, Presidio's broader recognizers claim the local part too (a random
#: marker reads as an NRP), which would erase the very token that tells output
#: masking apart from input masking.
_POST_CALL_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"}
def _post_call_prompt(marker: str, local_part: str) -> str:
"""Ask for the local part and the full address in one answer. Presidio masks
an EMAIL_ADDRESS entity and a bare local part is not one, so the two land
differently in the same response and pin the hook point behaviorally."""
return (
f"{marker} My email address is {local_part}@example.com and my phone number is {FAKE_PHONE}. "
"Reply with exactly three lines and nothing else. "
"Line 1: the part of the email address before the @ sign. "
"Line 2: the full email address. "
"Line 3: the phone number."
)
class TestPresidioPostCallMasking:
@pytest.mark.covers(
"guardrail.presidio.post_call.masks",
exercised_on=["chat_completions"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_post_call_masks_pii_in_model_output(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
"""A guardrail scoped to the output must anonymize the PII the model
repeats back, so a caller (or a downstream log of the response) never
receives it, while the request itself reaches the model untouched.
Both facts are asserted from one response: the local part comes back raw,
which is only possible if the model saw the real address, and the address
itself comes back as <EMAIL_ADDRESS> in the same answer.
"""
name = f"e2e-presidio-post-chat-{unique_marker()}"
_register_presidio(
client,
resources,
name=name,
mode="post_call",
filter_scope="output",
entities=_POST_CALL_ENTITIES,
)
local_part = f"e2euser{unique_marker()}"
email = f"{local_part}@example.com"
prompt = _post_call_prompt(unique_marker(), local_part)
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
last = "<no successful response yet>"
while True:
result = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=_POST_CALL_MAX_TOKENS)
match result:
case Success(data=data):
last = _first_content(data)
if MASKED_EMAIL_TOKEN in last and email not in last:
assert local_part in last, (
"the model must have seen the RAW address (it is asked for the local "
"part, which Presidio does not mask); the local part is missing, so "
f"this response cannot tell post_call masking from pre_call: {last[:300]!r}"
)
assert MASKED_PHONE_TOKEN in last and FAKE_PHONE not in last, (
f"the phone number in the model's answer must be masked too, got: {last[:300]!r}"
)
return
case _:
last = f"<non-Success result: {result}>"
if time.monotonic() >= deadline:
pytest.fail(
f"presidio post_call guardrail never masked the model's output within "
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
)
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
def _assert_eventually_masks_generated_card(fetch: Callable[[], str | None]) -> None:
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
last: str = "<no successful response yet>"
while True:
content = fetch()
if content is not None:
last = content
if _contains_card_number(content):
pytest.fail(
"the post_call output masking let a card number through: "
f"{content[:300]!r}"
)
if MASKED_CREDIT_CARD_TOKEN in content:
return
if time.monotonic() >= deadline:
pytest.fail(
"presidio post_call output masking never masked the generated card within "
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
)
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
class TestPresidioCreditCardOutputMasking:
"""Proves the UI-default Presidio scope masks model-generated card output."""
@pytest.mark.covers(
"guardrail.presidio.post_call.masks_generated_output",
exercised_on=["chat_completions"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = f"e2e-presidio-card-chat-{unique_marker()}"
_register_presidio(
client,
resources,
name=name,
mode=["pre_call", "post_call"],
filter_scope=None,
entities={"CREDIT_CARD": "MASK"},
)
prompt: Final = _credit_card_prompt(unique_marker())
def fetch() -> str | None:
result: Final = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512)
match result:
case Success(data=data):
return _first_content(data)
case _:
return None
_assert_eventually_masks_generated_card(fetch)
@pytest.mark.covers(
"guardrail.presidio.post_call.masks_generated_output",
exercised_on=["chat_completions_stream"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.STREAM,
)
)
def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = f"e2e-presidio-card-stream-{unique_marker()}"
_register_presidio(
client,
resources,
name=name,
mode=["pre_call", "post_call"],
filter_scope=None,
entities={"CREDIT_CARD": "MASK"},
)
prompt: Final = _credit_card_prompt(unique_marker())
def fetch() -> str | None:
result: Final = client.chat_stream_raw(
scoped_key,
MODEL,
prompt,
guardrails=[name],
max_tokens=512,
)
if not result.ok or result.stream_error:
return None
return _stream_content(result)
_assert_eventually_masks_generated_card(fetch)
@pytest.mark.covers(
"guardrail.presidio.post_call.masks_generated_output",
exercised_on=["anthropic_messages_stream"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.MESSAGES,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.STREAM,
)
)
def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = f"e2e-presidio-card-messages-stream-{unique_marker()}"
_register_presidio(
client,
resources,
name=name,
mode=["pre_call", "post_call"],
filter_scope=None,
entities={"CREDIT_CARD": "MASK"},
)
prompt: Final = _credit_card_prompt(unique_marker())
def fetch() -> str | None:
result: Final = client.messages_stream_raw(
scoped_key,
MODEL,
prompt,
guardrails=[name],
max_tokens=512,
)
if not result.ok or result.stream_error:
return None
return _anthropic_stream_content(result)
_assert_eventually_masks_generated_card(fetch)
def _wire_text(outcome: StreamingResponse) -> str:
return "\n".join(outcome.stream_events) if outcome.is_streaming else outcome.body
def _poll_until_generated_card_masked(fetch: Callable[[], StreamingResponse]) -> StreamingResponse:
"""The raw HTTP outcome once the output masker is in effect on the serving
worker: whichever wire shape the endpoint speaks, a masked body carries the
CREDIT_CARD placeholder and no Luhn-valid card run. A raw card is a worker
that has not loaded the guardrail yet, so it is polled through like any
other unmasked answer."""
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
last: str = "<no successful response yet>"
while True:
outcome = fetch()
if outcome.ok and not outcome.stream_error:
wire = _wire_text(outcome)
last = wire
if MASKED_CREDIT_CARD_TOKEN in wire and not _contains_card_number(wire):
return outcome
if time.monotonic() >= deadline:
pytest.fail(
"presidio post_call output masking never masked the generated card within "
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
)
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
def _spend_log_response_text(client: GuardrailsClient, key: str, call_id: str) -> str:
rows = client.proxy.poll_logs_for_key(
key,
predicate=lambda logged: any(row.litellm_call_id == call_id for row in logged),
)
row = next((row for row in rows if row.litellm_call_id == call_id), None)
assert row is not None, f"no spend log row ever appeared for x-litellm-call-id {call_id}"
return "\n".join(_stored_content(row.response))
def _stored_content(node: JsonValue, key: str | None = None) -> Iterator[str]:
match node:
case str() if key in _CONTENT_KEYS:
yield node
case dict():
for child_key, child in node.items():
yield from _stored_content(child, child_key)
case list():
for item in node:
yield from _stored_content(item, key)
case _:
return
class TestPresidioSpendLogStoresMaskedOutput:
"""The spend log stores the response the caller received, on every endpoint
and both stream modes, when an output-only post_call Presidio guardrail masks
a card number the model generated."""
_CELL: Final = "guardrail.presidio.post_call.spend_log_stores_masked_output"
def _assert_spend_log_is_masked(
self,
client: GuardrailsClient,
resources: ResourceManager,
key: str,
*,
name: str,
fetch: Callable[[str, str], StreamingResponse],
) -> None:
_register_presidio(
client,
resources,
name=name,
mode="post_call",
filter_scope="output",
entities={"CREDIT_CARD": "MASK"},
)
prompt: Final = _credit_card_prompt(unique_marker())
outcome = _poll_until_generated_card_masked(lambda: fetch(prompt, name))
assert outcome.call_id, f"the served response must carry x-litellm-call-id: {dict(outcome.headers)}"
logged = _spend_log_response_text(client, key, outcome.call_id)
assert not _contains_card_number(logged), (
"the caller got the masked response but the spend log stored the raw model output: "
f"{logged[:400]!r}"
)
assert MASKED_CREDIT_CARD_TOKEN in logged, (
f"the spend log response carries neither the card nor the placeholder: {logged[:400]!r}"
)
@pytest.mark.covers(_CELL, exercised_on=["chat_completions"])
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_spend_log_stores_masked_output_on_chat_completions(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
self._assert_spend_log_is_masked(
client,
resources,
scoped_key,
name=f"e2e-presidio-log-card-chat-{unique_marker()}",
fetch=lambda prompt, guardrail: client.chat_raw(
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
),
)
@pytest.mark.covers(_CELL, exercised_on=["chat_completions_stream"])
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.STREAM,
)
)
def test_spend_log_stores_masked_output_on_streaming_chat_completions(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
self._assert_spend_log_is_masked(
client,
resources,
scoped_key,
name=f"e2e-presidio-log-card-chat-stream-{unique_marker()}",
fetch=lambda prompt, guardrail: client.chat_stream_raw(
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
),
)
@pytest.mark.covers(_CELL, exercised_on=["messages"])
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.MESSAGES,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_spend_log_stores_masked_output_on_anthropic_messages(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
self._assert_spend_log_is_masked(
client,
resources,
scoped_key,
name=f"e2e-presidio-log-card-messages-{unique_marker()}",
fetch=lambda prompt, guardrail: client.messages_raw(
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
),
)
@pytest.mark.covers(_CELL, exercised_on=["anthropic_messages_stream"])
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.MESSAGES,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.STREAM,
)
)
def test_spend_log_stores_masked_output_on_streaming_anthropic_messages(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
self._assert_spend_log_is_masked(
client,
resources,
scoped_key,
name=f"e2e-presidio-log-card-messages-stream-{unique_marker()}",
fetch=lambda prompt, guardrail: client.messages_stream_raw(
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
),
)
@pytest.mark.covers(_CELL, exercised_on=["responses"])
@meta(
Subject(
domain=Domain.GUARDRAILS,
route=Route.RESPONSES,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_spend_log_stores_masked_output_on_responses(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
self._assert_spend_log_is_masked(
client,
resources,
scoped_key,
name=f"e2e-presidio-log-card-responses-{unique_marker()}",
fetch=lambda prompt, guardrail: client.responses(scoped_key, MODEL, prompt, guardrails=[guardrail]),
)
_LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"}
_ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch])
def _applied_guardrails(outcome: StreamingResponse) -> str:
return outcome.headers.get("x-litellm-applied-guardrails", "")
def _guardrail_records(row: SpendLogRow) -> tuple[GuardrailRunRecord, ...]:
metadata = row.metadata
if metadata is None:
return ()
return tuple(metadata.guardrail_information or ())
def _poll_until_guardrail_applied(
client: GuardrailsClient, key: str, guardrail_name: str, prompt: str
) -> StreamingResponse:
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
last = client.chat_raw(key, MODEL, prompt, guardrails=[guardrail_name], max_tokens=128)
while time.monotonic() < deadline:
if last.ok and guardrail_name in _applied_guardrails(last):
return last
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
last = client.chat_raw(key, MODEL, prompt, guardrails=[guardrail_name], max_tokens=128)
return last
class TestPresidioSpendLogRecord:
@pytest.mark.covers(
"guardrail.presidio.pre_call.logs_masked_entities",
exercised_on=["chat_completions"],
)
@meta(
Subject(
domain=Domain.GUARDRAILS,
providers=(Provider.GEMINI,),
models=(MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_masking_run_is_recorded_on_the_spend_log(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = f"e2e-presidio-log-{unique_marker()}"
_register_presidio(client, resources, name=name, entities=_LOGGED_ENTITIES)
email = _fake_email()
prompt = _pii_prompt(unique_marker(), email)
outcome = _poll_until_guardrail_applied(client, scoped_key, name, prompt)
assert outcome.ok, f"the guarded call must be served, got {outcome.status_code}: {outcome.body[:400]}"
assert name in _applied_guardrails(outcome), (
"the response must carry x-litellm-applied-guardrails naming the guardrail; without it "
f"the 200 only proves the guardrail never attached. Got {_applied_guardrails(outcome)!r}"
)
request_id = ChatResponse.model_validate_json(outcome.body).id
assert request_id, f"the served response must carry an id to look the spend log up by: {outcome.body[:400]}"
rows = client.proxy.poll_logs_for_request_id(
request_id,
predicate=lambda logged: bool(_guardrail_records(logged[0])),
)
assert rows, f"no spend log row ever appeared for request {request_id}"
records = _guardrail_records(rows[0])
assert records, (
f"the spend log for {request_id} carries no guardrail_information, so the dashboard's "
"guardrail panel would render nothing for a request the guardrail demonstrably ran on"
)
record = next(
(entry for entry in records if entry.guardrail_name == name and entry.guardrail_mode == "pre_call"),
None,
)
assert record is not None, (
"guardrail_information carries no pre_call record for this guardrail, only "
f"{[(entry.guardrail_name, entry.guardrail_mode) for entry in records]}"
)
assert record.guardrail_status == "success", (
f"the recorded status must be success for a run that masked and served, got {record.guardrail_status!r}"
)
assert record.guardrail_provider == "presidio", (
f"the record must attribute the run to presidio so the dashboard picks the right "
f"renderer, got {record.guardrail_provider!r}"
)
counts = record.masked_entity_count or {}
assert _LOGGED_ENTITIES.keys() <= counts.keys(), (
f"every entity the guardrail was configured to mask must appear in masked_entity_count, got {counts}"
)
assert all(counts[entity] >= 1 for entity in _LOGGED_ENTITIES), (
f"each masked entity must be counted at least once, got {counts}"
)
assert not isinstance(record.guardrail_response, str), (
"This audit test requires general_settings.store_prompts_in_spend_logs: true in the proxy config "
"or STORE_PROMPTS_IN_SPEND_LOGS=true in the proxy process environment before startup "
"(not just the pytest environment). Default prompt redaction is valid proxy behavior, "
f"but prevents entity-detail assertions; got guardrail_response={record.guardrail_response!r}"
)
entities = _ENTITY_LIST_ADAPTER.validate_python(record.guardrail_response)
assert entities, (
"guardrail_response must carry the detected entities; the dashboard's Detected Entities "
"list and its per-entity scores are rendered from exactly this array"
)
assert {entity.entity_type for entity in entities} >= _LOGGED_ENTITIES.keys(), (
f"the detected entities must cover what was masked, got "
f"{sorted(entity.entity_type for entity in entities)}"
)
for entity in entities:
assert 0.0 < entity.score <= 1.0, f"{entity.entity_type} carries an out-of-range score: {entity.score}"
assert 0 <= entity.start < entity.end, (
f"{entity.entity_type} carries a degenerate span: {entity.start}-{entity.end}"
)

View file

@ -47,6 +47,7 @@ class RealtimeProvider:
id: str
alias: str
litellm_params: LiteLLMParamsBody
modalities: tuple[str, ...] = ("text",)
PROVIDERS = (
@ -75,6 +76,7 @@ PROVIDERS = (
model="gemini/gemini-3.1-flash-live-preview",
api_key="os.environ/GEMINI_API_KEY",
),
modalities=("audio",),
),
RealtimeProvider(
"vertex_ai",
@ -84,6 +86,7 @@ PROVIDERS = (
vertex_location="us-central1",
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
),
modalities=("audio",),
),
RealtimeProvider(
"bedrock",

View file

@ -124,7 +124,8 @@ def test_text_conversation(
session.send(
SessionUpdate(
session=SessionConfig(
instructions="You are a terse assistant. Reply in one short sentence."
modalities=list(provider.modalities),
instructions="You are a terse assistant. Reply in one short sentence.",
)
)
)
@ -159,6 +160,7 @@ def test_tool_call_round_trip(
session.send(
SessionUpdate(
session=SessionConfig(
modalities=list(provider.modalities),
tools=[WEATHER_TOOL],
tool_choice="auto",
instructions=(

View file

@ -8,10 +8,13 @@ AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody:
def azure_openai_params(
api_version: str = AZURE_OPENAI_API_VERSION, responses_api_version: str | None = None
) -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=AZURE_OPENAI_BACKEND,
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
api_version=api_version,
responses_api_version=responses_api_version,
)

View file

@ -0,0 +1,71 @@
from __future__ import annotations
from typing import Final
import pytest
from anthropic.types import Message, MessageParam
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
BEDROCK_MODEL: Final = "bedrock/us.moonshotai.kimi-k3"
MODEL_NAME: Final = "claude-kimi-k3"
MODEL_SWITCH_BETA: Final = (
"claude-code-20250219,context-1m-2025-08-07,interleaved-thinking-2025-05-14,"
"redact-thinking-2026-02-12,context-management-2025-06-27,"
"prompt-caching-scope-2026-01-05,mid-conversation-system-2026-04-07"
)
def _register_model(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
model_id: Final = proxy.create_model(
MODEL_NAME,
LiteLLMParamsBody(
model=BEDROCK_MODEL,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
drop_params=True,
),
)
resources.defer(lambda: proxy.delete_model(model_id))
return MODEL_NAME, resources.key(models=[MODEL_NAME])
class TestBedrockKimiK3Messages:
@pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.BEDROCK,),
models=(BEDROCK_MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_model_switch_probe_max_tokens_1_returns_message(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model, key = _register_model(proxy, resources)
client: Final = sdk.anthropic(key)
message_param: Final[MessageParam] = {
"role": "user",
"content": [{"type": "text", "text": "Hi", "cache_control": {"type": "ephemeral"}}],
}
message: Final = client.messages.create(
model=model,
max_tokens=1,
stream=False,
system="You are Claude Code, an assistant for software development.",
messages=[message_param],
extra_headers={"anthropic-beta": MODEL_SWITCH_BETA},
extra_query={"beta": "true"},
extra_body=NO_PROXY_CACHE,
)
assert isinstance(message, Message)
assert message.role == "assistant"

View file

@ -0,0 +1,119 @@
"""Live provider checks for embeddings, the chat refusal, and chat fallback from an explicitly restricted model."""
from __future__ import annotations
import dataclasses
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody
from openai import BadRequestError, OpenAI
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
EMBEDDING_BACKEND: Final = "openai/text-embedding-3-small"
CHAT_FALLBACK_BACKEND: Final = "openai/gpt-5.4-nano"
EMBEDDINGS_ONLY: Final = ("/v1/embeddings",)
PROMPT: Final = "Say this is a test!"
def _subject(route: Route) -> Subject:
return Subject(
domain=Domain.LLM_TRANSLATION,
route=route,
providers=(Provider.OPENAI,),
models=(EMBEDDING_BACKEND,),
mode=Mode.NONSTREAM,
)
def _register(
proxy: ProxyClient,
resources: ResourceManager,
prefix: str,
supported_endpoints: tuple[str, ...] | None,
backend: str = EMBEDDING_BACKEND,
mode: ModelMode = "embedding",
) -> str:
model: Final = f"{prefix}-{unique_marker()}"
model_id: Final = proxy.register_model(
ModelNewBody(
model_name=model,
litellm_params=LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY"),
model_info=ModelInfoBody(
enforce_supported_endpoints=supported_endpoints is not None,
mode=mode,
supported_endpoints=None if supported_endpoints is None else list(supported_endpoints),
),
)
)
resources.defer(lambda: proxy.delete_model(model_id))
return model
def _openai_at(sdk: SdkClients, api_prefix: str, key: str) -> OpenAI:
return dataclasses.replace(sdk, base_url=f"{sdk.base_url}{api_prefix}").openai(key)
class TestEmbeddingOnlySupportedEndpoints:
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(_subject(Route.EMBEDDINGS))
@pytest.mark.parametrize(
("prefix", "supported_endpoints"),
(
pytest.param("e2e-embed-only", EMBEDDINGS_ONLY, id="embed-only"),
pytest.param("e2e-embed-control", None, id="control"),
),
)
def test_embeddings_return_a_vector(
self,
proxy: ProxyClient,
resources: ResourceManager,
sdk: SdkClients,
prefix: str,
supported_endpoints: tuple[str, ...] | None,
) -> None:
model: Final = _register(proxy, resources, prefix, supported_endpoints)
embeddings: Final = sdk.openai(resources.key()).embeddings.create(
model=model, input=PROMPT, extra_body=NO_PROXY_CACHE
)
assert embeddings.data, f"/embeddings returned no data for {model}: {embeddings!r}"
assert any(component != 0.0 for component in embeddings.data[0].embedding), (
f"/embeddings returned an empty or all-zero vector for {model}"
)
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(dataclasses.replace(_subject(Route.CHAT_COMPLETIONS), models=(EMBEDDING_BACKEND, CHAT_FALLBACK_BACKEND)))
def test_chat_completions_with_a_fallback_are_answered_by_the_fallback(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
fallback: Final = _register(proxy, resources, "e2e-chat-fallback", None, CHAT_FALLBACK_BACKEND, "chat")
completion: Final = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
model=model,
messages=[{"role": "user", "content": PROMPT}],
extra_body={**NO_PROXY_CACHE, "fallbacks": [fallback]},
)
assert completion.choices and completion.choices[0].message.content, (
f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}"
)
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
@meta(_subject(Route.CHAT_COMPLETIONS))
def test_chat_completions_without_a_fallback_are_refused_by_the_gateway(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
with pytest.raises(BadRequestError) as refused:
_ = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE
)
assert (
f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings"
in refused.value.message
), f"chat on {model} should be refused by the gateway before any provider call, got {refused.value!r}"

View file

@ -0,0 +1,187 @@
from __future__ import annotations
import re
from collections.abc import Iterator
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients
from models import LiteLLMParamsBody
from openai.types.chat import (
ChatCompletion,
ChatCompletionAssistantMessageParam,
ChatCompletionChunk,
ChatCompletionMessageFunctionToolCallParam,
ChatCompletionToolMessageParam,
ChatCompletionToolParam,
ChatCompletionUserMessageParam,
)
from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall
from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
BACKEND_MODEL: Final = "gemini/gemini-3.8-flash"
GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY"
# bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20
AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
WEATHER_TOOL: Final[ChatCompletionToolParam] = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
def _subject(mode: Mode) -> Subject:
return Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(BACKEND_MODEL,),
mode=mode,
capabilities=(Capability.FUNCTION_CALLING, Capability.TOOL_CHOICE),
)
def _register_model(client: PassthroughClient, resources: ResourceManager) -> str:
model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}"
model_id: Final = client.proxy.create_model(
model_name,
LiteLLMParamsBody(
model=BACKEND_MODEL,
api_key=GEMINI_API_KEY,
thought_signature_in_tool_call_id=False,
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model_name
def _user_message() -> ChatCompletionUserMessageParam:
return {"role": "user", "content": f"Use get_weather for San Francisco and return its result. {unique_marker()}"}
def _tool_calls(completion: ChatCompletion) -> tuple[ChatCompletionMessageFunctionToolCall, ...]:
assert completion.choices, f"Gemini returned no choices: {completion.model_dump_json()[:400]}"
calls: Final = tuple(
call
for call in completion.choices[0].message.tool_calls or ()
if isinstance(call, ChatCompletionMessageFunctionToolCall)
)
assert calls, f"Gemini returned no tool calls for tool_choice='required': {completion.model_dump_json()[:400]}"
return calls
def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None:
rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id))
assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}"
def _streamed_tool_calls(chunks: tuple[ChatCompletionChunk, ...]) -> Iterator[ChoiceDeltaToolCall]:
for chunk in chunks:
for choice in chunk.choices:
yield from choice.delta.tool_calls or ()
def _short_id_tool_call(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionMessageFunctionToolCallParam:
return {
"id": call.id,
"type": "function",
"function": {"name": call.function.name, "arguments": call.function.arguments},
}
def _tool_result(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionToolMessageParam:
return {"role": "tool", "tool_call_id": call.id, "content": '{"temperature": 21, "condition": "sunny"}'}
class TestGeminiToolCallIdLength:
@pytest.mark.covers(
"llm.chat_completions.gemini.tool_use.nonstream.works",
exercised_on=["chat_completions"],
)
@meta(_subject(Mode.NONSTREAM))
def test_non_streaming_tool_call_id_fits_agentcore(
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register_model(client, resources)
completion: Final = sdk.openai(resources.key()).chat.completions.create(
model=model,
messages=[_user_message()],
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=2048,
extra_body=NO_PROXY_CACHE,
)
_assert_agentcore_accepts_ids(tuple(call.id for call in _tool_calls(completion)))
@pytest.mark.covers(
"llm.chat_completions.gemini.tool_use.stream.works",
exercised_on=["chat_completions"],
)
@meta(_subject(Mode.STREAM))
def test_streaming_tool_call_id_fits_agentcore(
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register_model(client, resources)
chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple(
sdk.openai(resources.key()).chat.completions.create(
model=model,
messages=[_user_message()],
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=2048,
stream=True,
extra_body=NO_PROXY_CACHE,
)
)
ids: Final = tuple(call.id for call in _streamed_tool_calls(chunks) if call.id)
assert ids, f"Gemini stream returned no tool call ids in {len(chunks)} chunks"
_assert_agentcore_accepts_ids(ids)
@pytest.mark.covers(
"llm.chat_completions.gemini.multi_turn.nonstream.works",
exercised_on=["chat_completions"],
)
@meta(_subject(Mode.NONSTREAM))
def test_tool_call_round_trip_succeeds(
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register_model(client, resources)
openai_client: Final = sdk.openai(resources.key())
question: Final = _user_message()
first: Final = openai_client.chat.completions.create(
model=model,
messages=[question],
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=2048,
extra_body=NO_PROXY_CACHE,
)
calls: Final = _tool_calls(first)
assistant: Final[ChatCompletionAssistantMessageParam] = {
"role": "assistant",
"tool_calls": [_short_id_tool_call(call) for call in calls],
}
second: Final = openai_client.chat.completions.create(
model=model,
messages=[question, assistant, *(_tool_result(call) for call in calls)],
tools=[WEATHER_TOOL],
tool_choice="none",
max_tokens=2048,
extra_body=NO_PROXY_CACHE,
)
assert second.choices, f"Gemini turn 2 returned no choices: {second.model_dump_json()[:400]}"
answer: Final = second.choices[0].message.content
assert answer and answer.strip(), f"Gemini turn 2 returned an empty answer: {second.model_dump_json()[:400]}"

View file

@ -15,48 +15,85 @@ pytestmark = pytest.mark.e2e
_BETA_CONFIG: Final = files("litellm").joinpath("anthropic_beta_headers_config.json")
_ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5"
_VERTEX_MODEL: Final = "vertex_ai/claude-sonnet-5-5"
_VERTEX_LOCATION: Final = "global"
_AZURE_AI_MODEL: Final = "azure_ai/claude-fable-5-1"
_BEDROCK_MODEL_ID: Final = "us.anthropic.claude-fable-5-1"
_BETA_CONFIG_FILE: Final = TypeAdapter(dict[str, JsonValue])
_BETA_MAPPING: Final = TypeAdapter(dict[str, str | None])
_CODE_EXECUTION: Final[ToolUnionParam] = {"type": "code_execution_20250825", "name": "code_execution"}
# A live Vertex probe on 2026-10-31: each Claude 4.5 and 4.6 model takes one of these and Claude 5.5 takes neither
_VERTEX_PER_MODEL_BETAS: Final = frozenset({"computer-use-2025-01-24", "computer-use-2025-11-24"})
_NO_PER_MODEL_BETAS: Final[frozenset[str]] = frozenset()
def _mapped_beta_headers(provider: str) -> str:
mapping: Final = _BETA_MAPPING.validate_python(_BETA_CONFIG_FILE.validate_json(_BETA_CONFIG.read_text())[provider])
return ",".join(name for name, value in mapping.items() if value is not None)
def _mapped_beta_headers(provider: str, per_model: frozenset[str]) -> str:
config: Final = _BETA_CONFIG_FILE.validate_json(_BETA_CONFIG.read_text())
mapping: Final = _BETA_MAPPING.validate_python(config[provider])
header_only: Final = _BETA_MAPPING.validate_python(config.get(f"{provider}_header_only", {}))
forwarded: Final = {name for name, value in {**mapping, **header_only}.items() if value is not None}
assert per_model <= forwarded, per_model - forwarded
return ",".join(sorted(forwarded - per_model))
def _deployment(backend: str) -> LiteLLMParamsBody:
if backend.startswith("anthropic/"):
return LiteLLMParamsBody(model=backend, api_key="os.environ/ANTHROPIC_API_KEY")
return LiteLLMParamsBody(
model=backend,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="us-east-1",
)
match backend.split("/", 1)[0]:
case "anthropic":
return LiteLLMParamsBody(model=backend, api_key="os.environ/ANTHROPIC_API_KEY")
case "vertex_ai":
return LiteLLMParamsBody(
model=backend,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location=_VERTEX_LOCATION,
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
)
case "azure_ai":
return LiteLLMParamsBody(
model=backend,
api_base="os.environ/AZURE_AI_API_BASE",
api_key="os.environ/AZURE_AI_API_KEY",
)
case _:
return LiteLLMParamsBody(
model=backend,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="us-east-1",
)
class TestAnthropicBetaHeaders:
@pytest.mark.parametrize(
("backend", "beta_provider", "tools"),
("backend", "beta_provider", "tools", "per_model"),
[
pytest.param(_ANTHROPIC_MODEL, "anthropic", [_CODE_EXECUTION], id="anthropic"),
pytest.param(f"bedrock/invoke/{_BEDROCK_MODEL_ID}", "bedrock", [], id="bedrock-invoke"),
pytest.param(f"bedrock/converse/{_BEDROCK_MODEL_ID}", "bedrock_converse", [], id="bedrock-converse"),
pytest.param(_ANTHROPIC_MODEL, "anthropic", [_CODE_EXECUTION], _NO_PER_MODEL_BETAS, id="anthropic"),
pytest.param(_VERTEX_MODEL, "vertex_ai", [], _VERTEX_PER_MODEL_BETAS, id="vertex-ai"),
pytest.param(_AZURE_AI_MODEL, "azure_ai", [], _NO_PER_MODEL_BETAS, id="azure-ai"),
pytest.param(
f"bedrock/invoke/{_BEDROCK_MODEL_ID}", "bedrock", [], _NO_PER_MODEL_BETAS, id="bedrock-invoke"
),
pytest.param(
f"bedrock/converse/{_BEDROCK_MODEL_ID}",
"bedrock_converse",
[],
_NO_PER_MODEL_BETAS,
id="bedrock-converse",
),
],
)
@pytest.mark.covers(
"llm.messages.anthropic.basic.nonstream.works",
"llm.messages.bedrock_invoke.basic.nonstream.works",
"llm.messages.bedrock_converse.basic.nonstream.works",
"llm.messages.vertex.basic.nonstream.works",
"llm.messages.azure_foundry.basic.nonstream.works",
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC, Provider.BEDROCK),
models=(_ANTHROPIC_MODEL, _BEDROCK_MODEL_ID),
providers=(Provider.ANTHROPIC, Provider.BEDROCK, Provider.VERTEX_AI, Provider.AZURE_AI),
models=(_ANTHROPIC_MODEL, _BEDROCK_MODEL_ID, _VERTEX_MODEL, _AZURE_AI_MODEL),
mode=Mode.NONSTREAM,
)
)
@ -68,6 +105,7 @@ class TestAnthropicBetaHeaders:
backend: str,
beta_provider: str,
tools: list[ToolUnionParam],
per_model: frozenset[str],
) -> None:
model: Final = f"e2e-messages-beta-{unique_marker()}"
model_id: Final = proxy.create_model(model, _deployment(backend))
@ -78,7 +116,7 @@ class TestAnthropicBetaHeaders:
max_tokens=64,
messages=[{"role": "user", "content": "Say 'hello' and nothing else"}],
tools=tools,
extra_headers={"anthropic-beta": _mapped_beta_headers(beta_provider)},
extra_headers={"anthropic-beta": _mapped_beta_headers(beta_provider, per_model)},
extra_body=NO_PROXY_CACHE,
)

View file

@ -1,4 +1,4 @@
"""Live e2e for model-specific request features: service_tier.
"""Live e2e for model-specific request features: service_tier and prompt_cache_retention.
Each case asserts the feature took effect, not just a 200.
@ -11,26 +11,38 @@ avoided here because it is capacity-constrained and returns a transient 429 when
flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so
that cell is OpenAI-only by design.
Prompt caching lives in test_cache_control.py.
prompt_cache_retention is judged on the body OpenAI receives, captured by a live
provider edge, not on the reply: OpenAI echoes "24h" on gpt-5.5 even when the
request never carried the param, so an echo would pass with the param dropped.
Prompt caching hits and misses live in test_cache_control.py.
"""
from __future__ import annotations
import queue
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Final
import pytest
from pydantic import BaseModel
from e2e_config import unique_marker
from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker
from e2e_http import unwrap
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import ChatBody, ChatMessage, LiteLLMParamsBody
from passthrough_client import PassthroughClient
from provider_edge import LiveEdge, start_provider_edge
from proxy_client import ProxyClient
pytestmark = pytest.mark.e2e
SERVICE_TIER = "priority"
OPENAI_BACKEND: Final = "openai/gpt-5.5"
OPENAI_RESPONSES_BRIDGE_BACKEND: Final = "openai/responses/gpt-5.5"
PROMPT_CACHE_RETENTION: Final = "24h"
class TestServiceTier:
@ -74,3 +86,153 @@ class TestServiceTier:
f"service_tier not honored: sent {SERVICE_TIER!r}, response reported "
f"{response.service_tier!r} ({response})"
)
class _OutboundRetention(BaseModel, frozen=True):
prompt_cache_retention: str | None = None
@dataclass(frozen=True, slots=True)
class _OpenAIBodyCapture:
"""The raw bodies sent to one OpenAI path, seen by a live edge in front of
api.openai.com. Pass ``observe`` as the edge's ``observe_request``."""
path: str
_bodies: queue.SimpleQueue[bytes] = field(default_factory=queue.SimpleQueue)
def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
if body is not None and url.split("?")[0].endswith(self.path):
self._bodies.put(body)
def retentions(self) -> tuple[str | None, ...]:
bodies: Final = tuple(self._bodies.get_nowait() for _ in range(self._bodies.qsize()))
return tuple(_OutboundRetention.model_validate_json(body).prompt_cache_retention for body in bodies)
def _openai_edge_api_base(resources: ResourceManager, capture: _OpenAIBodyCapture, *, reachable_from_proxy: bool) -> str:
"""The proxy may run in a container, so its edge binds where the proxy can
reach it; an in-process SDK call reaches the edge on loopback."""
edge: Final = (
start_provider_edge(
LiveEdge(observe_request=capture.observe),
bind_host=PROVIDER_EDGE_BIND_HOST,
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
)
if reachable_from_proxy
else start_provider_edge(LiveEdge(observe_request=capture.observe))
)
resources.defer(edge.shutdown)
return f"{edge.edge.api_base('openai')}/v1"
def _retention_backends() -> tuple[object, ...]:
"""gpt-5.5 straight to /v1/chat/completions, and the openai/responses/ bridge
(chat completions in, Responses out) to /v1/responses."""
return (
pytest.param(
OPENAI_BACKEND,
"/chat/completions",
marks=(
pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention"),
meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.OPENAI,),
models=(OPENAI_BACKEND,),
mode=Mode.NONSTREAM,
)
),
),
id="chat_completions",
),
pytest.param(
OPENAI_RESPONSES_BRIDGE_BACKEND,
"/responses",
marks=(
pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention"),
meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.OPENAI,),
models=(OPENAI_RESPONSES_BRIDGE_BACKEND,),
mode=Mode.NONSTREAM,
)
),
),
id="responses_bridge",
),
)
class TestPromptCacheRetention:
@pytest.mark.provider_edge_host
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
def test_proxy_chat_completions_forwards_prompt_cache_retention_to_openai(
self, proxy: ProxyClient, resources: ResourceManager, backend: str, openai_path: str
) -> None:
capture: Final = _OpenAIBodyCapture(openai_path)
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=True)
model: Final = f"e2e-cache-retention-{unique_marker()}"
model_id: Final = proxy.create_model(
model, LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY", api_base=api_base)
)
resources.defer(lambda: proxy.delete_model(model_id))
unwrap(
proxy.chat(
resources.key(),
ChatBody(
model=model,
messages=[ChatMessage(role="user", content="reply with one word")],
prompt_cache_retention=PROMPT_CACHE_RETENTION,
),
)
)
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
)
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
def test_sdk_completion_forwards_prompt_cache_retention_to_openai(
self, resources: ResourceManager, backend: str, openai_path: str
) -> None:
import litellm
capture: Final = _OpenAIBodyCapture(openai_path)
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
litellm.completion(
model=backend,
messages=[{"role": "user", "content": "reply with one word"}],
api_base=api_base,
prompt_cache_retention=PROMPT_CACHE_RETENTION,
)
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
)
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
def test_router_completion_forwards_prompt_cache_retention_to_openai(
self, resources: ResourceManager, backend: str, openai_path: str
) -> None:
import litellm
capture: Final = _OpenAIBodyCapture(openai_path)
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
router: Final = litellm.Router(
model_list=[{"model_name": "cache-retention", "litellm_params": {"model": backend, "api_base": api_base}}]
)
router.completion( # pyright: ignore[reportUnknownMemberType] # Router.completion forwards an untyped **kwargs to litellm.completion
model="cache-retention",
messages=[{"role": "user", "content": "reply with one word"}],
prompt_cache_retention=PROMPT_CACHE_RETENTION,
)
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
)

View file

@ -0,0 +1,205 @@
"""Live e2e: Codex's client tool_search over /v1/responses on Claude.
Codex holds MCP tools back behind its client-executed `tool_search` tool when the model catalog says
`supports_search_tool`. Over the completion bridge the proxy offers that search to Claude as a plain function,
hands Claude's call back as a `tool_search_call` Codex runs itself, and loads the tools Codex returns in
`tool_search_output` so Claude can call them on the next turn
"""
from __future__ import annotations
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from openai.types.responses import (
FunctionToolParam,
Response,
ResponseFunctionToolCall,
ResponseInputParam,
ResponseToolSearchCall,
ToolParam,
ToolSearchToolParam,
)
from proxy_client import ProxyClient
from pydantic import BaseModel, ConfigDict
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-5-5"
DEFERRED_TOOL_COUNT: Final = 50
CALENDAR_TOOL_NAME: Final = "mcp__codex_apps__calendar_create_event"
CODEX_TOOL_SEARCH: Final[ToolSearchToolParam] = {
"type": "tool_search",
"execution": "client",
"description": (
"# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next "
"model call. Some of the tools may not have been provided to you upfront, and you should use this tool "
"(`tool_search`) to search for the required tools."
),
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string", "description": "Search query for deferred tools."},
"limit": {"type": "number", "description": "Maximum number of tools to return. Defaults to 8."},
},
"required": ["query"],
"additionalProperties": False,
},
}
SHELL_TOOL: Final[FunctionToolParam] = {
"type": "function",
"name": "shell",
"description": "Runs a shell command.",
"parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]},
"strict": False,
}
CALENDAR_TOOL: Final[FunctionToolParam] = {
"type": "function",
"name": CALENDAR_TOOL_NAME,
"description": "Create a calendar event.",
"parameters": {
"type": "object",
"properties": {"title": {"type": "string"}},
"required": ["title"],
"additionalProperties": False,
},
"strict": False,
"defer_loading": True,
}
PROMPT: Final = "Create a calendar event titled Standup. Find the calendar tool with tool_search first, then call it."
class _CalendarArguments(BaseModel):
model_config = ConfigDict(frozen=True)
title: str
def _filler_tool(index: int) -> FunctionToolParam:
return {
"type": "function",
"name": f"mcp__codex_apps__records_{index:02d}",
"description": f"List, inspect, update and archive records of family {index} in the connected workspace.",
"parameters": {
"type": "object",
"properties": {
"title": {"type": "string", "description": "Title of the record to act on"},
"notes": {"type": "string", "description": "Free-form notes and context for the operation"},
"tags": {"type": "array", "items": {"type": "string"}},
"priority": {"type": "integer", "minimum": 0, "maximum": 5},
},
"required": ["title"],
"additionalProperties": False,
},
"strict": False,
"defer_loading": True,
}
DEFERRED_TOOLS: Final[tuple[FunctionToolParam, ...]] = (
*(_filler_tool(index) for index in range(DEFERRED_TOOL_COUNT - 1)),
CALENDAR_TOOL,
)
def _eager(tool: FunctionToolParam) -> ToolParam:
return FunctionToolParam(
type="function",
name=tool["name"],
description=tool.get("description"),
parameters=tool.get("parameters"),
strict=False,
)
def _input_tokens(response: Response) -> int:
assert response.usage is not None, f"response carried no usage: {response.id}"
return response.usage.input_tokens
def _search_call(response: Response) -> ResponseToolSearchCall:
calls: Final = tuple(item for item in response.output if isinstance(item, ResponseToolSearchCall))
assert len(calls) == 1, f"expected one tool_search_call, got {response.output!r}"
return calls[0]
class TestResponsesCodexToolSearchOverAnthropic:
@pytest.mark.covers("llm.responses.anthropic.tool_search.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.ANTHROPIC,),
models=(ANTHROPIC_BACKEND,),
capabilities=(Capability.TOOL_SEARCH, Capability.FUNCTION_CALLING),
mode=Mode.NONSTREAM,
)
)
def test_claude_finds_a_deferred_tool_through_codex_tool_search_and_calls_it(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = f"e2e-codex-tool-search-{unique_marker()}"
model_id: Final = proxy.create_model(
model, LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY")
)
resources.defer(lambda: proxy.delete_model(model_id))
client: Final = sdk.openai(resources.key())
searchable_tools: Final[list[ToolParam]] = [SHELL_TOOL, CODEX_TOOL_SEARCH]
eager: Final = client.responses.create(
model=model,
input=PROMPT,
tools=[SHELL_TOOL, *(_eager(tool) for tool in DEFERRED_TOOLS)],
extra_body=NO_PROXY_CACHE,
)
first: Final = client.responses.create(
model=model, input=PROMPT, tools=searchable_tools, extra_body=NO_PROXY_CACHE
)
search: Final = _search_call(first)
assert search.execution == "client", f"tool_search ran on the wrong side: {search!r}"
assert search.call_id, f"tool_search_call had no call_id: {search!r}"
assert not any(isinstance(item, ResponseFunctionToolCall) for item in first.output), first.output
assert _input_tokens(first) * 4 < _input_tokens(eager), (
f"deferred first turn used {_input_tokens(first)} input tokens against {_input_tokens(eager)} eager"
)
follow_up: Final[ResponseInputParam] = [
{"role": "user", "content": PROMPT},
{
"type": "tool_search_call",
"call_id": search.call_id,
"execution": "client",
"arguments": search.arguments,
},
{
"type": "tool_search_output",
"call_id": search.call_id,
"status": "completed",
"execution": "client",
"tools": [CALENDAR_TOOL],
},
]
second: Final = client.responses.create(
model=model, input=follow_up, tools=searchable_tools, extra_body=NO_PROXY_CACHE
)
calendar_call: Final = next(
(
item
for item in second.output
if isinstance(item, ResponseFunctionToolCall) and item.name == CALENDAR_TOOL_NAME
),
None,
)
assert calendar_call is not None, f"Claude did not call the loaded tool: {second.output!r}"
assert "standup" in _CalendarArguments.model_validate_json(calendar_call.arguments).title.lower()

View file

@ -40,7 +40,7 @@ from provider_edge import LiveEdge, start_provider_edge
from provider_edge_bedrock import ConverseBodyCapture, bedrock_signer
from proxy_client import ProxyClient
from pydantic import BaseModel, TypeAdapter
from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params
from responses_helpers import AZURE_OPENAI_API_VERSION, AZURE_OPENAI_BACKEND, azure_openai_params
from sdk_clients import NO_PROXY_CACHE, SdkClients
pytestmark = pytest.mark.e2e
@ -183,6 +183,48 @@ class TestResponses:
)
assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}"
@pytest.mark.covers("llm.responses.openai.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.OPENAI,),
models=(OPENAI_RESPONSES_BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_responses_forwards_moderation(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(proxy, resources, _openai_responses_params())
client: Final = sdk.openai(resources.key())
response: Final = client.responses.create(
model=model,
input="hi",
extra_body={**NO_PROXY_CACHE, "moderation": {"model": "omni-moderation-latest"}},
)
response_body: Final = TypeAdapter(dict[str, object]).validate_python(cast(object, response.model_dump()))
moderation_value: Final = response_body.get("moderation")
assert moderation_value is not None, f"response had no moderation results: {response_body!r}"
moderation: Final = TypeAdapter(dict[str, object]).validate_python(moderation_value)
moderation_input: Final = TypeAdapter(dict[str, object]).validate_python(moderation["input"])
moderation_output: Final = TypeAdapter(dict[str, object]).validate_python(moderation["output"])
assert moderation_input.get("type") == "moderation_result", (
f"unexpected input moderation: {moderation_input!r}"
)
assert (
moderation_output.get("type") == "moderation_result"
), f"unexpected output moderation: {moderation_output!r}"
with pytest.raises(openai.BadRequestError) as exc_info:
client.responses.create(
model=model,
input="hi",
extra_body={**NO_PROXY_CACHE, "moderation": {"model": "does-not-exist"}},
)
assert exc_info.value.status_code == 400
@pytest.mark.covers("llm.responses.openai.basic.stream.works")
@meta(
Subject(
@ -748,6 +790,70 @@ class TestResponses:
f"/responses over azure openai preview returned no output text: {response.output!r}"
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=(AZURE_OPENAI_BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_responses_azure_openai_dated_ga_chat_api_version_serves_responses(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register(
proxy,
resources,
azure_openai_params(api_version="2024-10-21"),
prefix="e2e-responses-azure-dated-chat",
)
client: Final = sdk.openai(resources.key())
response: Final = client.responses.create(
model=model,
input="reply with one word",
instructions=INSTRUCTIONS,
extra_body=NO_PROXY_CACHE,
)
assert response.output_text.strip(), (
f"/responses over an azure deployment pinned to a dated GA chat api_version returned no output text: "
f"{response.output!r}"
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.RESPONSES,
providers=(Provider.AZURE,),
models=(AZURE_OPENAI_BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_responses_azure_openai_responses_api_version_overrides_a_pre_responses_api_version(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
# Azure rejects /openai/responses below api-version 2025-03-01-preview (probed live 2026-10-15),
# so this call only succeeds if the proxy sends responses_api_version instead of api_version.
model: Final = _register(
proxy,
resources,
azure_openai_params(api_version="2024-10-01-preview", responses_api_version=AZURE_OPENAI_API_VERSION),
prefix="e2e-responses-azure-override",
)
client: Final = sdk.openai(resources.key())
response: Final = client.responses.create(
model=model,
input="reply with one word",
instructions=INSTRUCTIONS,
extra_body=NO_PROXY_CACHE,
)
assert response.output_text.strip(), (
f"/responses over an azure deployment with responses_api_version set returned no output text: "
f"{response.output!r}"
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,

View file

@ -1,256 +0,0 @@
"""Live e2e: Sail through the gateway, where LiteLLM turns ``service_tier`` into Sail's
``metadata.completion_window`` and bills the price columns of the window it sent.
The deployment carries its own base, balanced and flex rates, each distinct, so a bill at
the wrong tier cannot pass. They are registered on the deployment instead of read from the
proxy's cost map, because a stack that loads the published map has no ``sail/`` rows until
this provider ships. Requires SAIL_API_KEY on the proxy; no skip gate.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Literal
import pytest
from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS, unique_marker
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, SpendLogRow
from openai import OpenAI
from proxy_client import ProxyClient
from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header
pytestmark = pytest.mark.e2e
BACKEND: Final = "sail/zai-org/GLM-5.3"
PricedTier = Literal["base", "balanced", "flex"]
PRICED_TIERS: Final[tuple[PricedTier, ...]] = ("base", "balanced", "flex")
PROMPT: Final = "Reply with one word."
MAX_TOKENS: Final = 512
@dataclass(frozen=True, slots=True)
class _Rates:
input: float
output: float
cache_read: float
RATES: Final[Mapping[PricedTier, _Rates]] = {
"base": _Rates(input=3e-06, output=9e-06, cache_read=1e-06),
"balanced": _Rates(input=2e-06, output=6e-06, cache_read=7e-07),
"flex": _Rates(input=1e-06, output=3e-06, cache_read=4e-07),
}
@dataclass(frozen=True, slots=True)
class _Tokens:
prompt: int
cached: int
completion: int
def _approx_equal(actual: float, expected: float) -> bool:
return abs(actual - expected) <= max(1e-12, abs(expected) * 1e-2)
def _cost(rates: _Rates, tokens: _Tokens) -> float:
return (
(tokens.prompt - tokens.cached) * rates.input
+ tokens.cached * rates.cache_read
+ tokens.completion * rates.output
)
def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
model: Final = f"e2e-sail-{unique_marker()}"
model_id: Final = proxy.create_model(
model,
LiteLLMParamsBody(
model=BACKEND,
api_key="os.environ/SAIL_API_KEY",
input_cost_per_token=RATES["base"].input,
output_cost_per_token=RATES["base"].output,
cache_read_input_token_cost=RATES["base"].cache_read,
input_cost_per_token_balanced=RATES["balanced"].input,
output_cost_per_token_balanced=RATES["balanced"].output,
cache_read_input_token_cost_balanced=RATES["balanced"].cache_read,
input_cost_per_token_flex=RATES["flex"].input,
output_cost_per_token_flex=RATES["flex"].output,
cache_read_input_token_cost_flex=RATES["flex"].cache_read,
),
)
resources.defer(lambda: proxy.delete_model(model_id))
return model, resources.key()
def _openai(sdk: SdkClients, key: str) -> OpenAI:
return sdk.openai(key).with_options(timeout=SLOW_PROVIDER_TIMEOUT_SECONDS)
def _assert_billed_at(tier: PricedTier, tokens: _Tokens, header_cost: str | None) -> float:
assert tokens.prompt > 0 and tokens.completion > 0, f"Sail reported no usage, so no cost is real: {tokens}"
assert header_cost is not None, "x-litellm-response-cost header missing"
costs: Final = {priced: _cost(rates, tokens) for priced, rates in RATES.items()}
assert not any(_approx_equal(costs[other], costs[tier]) for other in PRICED_TIERS if other != tier), (
f"{BACKEND} tier rates too close together to tell {tier} apart at {tokens}: {costs}"
)
assert _approx_equal(float(header_cost), costs[tier]), (
f"header cost {header_cost} is not the {tier} price at {tokens}: expected {costs[tier]}, all tiers {costs}"
)
return float(header_cost)
def _assert_spend_row_matches(proxy: ProxyClient, key: str, header_cost: float) -> None:
def priced(rows: list[SpendLogRow]) -> bool:
return any((row.spend or 0) > 0 for row in rows)
rows: Final = [row for row in proxy.poll_logs_for_key(key, predicate=priced) if (row.spend or 0) > 0]
assert rows, f"no priced spend row landed for key {key}"
assert rows[0].custom_llm_provider == "sail", f"spend row misattributed: {rows[0]}"
assert rows[0].spend is not None and _approx_equal(rows[0].spend, header_cost), (
f"logged spend {rows[0].spend} disagrees with the x-litellm-response-cost header {header_cost}"
)
class TestSailChatCompletions:
@pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.cost_logged")
@meta(
Subject(
domain=Domain.SPEND_BUDGETS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.SAIL,),
models=(BACKEND,),
mode=Mode.NONSTREAM,
)
)
@pytest.mark.parametrize(
("service_tier", "billed_tier"), [("balanced", "balanced"), ("auto", "base")]
)
def test_service_tier_bills_the_matching_completion_window(
self,
proxy: ProxyClient,
resources: ResourceManager,
sdk: SdkClients,
service_tier: str,
billed_tier: PricedTier,
) -> None:
model, key = _register(proxy, resources)
raw: Final = _openai(sdk, key).chat.completions.with_raw_response.create(
model=model,
messages=[{"role": "user", "content": f"{PROMPT} {unique_marker()}"}],
max_completion_tokens=MAX_TOKENS,
extra_body={**NO_PROXY_CACHE, "service_tier": service_tier},
)
usage: Final = raw.parse().usage
assert usage is not None, "chat response carries no usage"
details: Final = usage.prompt_tokens_details
tokens: Final = _Tokens(
prompt=usage.prompt_tokens,
cached=(details.cached_tokens or 0) if details else 0,
completion=usage.completion_tokens,
)
header_cost: Final = _assert_billed_at(
billed_tier, tokens, response_header(raw.headers, "x-litellm-response-cost")
)
_assert_spend_row_matches(proxy, key, header_cost)
@pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.drops_unknown_tier_and_bills_asap")
@meta(
Subject(
domain=Domain.SPEND_BUDGETS,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.SAIL,),
models=(BACKEND,),
mode=Mode.NONSTREAM,
)
)
@pytest.mark.parametrize("service_tier", ["bogus", 5])
def test_unknown_service_tier_is_dropped_and_billed_asap(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, service_tier: str | int
) -> None:
model, key = _register(proxy, resources)
raw: Final = _openai(sdk, key).chat.completions.with_raw_response.create(
model=model,
messages=[{"role": "user", "content": f"{PROMPT} {unique_marker()}"}],
max_completion_tokens=MAX_TOKENS,
extra_body={**NO_PROXY_CACHE, "service_tier": service_tier, "drop_params": True},
)
usage: Final = raw.parse().usage
assert usage is not None, "chat response carries no usage"
details: Final = usage.prompt_tokens_details
tokens: Final = _Tokens(
prompt=usage.prompt_tokens,
cached=(details.cached_tokens or 0) if details else 0,
completion=usage.completion_tokens,
)
header_cost: Final = _assert_billed_at("base", tokens, response_header(raw.headers, "x-litellm-response-cost"))
_assert_spend_row_matches(proxy, key, header_cost)
class TestSailResponses:
@pytest.mark.covers("llm.responses.sail.service_tier.nonstream.cost_logged")
@meta(
Subject(
domain=Domain.SPEND_BUDGETS,
route=Route.RESPONSES,
providers=(Provider.SAIL,),
models=(BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_caller_completion_window_bills_its_rates(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model, key = _register(proxy, resources)
raw: Final = _openai(sdk, key).responses.with_raw_response.create(
model=model,
input=f"{PROMPT} {unique_marker()}",
max_output_tokens=MAX_TOKENS,
metadata={"completion_window": "balanced"},
extra_body=NO_PROXY_CACHE,
)
usage: Final = raw.parse().usage
assert usage is not None, "responses answer carries no usage"
tokens: Final = _Tokens(
prompt=usage.input_tokens,
cached=usage.input_tokens_details.cached_tokens,
completion=usage.output_tokens,
)
header_cost: Final = _assert_billed_at(
"balanced", tokens, response_header(raw.headers, "x-litellm-response-cost")
)
_assert_spend_row_matches(proxy, key, header_cost)
class TestSailMessages:
@pytest.mark.covers("llm.messages.sail.basic.nonstream.works")
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.SAIL,),
models=(BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_plain_call_returns_a_message(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model, key = _register(proxy, resources)
message: Final = sdk.anthropic(key).messages.create(
model=model,
max_tokens=MAX_TOKENS,
messages=[{"role": "user", "content": PROMPT}],
extra_body=NO_PROXY_CACHE,
)
assert message.role == "assistant" and message.content, f"/v1/messages returned no content: {message}"
assert message.usage.output_tokens > 0, f"/v1/messages reported no output usage: {message.usage}"

View file

@ -213,45 +213,6 @@ class LangfuseCreds:
)
@dataclass(frozen=True, slots=True)
class WeaveCreds:
"""Weights & Biases Weave credentials for a key-scoped ``weave_otel`` callback.
The proxy still needs WANDB_API_KEY / WANDB_PROJECT_ID in its own environment:
the weave_otel logger is constructed from those before the per-key vars are
applied, so a key-scoped callback on a proxy without them never initializes.
The per-key vars are what direct THIS key's spans at this project.
"""
api_key: str
project_id: str
def key_logging_metadata(self) -> KeyMetadata:
return KeyMetadata(
logging=[
KeyLoggingCallback(
callback_name="weave_otel",
callback_type="success_and_failure",
callback_vars=KeyLoggingCallbackVars(
wandb_api_key=self.api_key,
weave_project_id=self.project_id,
),
)
]
)
def load_weave_creds() -> WeaveCreds:
api_key = os.getenv("WANDB_API_KEY")
project_id = (os.getenv("WEAVE_PROJECT_ID") or os.getenv("WANDB_PROJECT_ID") or "").strip()
if not (api_key and project_id):
pytest.fail(
"Weave e2e requires WANDB_API_KEY and WEAVE_PROJECT_ID (or WANDB_PROJECT_ID, "
"format <entity>/<project>); missing credentials is a hard failure, not a skip"
)
return WeaveCreds(api_key=api_key, project_id=project_id)
def load_langfuse_creds() -> LangfuseCreds:
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
secret_key = os.getenv("LANGFUSE_SECRET_KEY")

View file

@ -54,7 +54,7 @@ def _assert_otel_destination_configured(client: LoggingClient) -> None:
details = _ReadinessDetails.model_validate_json(readiness_details_body(client))
assert OTEL_V2_LOGGER_NAME in details.success_callbacks, (
f"the proxy must report the {OTEL_V2_LOGGER_NAME} callback active "
f"(LITELLM_OTEL_V2 + arize_phoenix preset in the compose config); got: {details.success_callbacks}"
f"(LITELLM_OTEL_V2=true + the otel callback in the stack config); got: {details.success_callbacks}"
)

View file

@ -1,212 +0,0 @@
"""Live e2e: key-scoped Weave (Weights & Biases) delivery, success and failure.
Covers the two `logging.niche_integrations.*.logs_spend` cells with a real member
of that cohort. A key carrying a `weave_otel` callback in its logging metadata
must deliver its calls to the real Weave project, and each call must arrive
exactly once, carrying the same cost the response header reported:
- success: one `litellm_request` call, OTEL status OK, `llm.response.cost` equal
to `x-litellm-response-cost`, and non-zero tokens
- failure: a provider-rejected call arrives too, as one call with OTEL status
ERROR naming the provider exception, and with no cost - a failed call that
silently never reaches the destination is an invisible outage, and a billed
one is worse
Both halves assert the recorded state (the key's callback registration answers
success and the destination holds the call) and the enforced behavior (the
delivered payload's status and cost). Delivery is read back through Weave's own
query API; nothing is mocked.
"""
from __future__ import annotations
import time
from typing import Final
import pytest
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_http import StreamingResponse
from e2e_metadata import Domain, Mode, Provider, Subject, meta
from lifecycle import ResourceManager
from logging_client import (
INVALID_UPSTREAM_API_KEY,
LoggingClient,
WeaveCreds,
costs_agree,
first_ok,
load_weave_creds,
)
from models import LiteLLMParamsBody
from weave_reader import WeaveCall, WeaveReader, build_weave_reader
pytestmark = pytest.mark.e2e
UNREACHABLE_ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
@pytest.fixture(scope="session")
def weave_creds() -> WeaveCreds:
return load_weave_creds()
@pytest.fixture(scope="session")
def weave_reader() -> WeaveReader:
return build_weave_reader()
#: How far before the request the Weave read-back window opens, to absorb clock
#: skew between this host and Weave. Without it a host running slightly fast
#: would filter out its own call.
_WINDOW_SKEW_SECONDS = 120.0
def _window_start() -> float:
return time.time() - _WINDOW_SKEW_SECONDS
def _exactly_one(calls: tuple[WeaveCall, ...], *, marker: str, what: str) -> WeaveCall:
assert calls, f"no Weave call for the {what} (marker {marker}) reached the project within the deadline"
assert len(calls) == 1, (
f"expected exactly ONE Weave call for the {what} (marker {marker}), got {len(calls)}: "
f"{[call.id for call in calls]} - more than one call for one request is the "
"duplicate-delivery bug"
)
return calls[0]
WEAVE_STAGE_RED_REASON = (
"stage red: product gap, key-scoped weave_otel spans are not delivered when the OTEL v2 callback is active"
)
class TestWeaveLogDelivery:
@pytest.mark.skip(reason=WEAVE_STAGE_RED_REASON)
@pytest.mark.covers("logging.niche_integrations.success.logs_spend", exercised_on=["chat_completions"])
@meta(
Subject(
domain=Domain.OBSERVABILITY,
providers=(Provider.ANTHROPIC,),
models=(CHEAP_ANTHROPIC_MODEL,),
mode=Mode.NONSTREAM,
)
)
def test_chat_completions_delivers_one_call_with_spend(
self,
client: LoggingClient,
weave_creds: WeaveCreds,
weave_reader: WeaveReader,
resources: ResourceManager,
) -> None:
alias = f"weave-key-{unique_marker()}"
key = client.key_with_alias(
alias,
models=[CHEAP_ANTHROPIC_MODEL],
metadata=weave_creds.key_logging_metadata(),
)
resources.defer(lambda: client.delete_key(key))
marker = unique_marker()
since = _window_start()
outcome = first_ok(
client,
lambda: client.chat_raw(key, CHEAP_ANTHROPIC_MODEL, f"reply with one word {marker}", max_tokens=64),
)
assert outcome.response_cost is not None and outcome.response_cost > 0, (
f"the response must report x-litellm-response-cost, got {outcome.response_cost!r}"
)
call = _exactly_one(
weave_reader.poll_calls_matching(marker, since=since), marker=marker, what="successful call"
)
assert call.status_code == "OK", f"a successful call must land at OK span status, got {call.status_code!r}"
cost = call.response_cost
assert cost is not None and costs_agree(outcome.response_cost, cost), (
f"the Weave call's llm.response.cost {cost!r} must agree with the header cost "
f"{outcome.response_cost} - a delivered span with the wrong cost is a silent "
"billing-attribution bug"
)
assert call.total_tokens is not None and call.total_tokens > 0, (
f"the delivered call must carry token usage, got {call.total_tokens!r}"
)
@pytest.mark.skip(reason=WEAVE_STAGE_RED_REASON)
@pytest.mark.covers("logging.niche_integrations.failure.logs_spend", exercised_on=["chat_completions"])
@meta(
Subject(
domain=Domain.OBSERVABILITY,
providers=(Provider.ANTHROPIC,),
models=(UNREACHABLE_ANTHROPIC_BACKEND,),
mode=Mode.NONSTREAM,
)
)
def test_failed_chat_completions_delivers_one_error_call(
self,
client: LoggingClient,
weave_creds: WeaveCreds,
weave_reader: WeaveReader,
resources: ResourceManager,
) -> None:
"""A deployment with an invalid upstream key passes proxy auth and fails
at the provider, so exactly one provider failure exists for it. Proxy-side
401s during key propagation never reach the provider and ship no payload,
which is what the retry loop below relies on."""
model_name = f"weave-err-{unique_marker()}"
model_id = client.create_model(
model_name,
LiteLLMParamsBody(model=UNREACHABLE_ANTHROPIC_BACKEND, api_key=INVALID_UPSTREAM_API_KEY),
)
resources.defer(lambda: client.delete_model(model_id))
key = client.key_with_alias(
f"weave-err-key-{unique_marker()}",
models=[model_name],
metadata=weave_creds.key_logging_metadata(),
)
resources.defer(lambda: client.delete_key(key))
marker = unique_marker()
since = _window_start()
outcome = _provoke_provider_failure(client, key, model_name, marker)
call = _exactly_one(weave_reader.poll_calls_matching(marker, since=since), marker=marker, what="failed call")
assert call.status_code == "ERROR", (
f"a failed call must land at ERROR span status, got {call.status_code!r} - "
"Weave's own summary.weave.status reads success either way, which is exactly "
"why the span status is what this asserts on"
)
error = call.error
assert error is not None and error.message is not None and "AnthropicException" in error.message, (
f"the delivered call must carry the provider error, got {error!r}"
)
assert not call.response_cost, f"a failed call must not be billed, got llm.response.cost={call.response_cost!r}"
assert outcome.status_code == 401, (
f"an upstream auth failure must map to 401, got {outcome.status_code}: {outcome.body[:200]}"
)
def _provoke_provider_failure(client: LoggingClient, key: str, model_name: str, marker: str) -> StreamingResponse:
"""Send until the provider (not the proxy) is the one rejecting the call.
A network failure between the test and the proxy is NOT retried: the request
may have been served, and a retry would double-log the failure payload and
falsely trip the exactly-one assertion.
"""
deadline = time.monotonic() + client.proxy.poll_timeout
while True:
outcome = client.chat_raw(key, model_name, f"trigger an upstream auth failure {marker}", max_tokens=16)
assert not outcome.ok, "the call must fail; the deployment's upstream key is invalid"
assert outcome.status_code != -1, (
"network failure between the test and the proxy while provoking the provider failure; "
"retrying now could double-log the failure payload and falsely trip the exactly-one "
f"assertion - fix the rig connectivity first: {outcome.body[:200]}"
)
if "AnthropicException" in outcome.body or time.monotonic() >= deadline:
break
time.sleep(client.proxy.poll_interval)
assert "AnthropicException" in outcome.body, (
"never saw the upstream provider failure before the deadline; the key may still be "
f"propagating - last outcome {outcome.status_code}: {outcome.body[:200]}"
)
return outcome

View file

@ -1,285 +0,0 @@
"""Read-back for the Weave (Weights & Biases) logging tests against the real
Weave project.
The proxy ships OTEL spans to https://trace.wandb.ai/otel/v1/traces with the
``weave_otel`` callback, and the tests read the ingested calls back through
Weave's own query API (``POST /calls/stream_query``), which answers JSON Lines:
one JSON object per call, so the body is parsed line by line rather than as one
document.
The project is shared with other traffic, so the read never relies on the target
being among the newest N calls: the query is scoped server-side to the
``litellm_request`` op and to calls that started after the test's own request,
and pages with ``offset`` until the window is exhausted.
Weave's own ``summary.weave.status`` is a rollup that reads "success" even for a
span the exporter marked failed, so status comes from the OTEL span itself
(``attributes.otel_span.status.code``), and the shipped cost from
``attributes.otel_span.attributes.llm.response.cost`` - the StandardLogging
``response_cost``, which is what makes this a spend assertion rather than a
delivery ping.
Missing configuration is a hard failure, never a skip.
"""
from __future__ import annotations
import base64
import json
import os
import time
from dataclasses import dataclass, field
from itertools import count, takewhile
from typing import Final
import pytest
from pydantic import BaseModel, ConfigDict, Field
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT
from e2e_http import URL, AuthHeaders, send
from e2e_metadata import step
_WEAVE_TRACE_API: Final = "https://trace.wandb.ai"
#: The op every litellm LLM call lands under. The proxy also exports a root
#: server span ("Received Proxy Server Request") and management spans; only the
#: LLM call carries the usage and cost this suite asserts on.
LITELLM_REQUEST_OP: Final = "litellm_request"
#: How long to keep re-reading after the first matching call before trusting the
#: exactly-one assertion. The OTEL batch exporter flushes on its own schedule, so
#: a duplicate export can surface well after the first one, and a duplicate IS
#: the bug being guarded against.
WEAVE_SETTLE_SECONDS: Final = 45.0
#: Rows per page. The query is already scoped to this run's time window, so this
#: only bounds one round trip, not what the read can see.
_PAGE_SIZE: Final = 500
class _WeaveSortBy(BaseModel):
field: str
direction: str
class _WeaveOpFilter(BaseModel):
op_names: list[str]
class _WeaveGetField(BaseModel):
get_field: str = Field(serialization_alias="$getField")
class _WeaveLiteral(BaseModel):
literal: float = Field(serialization_alias="$literal")
class _WeaveGreaterThan(BaseModel):
gt: tuple[_WeaveGetField, _WeaveLiteral] = Field(serialization_alias="$gt")
class _WeaveQuery(BaseModel):
expr: _WeaveGreaterThan = Field(serialization_alias="$expr")
class _WeaveQueryBody(BaseModel):
project_id: str
filter: _WeaveOpFilter
query: _WeaveQuery
limit: int = _PAGE_SIZE
offset: int = 0
sort_by: list[_WeaveSortBy] = [_WeaveSortBy(field="started_at", direction="asc")]
class _OtelStatus(BaseModel):
model_config = ConfigDict(extra="ignore")
code: str | None = None
message: str | None = None
class _OtelError(BaseModel):
model_config = ConfigDict(extra="ignore")
code: str | None = None
type: str | None = None
message: str | None = None
class _LlmResponse(BaseModel):
model_config = ConfigDict(extra="ignore")
cost: float | None = None
class _LlmAttributes(BaseModel):
model_config = ConfigDict(extra="ignore")
response: _LlmResponse | None = None
class _OtelSpanAttributes(BaseModel):
model_config = ConfigDict(extra="ignore")
llm: _LlmAttributes | None = None
error: _OtelError | None = None
class _OtelSpan(BaseModel):
model_config = ConfigDict(extra="ignore")
name: str | None = None
status: _OtelStatus | None = None
attributes: _OtelSpanAttributes | None = None
class _CallAttributes(BaseModel):
model_config = ConfigDict(extra="ignore")
otel_span: _OtelSpan | None = None
class _Usage(BaseModel):
model_config = ConfigDict(extra="ignore")
total_tokens: int | None = None
class _WeaveSummary(BaseModel):
model_config = ConfigDict(extra="ignore")
usage: dict[str, _Usage] = {}
class WeaveCall(BaseModel):
"""One ingested Weave call, reduced to what the scenarios assert on."""
model_config = ConfigDict(extra="ignore")
id: str
op_name: str
started_at: str | None = None
inputs: dict[str, object] = {}
attributes: _CallAttributes | None = None
summary: _WeaveSummary | None = Field(default=None)
@property
def op(self) -> str:
"""The bare op name out of ``weave:///<entity>/<project>/op/<op>:<digest>``."""
return self.op_name.split("/op/")[-1].split(":")[0]
@property
def status_code(self) -> str | None:
"""The OTEL span status, not Weave's own rollup (which reads "success"
even for a span the exporter marked ERROR)."""
span = self.attributes.otel_span if self.attributes else None
return span.status.code if span and span.status else None
@property
def error(self) -> _OtelError | None:
span = self.attributes.otel_span if self.attributes else None
return span.attributes.error if span and span.attributes else None
@property
def response_cost(self) -> float | None:
span = self.attributes.otel_span if self.attributes else None
llm = span.attributes.llm if span and span.attributes else None
return llm.response.cost if llm and llm.response else None
@property
def total_tokens(self) -> int | None:
"""Weave keys usage by model, so the total is summed across whatever
models the call reported."""
if not self.summary or not self.summary.usage:
return None
totals = [usage.total_tokens for usage in self.summary.usage.values() if usage.total_tokens is not None]
return sum(totals) if totals else None
def mentions(self, needle: str) -> bool:
return needle in json.dumps(self.inputs, default=str)
@dataclass(frozen=True, slots=True)
class WeaveReader:
project_id: str
api_key: str = field(repr=False)
@property
def _headers(self) -> AuthHeaders:
"""Weave authenticates with HTTP Basic as the fixed user ``api``."""
token = base64.b64encode(f"api:{self.api_key}".encode()).decode()
return AuthHeaders(authorization=f"Basic {token}")
def _query_body(self, *, since: float, offset: int, op: str) -> _WeaveQueryBody:
return _WeaveQueryBody(
project_id=self.project_id,
filter=_WeaveOpFilter(op_names=[f"weave:///{self.project_id}/op/{op}:*"]),
query=_WeaveQuery(
expr=_WeaveGreaterThan(gt=(_WeaveGetField(get_field="started_at"), _WeaveLiteral(literal=since)))
),
offset=offset,
)
def _page(self, *, since: float, offset: int, op: str) -> tuple[WeaveCall, ...]:
outcome = send(
URL(f"{_WEAVE_TRACE_API}/calls/stream_query"),
headers=self._headers,
json=self._query_body(since=since, offset=offset, op=op),
)
if not outcome.ok:
pytest.fail(
f"Weave calls query for project {self.project_id!r} failed "
f"({outcome.status_code}): {outcome.body[:300]}"
)
return tuple(WeaveCall.model_validate_json(line) for line in outcome.body.splitlines() if line.strip())
@step("Read the Weave {op} calls carrying the marker {marker}")
def calls_matching(self, marker: str, *, since: float, op: str = LITELLM_REQUEST_OP) -> tuple[WeaveCall, ...]:
"""Every call under ``op`` started after ``since`` whose inputs carry
``marker``, paging until the window is exhausted.
More than one is the duplicate-delivery bug, so this never collapses to a
single call.
"""
pages = tuple(
takewhile(
bool,
(self._page(since=since, offset=offset, op=op) for offset in count(0, _PAGE_SIZE)),
)
)
return tuple(call for page in pages for call in page if call.mentions(marker))
@step("Wait for Weave to ingest a {op} call carrying the marker {marker}, then watch for duplicates")
def poll_calls_matching(self, marker: str, *, since: float, op: str = LITELLM_REQUEST_OP) -> tuple[WeaveCall, ...]:
"""Poll until the call is readable, then keep re-reading for
WEAVE_SETTLE_SECONDS so a duplicate exported by a later batch flush
cannot hide from the exactly-one assertion. A duplicate ends the settle
early, because more waiting cannot clear it."""
deadline = time.monotonic() + POLL_TIMEOUT
while time.monotonic() < deadline:
calls = self.calls_matching(marker, since=since, op=op)
if calls:
return self._settled(marker, since=since, op=op, first=calls)
time.sleep(POLL_INTERVAL)
return ()
def _settled(self, marker: str, *, since: float, op: str, first: tuple[WeaveCall, ...]) -> tuple[WeaveCall, ...]:
"""A transiently empty re-read never downgrades what was already seen."""
settle_deadline = time.monotonic() + WEAVE_SETTLE_SECONDS
latest = first # rebind-ok: one settle window, re-read per poll interval
while time.monotonic() < settle_deadline and len(latest) <= 1:
time.sleep(POLL_INTERVAL)
latest = self.calls_matching(marker, since=since, op=op) or latest
return latest
def build_weave_reader() -> WeaveReader:
project_id = (os.environ.get("WEAVE_PROJECT_ID") or os.environ.get("WANDB_PROJECT_ID") or "").strip()
api_key = os.environ.get("WANDB_API_KEY", "").strip()
if not project_id or not api_key:
pytest.fail(
"Weave e2e requires WANDB_API_KEY and WEAVE_PROJECT_ID (or WANDB_PROJECT_ID, "
"format <entity>/<project>): the test reads the proxy's weave_otel delivery "
"back from the real Weave project; missing credentials is a hard failure, not a skip"
)
return WeaveReader(project_id=project_id, api_key=api_key)

View file

@ -1,58 +0,0 @@
"""One-time helper to capture a logged-in Linear browser session for the
real-Linear MCP e2e test.
The real-Linear test drives the genuine gateway-managed authorization_code
dance against ``mcp.linear.app``. The only step that cannot be scripted is
Linear's login (magic link / SSO), so a human authenticates once here and the
resulting session (cookies + local storage) is persisted to disk. The e2e test
then loads that session in a headless Playwright context and clicks Approve on
Linear's consent screen every run, with no human and no login automation.
Run it with the e2e venv, log into Linear in the window that opens, then return
to the terminal and press Enter:
LITELLM=~/litellm-mcpe2e
"$LITELLM"/.venv/bin/python "$LITELLM"/tests/e2e/mcp/linear_session_capture.py
The session is written to ``E2E_LINEAR_STORAGE_STATE`` (default
``~/.litellm-e2e/linear_storage_state.json``), outside the repo. It is a
secret: never commit it. Re-run this whenever Linear expires the session.
"""
from __future__ import annotations
import os
from pathlib import Path
from playwright.sync_api import sync_playwright
DEFAULT_STATE_PATH = Path.home() / ".litellm-e2e" / "linear_storage_state.json"
def capture(state_path: Path) -> None:
"""Open a headed browser at Linear, wait for the human to log in, then save
the authenticated session to ``state_path``."""
state_path.parent.mkdir(parents=True, exist_ok=True)
with sync_playwright() as playwright:
browser = playwright.chromium.launch(headless=False)
context = browser.new_context()
page = context.new_page()
page.goto("https://linear.app/login", wait_until="domcontentloaded")
print("\n" + "=" * 72)
print("Log into Linear in the browser window that just opened.")
print("If Linear emails you a magic link, paste the link into THIS window's")
print("address bar (opening it in your default browser won't capture the")
print("session). Google SSO works too as long as you complete it here.")
print("When your Linear workspace has loaded, come back and press Enter.")
print("=" * 72)
input("Press Enter once you are logged in... ")
page.goto("https://mcp.linear.app/", wait_until="domcontentloaded")
context.storage_state(path=str(state_path))
browser.close()
print(f"\nSaved Linear session to {state_path}")
print("Point the e2e test at it with:")
print(f' export E2E_LINEAR_STORAGE_STATE="{state_path}"')
if __name__ == "__main__":
capture(Path(os.environ.get("E2E_LINEAR_STORAGE_STATE", str(DEFAULT_STATE_PATH))))

View file

@ -1,442 +0,0 @@
"""Client for the mcp chat-completion OAuth e2e suite.
Registers a gateway-managed OAuth (authorization_code) MCP server, seeds the
per-user upstream token by driving the interactive authorize dance with the
official mcp SDK's OAuthClientProvider (the browser leg is a headless Chromium
primed with a human's saved Linear session), then exercises the server through
/chat/completions, where the gateway lists and executes its tools with the
stored per-user token.
Management routes (/v1/mcp/server CRUD, /chat/completions) go through the
shared ProxyClient transport. The MCP protocol used to seed the token goes through
the mcp SDK, the same library production MCP hosts run.
"""
from __future__ import annotations
import asyncio
import re
import time
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from urllib.parse import parse_qsl
import httpx
import httpx2
import pytest
from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT
from e2e_http import AuthHeaders, NoBody, unwrap
from e2e_metadata import step
from idp import Identity
from mcp import ClientSession
from mcp.client.auth import OAuthClientProvider
from mcp.client.streamable_http import streamable_http_client
from mcp.shared.auth import AuthorizationCodeResult, OAuthClientInformationFull, OAuthClientMetadata, OAuthToken
from mcp.types import TextContent
from models import (
ChatBody,
ChatResponse,
McpOauthUserCredentialStatus,
McpServerCreateBody,
McpServerInfo,
McpServerUserCredentialListResponse,
McpServerUserCredentialRow,
)
from proxy_client import ProxyClient
if TYPE_CHECKING:
from playwright.async_api import Route
# Where the "browser" lands at the end of the authorize dance. Nothing listens
# here: the route interceptor short-circuits the final redirect and reads the
# code/state off its query string, exactly like a desktop MCP host intercepting
# its loopback redirect.
OAUTH_CLIENT_REDIRECT_URI = "http://127.0.0.1:53682/e2e/callback"
BROWSER_CONSENT_TIMEOUT = 60.0
def _mcp_url(alias: str, base_url: str = PROXY_BASE_URL) -> str:
return f"{base_url.rstrip('/')}/{alias}/mcp"
class InMemoryTokenStorage:
"""The mcp SDK's TokenStorage protocol, in memory for one dance: the
DCR-registered client and the gateway tokens minted for it."""
def __init__(self) -> None:
self._tokens: OAuthToken | None = None
self._client_info: OAuthClientInformationFull | None = None
async def get_tokens(self) -> OAuthToken | None:
return self._tokens
async def set_tokens(self, tokens: OAuthToken) -> None:
self._tokens = tokens
async def get_client_info(self) -> OAuthClientInformationFull | None:
return self._client_info
async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:
self._client_info = client_info
async def _browser_follow_authorize(
start_url: str,
storage_state_path: str,
identity: Identity | None = None,
server_alias: str | None = None,
allow_upstream_consent: bool = True,
) -> tuple[str, str | None]:
"""Play the browser's role for a real upstream whose authorize endpoint
serves an interactive consent page (Linear). A headless Chromium primed
with a human's saved Linear session opens the gateway authorize URL and
clicks through Linear's consent screens (the mcp.linear.app Approve form,
then the linear.app workspace-selection page), riding the rest of the chain
(Linear -> gateway callback -> host redirect_uri). The final hop is
intercepted and short-circuited, since nothing listens there, and its
code/state are read off the query string."""
from playwright.async_api import async_playwright
captured: dict[str, str] = {} # mutable-ok: hand-off from the request listener
trail: list[str] = [] # mutable-ok: navigation diagnostics for a failed dance
def _note_request(request: object) -> None:
url = getattr(request, "url", "")
host = httpx.URL(url).host
if not allow_upstream_consent and (host == "linear.app" or host.endswith(".linear.app")):
captured["upstream_consent"] = "seen"
if url.startswith(OAUTH_CLIENT_REDIRECT_URI) and "url" not in captured:
captured["url"] = url
async def _swallow_redirect(route: Route) -> None:
await route.fulfill(status=200, content_type="text/plain", body="ok")
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=True)
context = await browser.new_context(storage_state=storage_state_path)
await context.route(re.compile(re.escape(OAUTH_CLIENT_REDIRECT_URI) + r".*"), _swallow_redirect)
page = await context.new_page()
context.on("request", _note_request)
page.on("framenavigated", lambda frame: trail.append(frame.url.split("?", 1)[0]))
await page.goto(start_url, wait_until="domcontentloaded")
deadline = time.monotonic() + BROWSER_CONSENT_TIMEOUT
while "url" not in captured and time.monotonic() < deadline:
try:
await page.wait_for_load_state("networkidle", timeout=8000)
except Exception: # noqa: BLE001 - a busy consent page never idles; fall through and try to advance it
pass
if "upstream_consent" in captured or "url" in captured:
break
if await page.locator("#username").count() and identity is not None:
await page.locator("#username").fill(identity.username)
await page.locator("#password").fill(identity.password)
await page.locator("#kc-login").click()
continue
if "/ui/connect" in page.url and server_alias is not None:
card = page.locator("div.cursor-pointer").filter(has=page.get_by_text(server_alias, exact=True))
if await card.count() != 1:
await asyncio.sleep(0.5)
continue
connect = card.get_by_text("Connect", exact=True)
if await connect.count():
await connect.click()
continue
if not await card.locator("svg.text-success").count():
await asyncio.sleep(0.5)
continue
finish = page.get_by_role("button", name="Finish connecting", exact=True)
if await finish.count() and await finish.is_enabled():
await finish.click()
continue
control = page.locator(
'button[name="action"][value="approve"], button:has-text("Authorize"), '
'button:has-text("Allow"), button:has-text("@"), a:has-text("@")'
).first
try:
await control.click(timeout=5000)
except Exception: # noqa: BLE001 - nothing to advance yet; loop and re-check
await asyncio.sleep(0.5)
final_url = page.url
await browser.close()
# A redirect chain can finish inside goto/networkidle before the loop checks the page.
assert "upstream_consent" not in captured, "cold reconnect required upstream consent"
landing = captured.get("url")
assert landing is not None, (
f"consent flow never reached {OAUTH_CLIENT_REDIRECT_URI}; "
f"final={final_url.split('?', 1)[0]!r}; trail={trail[-6:]}"
)
params = dict(parse_qsl(httpx.URL(landing).query.decode()))
assert "code" in params, "client redirect_uri carried no authorization code"
return params["code"], params.get("state")
def _oauth_provider(
url: str,
storage: InMemoryTokenStorage,
storage_state_path: str | None,
identity: Identity | None = None,
server_alias: str | None = None,
allow_upstream_consent: bool = True,
) -> OAuthClientProvider:
"""The SDK's real OAuth machinery (RFC 9728/8414 discovery, RFC 7591 DCR,
PKCE, token exchange) with the browser leg driven by Playwright against the
upstream's consent screen."""
code_holder: dict[str, str | None] = {} # mutable-ok: hand-off between the two SDK callbacks
async def _reject_redirect(_: str) -> None:
raise AssertionError("gateway demanded a fresh upstream consent; stored per-user token was not reused")
async def _follow_redirect(authorize_url: str) -> None:
assert storage_state_path is not None
code, state = await _browser_follow_authorize(
authorize_url, storage_state_path, identity, server_alias, allow_upstream_consent
)
code_holder["code"] = code
code_holder["state"] = state
redirect_handler: Final = _reject_redirect if storage_state_path is None else _follow_redirect
async def callback_handler() -> AuthorizationCodeResult:
code = code_holder.get("code")
assert code is not None, "callback_handler ran before the authorize redirect completed"
return AuthorizationCodeResult(code=code, state=code_holder.get("state"))
return OAuthClientProvider(
server_url=url,
client_metadata=OAuthClientMetadata.model_validate(
{
"redirect_uris": [OAUTH_CLIENT_REDIRECT_URI],
"token_endpoint_auth_method": "none",
"grant_types": ["authorization_code", "refresh_token"],
"response_types": ["code"],
"client_name": "e2e-mcp-host",
}
),
storage=storage,
redirect_handler=redirect_handler,
callback_handler=callback_handler,
)
class _HeaderInjectingTransport(httpx2.AsyncBaseTransport):
"""Adds the caller's LiteLLM key header to every outgoing SDK request
(discovery, DCR, token exchange), so the gateway resolves which user to
store the upstream token for from the key on the token exchange, exactly
like a production MCP host configured with a LiteLLM key header."""
def __init__(self, inner: httpx2.AsyncBaseTransport, headers: dict[str, str], gateway_url: str) -> None:
self._inner = inner
self._headers = headers
self._gateway_url = httpx2.URL(gateway_url)
@staticmethod
def _port(url: httpx2.URL) -> int | None:
if url.port is not None:
return url.port
return {"http": 80, "https": 443}.get(url.scheme)
async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
same_origin: Final = (
request.url.scheme == self._gateway_url.scheme
and request.url.host == self._gateway_url.host
and self._port(request.url) == self._port(self._gateway_url)
)
if same_origin:
for name, value in self._headers.items():
if name not in request.headers:
request.headers[name] = value
else:
for name, value in self._headers.items():
if request.headers.get(name) == value:
del request.headers[name]
return await self._inner.handle_async_request(request)
async def aclose(self) -> None:
await self._inner.aclose()
def _oauth_http_client(
headers: dict[str, str], auth: OAuthClientProvider, gateway_url: str = PROXY_BASE_URL
) -> httpx2.AsyncClient:
return httpx2.AsyncClient(
auth=auth,
timeout=httpx2.Timeout(REQUEST_TIMEOUT),
follow_redirects=True,
transport=_HeaderInjectingTransport(httpx2.AsyncHTTPTransport(), headers, gateway_url),
)
async def _seed_via_dance(
url: str, headers: dict[str, str], storage: InMemoryTokenStorage, storage_state_path: str
) -> tuple[str, ...]:
async with _oauth_http_client(headers, _oauth_provider(url, storage, storage_state_path)) as http_client:
async with streamable_http_client(url, http_client=http_client) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
listed = await session.list_tools()
return tuple(sorted(tool.name for tool in listed.tools))
@dataclass(frozen=True, slots=True)
class OauthToolRun:
tools: tuple[str, ...]
is_error: bool
text: str
async def _list_and_call(
url: str,
headers: dict[str, str],
storage: InMemoryTokenStorage,
storage_state_path: str | None,
tool: str,
arguments: dict[str, str],
gateway_url: str = PROXY_BASE_URL,
identity: Identity | None = None,
server_alias: str | None = None,
allow_upstream_consent: bool = True,
) -> OauthToolRun:
async with _oauth_http_client(
headers,
_oauth_provider(url, storage, storage_state_path, identity, server_alias, allow_upstream_consent),
gateway_url,
) as http_client:
async with streamable_http_client(url, http_client=http_client) as (read, write):
async with ClientSession(read, write) as session:
await session.initialize()
listed: Final = await session.list_tools()
result: Final = await session.call_tool(tool, arguments)
text: Final = "".join(content.text for content in result.content if isinstance(content, TextContent))
return OauthToolRun(
tools=tuple(sorted(tool_item.name for tool_item in listed.tools)),
is_error=result.is_error,
text=text,
)
@dataclass(frozen=True, slots=True)
class ChatMcpClient:
proxy: ProxyClient
@step("Register the MCP server with the alias {body.alias}")
def create_server(self, body: McpServerCreateBody) -> McpServerInfo:
return unwrap(
self.proxy.transport.post(
"/v1/mcp/server",
headers=self.proxy.transport.master,
json=body,
response_type=McpServerInfo,
)
)
@step("Read the MCP server back from /v1/mcp/server")
def server_info(self, server_id: str) -> McpServerInfo:
return unwrap(
self.proxy.transport.get(
f"/v1/mcp/server/{server_id}",
headers=self.proxy.transport.master,
params=NoBody(),
response_type=McpServerInfo,
)
)
@step("Delete the MCP server")
def delete_server(self, server_id: str) -> None:
_ = self.proxy.transport.delete(
f"/v1/mcp/server/{server_id}",
headers=self.proxy.transport.master,
json=NoBody(),
response_type=NoBody,
)
@step("Sign the key's user in to the MCP server {alias} through the OAuth consent flow")
def seed_user_token(self, alias: str, key: str, storage_state_path: str) -> tuple[str, ...]:
"""Drive the interactive authorize dance for `key`'s user so the gateway
stores their upstream token, retried to the shared deadline since the
just-created server and key propagate asynchronously. The LiteLLM key
rides x-litellm-api-key so the gateway binds the token to that user.
Returns the upstream tool names the dance listed, proof the token works."""
headers = {"x-litellm-api-key": f"Bearer {key}"}
storage = InMemoryTokenStorage()
deadline = time.monotonic() + self.proxy.poll_timeout
last_error: Exception | None = None
while time.monotonic() < deadline:
try:
return asyncio.run(_seed_via_dance(_mcp_url(alias), headers, storage, storage_state_path))
except Exception as exc: # noqa: BLE001 - retried to the deadline; the last error surfaces below
last_error = exc
time.sleep(self.proxy.poll_interval)
pytest.fail(
f"authorize dance for {alias!r} never completed within {self.proxy.poll_timeout}s; "
f"last error: {last_error!r}"
)
@step("List the tools on the MCP server {alias} and call {tool} over the MCP protocol")
def list_and_call(
self,
alias: str,
headers: dict[str, str],
storage: InMemoryTokenStorage,
storage_state_path: str | None,
tool: str,
arguments: dict[str, str],
base_url: str = PROXY_BASE_URL,
identity: Identity | None = None,
allow_upstream_consent: bool = True,
) -> OauthToolRun:
return asyncio.run(
_list_and_call(
f"{base_url.rstrip('/')}/mcp" if identity is not None else _mcp_url(alias, base_url),
headers,
storage,
storage_state_path,
tool,
arguments,
base_url,
identity,
alias,
allow_upstream_consent,
)
)
@step("List the users with a stored OAuth token for the MCP server")
def server_user_credentials(self, server_id: str) -> tuple[McpServerUserCredentialRow, ...]:
return unwrap(
self.proxy.transport.get(
f"/v1/mcp/server/{server_id}/user-credentials",
headers=self.proxy.transport.master,
params=NoBody(),
response_type=McpServerUserCredentialListResponse,
)
).root
@step("Revoke the user's stored OAuth token for the MCP server")
def revoke_user_token(self, server_id: str, headers: AuthHeaders) -> None:
_ = unwrap(
self.proxy.transport.delete(
f"/v1/mcp/server/{server_id}/oauth-user-credential",
headers=headers,
json=NoBody(),
response_type=McpOauthUserCredentialStatus,
)
)
@step("Send a /chat/completions request to {body.model} with an MCP server attached as a tool")
def chat_with_mcp(self, headers: AuthHeaders, body: ChatBody) -> ChatResponse:
"""POST /chat/completions carrying the LiteLLM key in `headers` (either
ingress form) with an MCP server attached in `body.tools`. The gateway
resolves the user from the key and lists/executes the server's tools
with that user's stored upstream token."""
return unwrap(
self.proxy.transport.post(
"/chat/completions",
headers=headers,
json=body,
response_type=ChatResponse,
)
)
def build_chat_client(proxy: ProxyClient) -> ChatMcpClient:
return ChatMcpClient(proxy=proxy)

View file

@ -1,197 +0,0 @@
"""An owned, source-built OAuth gateway with cold restarts and credential observations.
Only this child process is restarted. Its database and SSO client survive while
its process-local caches do not; Redis is deliberately absent from its config.
The optional live edge measures headers without recording credentials or bodies.
"""
from __future__ import annotations
import os
import subprocess
import sys
import threading
import time
from collections.abc import Callable, Mapping
from contextlib import ExitStack
from dataclasses import dataclass, field
from pathlib import Path
from typing import Final
import psycopg
from e2e_config import INHERITED_ENV_PREFIXES, available_port
from e2e_http import NoBody
from e2e_metadata import step
from idp import Keycloak, stop_process_group
from proxy_client import ProxyClient, build_proxy_client
from psycopg.rows import class_row
from pydantic import BaseModel, SecretStr, ValidationError
class StoredOAuth(BaseModel):
type: str
access_token: SecretStr
@dataclass(frozen=True, slots=True)
class CredentialRow:
credential_b64: str = field(repr=False)
@step("Read the user's stored OAuth credential for the MCP server from the database and decrypt it")
def stored_oauth(user_id: str, server_id: str) -> StoredOAuth:
"""Read the encrypted credential because management APIs omit the plaintext token."""
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
with psycopg.Connection[CredentialRow].connect(
os.environ["DATABASE_URL"], row_factory=class_row(CredentialRow)
) as conn:
row: Final = conn.execute(
'SELECT credential_b64 FROM "LiteLLM_MCPUserCredentials" WHERE user_id = %s AND server_id = %s',
(user_id, server_id),
).fetchone()
assert row is not None, "canonical user/server has no persisted credential"
plaintext: Final = decrypt_value_helper(
row.credential_b64, "e2e_mcp_oauth", exception_type="debug", return_original_value=False
)
assert plaintext is not None, "persisted credential must decrypt with the gateway salt"
assert plaintext != row.credential_b64, "persisted credential must be encrypted"
try:
credential: Final = StoredOAuth.model_validate_json(plaintext)
except ValidationError:
raise AssertionError("decrypted credential is not an OAuth payload") from None
assert credential.type == "oauth2"
assert bool(credential.access_token.get_secret_value()), "stored upstream token is empty"
return credential
class RpcMethod(BaseModel):
method: str = ""
@dataclass(slots=True)
class OAuthObservation:
gateway_token: str = field(default="", repr=False)
_seen: tuple[tuple[str, str, bool], ...] = field(default=(), init=False, repr=False)
_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
if body is None or not url.endswith("/mcp"):
return
try:
operation: Final = RpcMethod.model_validate_json(body).method
except ValidationError:
return
if operation not in ("tools/list", "tools/call"):
return
received: Final = headers.get("authorization", "")
gateway_leaked: Final = any(
value in (self.gateway_token, f"Bearer {self.gateway_token}") for value in headers.values()
)
with self._lock:
self._seen = (*self._seen, (operation, received, gateway_leaked))
def assert_forwarded(self, expected: StoredOAuth) -> None:
with self._lock:
snapshot: Final = self._seen
self._seen = ()
assert {item[0] for item in snapshot} == {"tools/list", "tools/call"}, "missing upstream observations"
expected_header: Final = f"Bearer {expected.access_token.get_secret_value()}"
assert all(item[1] == expected_header for item in snapshot), "upstream bearer did not match the stored token"
assert all(not item[2] for item in snapshot), "gateway bearer leaked to the upstream"
@dataclass(slots=True)
class OAuthGateway:
base_url: str
proxy: ProxyClient
_environment: Mapping[str, str] = field(repr=False)
_command: tuple[str, ...] = field(repr=False)
_log_path: Path
_child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False)
@step("Start the separate LiteLLM proxy and wait for /health/liveliness")
def start(self) -> None:
with self._log_path.open("ab") as log:
self._child = subprocess.Popen(
self._command,
env=self._environment,
stdout=log,
stderr=log,
start_new_session=True,
)
deadline: Final = time.monotonic() + 120
while time.monotonic() < deadline:
assert self._child.poll() is None, "owned OAuth gateway exited; inspect its private log"
result = self.proxy.transport.probe("/health/liveliness", params=NoBody())
if result.status_code == 200:
return
time.sleep(0.5)
raise AssertionError("owned OAuth gateway did not become ready")
@step("Stop the separate LiteLLM proxy")
def stop(self) -> None:
if self._child is not None:
stop_process_group(self._child)
assert self._child.poll() is not None, "old gateway process is still alive"
@step("Restart the separate LiteLLM proxy process so its in-memory caches start empty")
def restart(self) -> None:
assert self._child is not None
previous: Final = self._child.pid
self.stop()
self.start()
assert self._child.pid != previous, "gateway restart did not create a new process"
@step("Start a separate LiteLLM proxy from source with JWT auth against Keycloak")
def owned_gateway(idp: Keycloak, directory: Path, cleanup: ExitStack) -> OAuthGateway:
for name in ("DATABASE_URL", "LITELLM_LICENSE", "LITELLM_SALT_KEY", "LITELLM_MASTER_KEY"):
assert os.environ.get(name), f"{name} is required for the owned OAuth gateway"
port: Final = available_port()
base_url: Final = f"http://127.0.0.1:{port}"
def defer(callback: Callable[[], object]) -> None:
cleanup.callback(callback)
browser: Final = idp.browser_client(callback_url=f"{base_url}/sso/callback", defer=defer)
config: Final = directory / "oauth-gateway.yaml"
config.write_text(
"model_list: []\n"
"general_settings:\n"
" master_key: os.environ/LITELLM_MASTER_KEY\n"
" database_url: os.environ/DATABASE_URL\n"
" enable_jwt_auth: true\n"
" litellm_jwtauth:\n"
" user_id_jwt_field: sub\n"
" user_email_jwt_field: email\n"
" team_ids_jwt_field: groups\n"
" user_id_upsert: true\n"
)
environment: Final = {
**{key: value for key, value in os.environ.items() if not key.startswith(INHERITED_ENV_PREFIXES)},
**browser.environment(idp.discovery()),
"PROXY_BASE_URL": base_url,
"JWT_PUBLIC_KEY_URL": idp.jwks_url,
"JWT_ISSUER": idp.issuer,
"JWT_AUDIENCE": "litellm-e2e",
"DISABLE_SCHEMA_UPDATE": "true",
"STORE_MODEL_IN_DB": "True",
"PYTHONPATH": str(Path(__file__).resolve().parents[3]),
}
gateway: Final = OAuthGateway(
base_url=base_url,
proxy=build_proxy_client(
base_url=base_url,
control_plane_base_url=base_url,
replica_urls=(base_url,),
control_replica_urls=(base_url,),
master_key=os.environ["LITELLM_MASTER_KEY"],
),
_environment=environment,
_command=(sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config), "--port", str(port)),
_log_path=directory / "oauth-gateway.log",
)
cleanup.callback(gateway.stop)
gateway.start()
return gateway

View file

@ -1,218 +0,0 @@
"""On-demand e2e: a chat completion drives a gateway-managed OAuth MCP server.
The real end-user flow for MCP over an OAuth server: a user registers a Linear
authorization_code server, authorizes it once so the gateway stores their
upstream token, then sends a normal /chat/completions request with the Linear
MCP attached. The gateway resolves the user from the LiteLLM key, lists Linear's
tools with the stored per-user token, lets the model call one, executes it
upstream with that token, and returns the answer. This is proven against the
real Linear MCP server (mcp.linear.app) and a real Anthropic model, once per
documented ingress header (x-litellm-api-key and Authorization).
The authorize dance is seeded through the mcp SDK's OAuthClientProvider; the one
step Linear cannot auto-approve is the human consent, so it is captured once out
of band (mcp/linear_session_capture.py) into a saved browser session and a
headless Chromium clicks Approve every run. The test therefore skips unless
E2E_LINEAR_STORAGE_STATE points at that session, so it never runs on the per-PR
CI path; it is a nightly/on-demand real-server smoke test.
Fail-before-fix: without the stored per-user token the gateway lists no Linear
tools, so mcp_list_tools comes back empty, nothing is called, and the
assertions fail; a served, called, non-empty Linear tool proves the gateway
pulled and used the user's token.
"""
from __future__ import annotations
import os
import pytest
from e2e_config import CHEAP_ANTHROPIC_MODEL, LINEAR_MCP_URL, LINEAR_STORAGE_STATE, unique_marker
from e2e_http import AuthHeaders
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from models import ChatBody, ChatMessage, KeyGenerateBody, McpChatTool, McpServerCreateBody, ObjectPermission
from proxy_client import ProxyClient
pytest.importorskip("mcp", reason="mcp SDK not installed; run `uv sync --inexact --group e2e-dev`")
pytest.importorskip(
"playwright.async_api",
reason="playwright not installed; run `uv pip install playwright` and `playwright install chromium`",
)
from oauth_chat_client import ChatMcpClient, build_chat_client # noqa: E402 # imports follow the importorskip guards
pytestmark = [
pytest.mark.e2e,
pytest.mark.skipif(
not LINEAR_STORAGE_STATE or not os.path.exists(LINEAR_STORAGE_STATE),
reason="set E2E_LINEAR_STORAGE_STATE to a Linear session captured via mcp/linear_session_capture.py",
),
]
# Pinned from a live dance during verification (never guessed); the gateway
# prefixes every upstream tool name with the server alias. list_teams is a
# read-only Linear tool that takes no arguments and returns the caller's teams.
LINEAR_READONLY_TOOL = "list_teams"
LINEAR_PROMPT = "Use the list_teams tool to list my Linear teams, then reply with the name of one of them."
@pytest.fixture(scope="session")
def chat_client(proxy: ProxyClient) -> ChatMcpClient:
return build_chat_client(proxy)
class TestMcpChatCompletionOauth:
"""A scoped internal-user key on a real Linear authorization_code server,
used through /chat/completions once per ingress header: the gateway pulls
the user's stored upstream token, lists and executes Linear's tools during
the completion, and returns the answer."""
@pytest.mark.covers("mcp.list_tools.oauth.succeeds")
@pytest.mark.covers("mcp.call_tool.oauth.succeeds")
@meta(
Subject(
domain=Domain.MCP,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.ANTHROPIC,),
models=(CHEAP_ANTHROPIC_MODEL,),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_chat_completion_uses_linear_with_x_litellm_api_key_header(
self, chat_client: ChatMcpClient, resources: ResourceManager
) -> None:
marker = unique_marker()
alias = f"e2elinear{marker}"
created = chat_client.create_server(
McpServerCreateBody(
alias=alias,
url=LINEAR_MCP_URL,
allow_all_keys=False,
auth_type="oauth2",
oauth2_flow="authorization_code",
)
)
resources.defer(lambda: chat_client.delete_server(created.server_id))
stored = chat_client.server_info(created.server_id)
assert stored.auth_type == "oauth2"
assert stored.oauth2_flow == "authorization_code"
assert stored.allow_all_keys is False
key = chat_client.proxy.generate_key(
KeyGenerateBody(
user_id="e2e-test-user",
object_permission=ObjectPermission(mcp_servers=[created.server_id]),
)
)
resources.defer(lambda: chat_client.proxy.delete_key(key))
seeded = chat_client.seed_user_token(alias, key, LINEAR_STORAGE_STATE)
assert f"{alias}-{LINEAR_READONLY_TOOL}" in seeded, (
f"the authorize dance listed {seeded}, expected it to include {alias}-{LINEAR_READONLY_TOOL}"
)
response = chat_client.chat_with_mcp(
AuthHeaders.model_validate({"x-litellm-api-key": f"Bearer {key}"}),
ChatBody(
model=CHEAP_ANTHROPIC_MODEL,
messages=[ChatMessage(role="user", content=LINEAR_PROMPT)],
tools=[
McpChatTool(
server_url=f"litellm_proxy/mcp/{alias}",
server_label=alias,
require_approval="never",
)
],
),
)
message = response.choices[0].message
assert message is not None and message.content, f"completion returned no answer: {response}"
meta = message.provider_specific_fields
assert meta is not None, f"no MCP metadata on the completion: {response}"
listed = {t.function.name for t in (meta.mcp_list_tools or []) if t.function}
assert f"{alias}-{LINEAR_READONLY_TOOL}" in listed, (
f"the gateway listed {sorted(listed)}, expected the stored token to surface {alias}-{LINEAR_READONLY_TOOL}"
)
results = [r for r in (meta.mcp_call_results or []) if r.name == f"{alias}-{LINEAR_READONLY_TOOL}"]
assert results and results[0].result, (
f"Linear tool {alias}-{LINEAR_READONLY_TOOL} was not executed with a result: {meta.mcp_call_results}"
)
@pytest.mark.covers("mcp.list_tools.oauth.succeeds")
@pytest.mark.covers("mcp.call_tool.oauth.succeeds")
@meta(
Subject(
domain=Domain.MCP,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.ANTHROPIC,),
models=(CHEAP_ANTHROPIC_MODEL,),
capabilities=(Capability.FUNCTION_CALLING,),
mode=Mode.NONSTREAM,
)
)
def test_chat_completion_uses_linear_with_authorization_bearer_header(
self, chat_client: ChatMcpClient, resources: ResourceManager
) -> None:
marker = unique_marker()
alias = f"e2elinear{marker}"
created = chat_client.create_server(
McpServerCreateBody(
alias=alias,
url=LINEAR_MCP_URL,
allow_all_keys=False,
auth_type="oauth2",
oauth2_flow="authorization_code",
)
)
resources.defer(lambda: chat_client.delete_server(created.server_id))
stored = chat_client.server_info(created.server_id)
assert stored.auth_type == "oauth2"
assert stored.oauth2_flow == "authorization_code"
assert stored.allow_all_keys is False
key = chat_client.proxy.generate_key(
KeyGenerateBody(
user_id="e2e-test-user",
object_permission=ObjectPermission(mcp_servers=[created.server_id]),
)
)
resources.defer(lambda: chat_client.proxy.delete_key(key))
seeded = chat_client.seed_user_token(alias, key, LINEAR_STORAGE_STATE)
assert f"{alias}-{LINEAR_READONLY_TOOL}" in seeded, (
f"the authorize dance listed {seeded}, expected it to include {alias}-{LINEAR_READONLY_TOOL}"
)
response = chat_client.chat_with_mcp(
AuthHeaders.model_validate({"authorization": f"Bearer {key}"}),
ChatBody(
model=CHEAP_ANTHROPIC_MODEL,
messages=[ChatMessage(role="user", content=LINEAR_PROMPT)],
tools=[
McpChatTool(
server_url=f"litellm_proxy/mcp/{alias}",
server_label=alias,
require_approval="never",
)
],
),
)
message = response.choices[0].message
assert message is not None and message.content, f"completion returned no answer: {response}"
meta = message.provider_specific_fields
assert meta is not None, f"no MCP metadata on the completion: {response}"
listed = {t.function.name for t in (meta.mcp_list_tools or []) if t.function}
assert f"{alias}-{LINEAR_READONLY_TOOL}" in listed, (
f"the gateway listed {sorted(listed)}, expected the stored token to surface {alias}-{LINEAR_READONLY_TOOL}"
)
results = [r for r in (meta.mcp_call_results or []) if r.name == f"{alias}-{LINEAR_READONLY_TOOL}"]
assert results and results[0].result, (
f"Linear tool {alias}-{LINEAR_READONLY_TOOL} was not executed with a result: {meta.mcp_call_results}"
)

View file

@ -1,216 +0,0 @@
"""Real OAuth consent, immediate MCP operations and cold-restart persistence.
Aggregate SSO uses the SDK's normal authentication. The per-server variant is
explicitly a configured two-header client, not an Authorization-only OAuth host.
The observed variants forward to the same real Linear upstream and compare its
bearer at the forwarding boundary; direct variants retain unmodified discovery.
"""
from __future__ import annotations
import os
from collections.abc import Iterator
from contextlib import ExitStack
from pathlib import Path
from types import MappingProxyType
from typing import Final, Literal
import pytest
from e2e_config import LINEAR_MCP_URL, LINEAR_READONLY_TOOL, LINEAR_STORAGE_STATE, unique_marker
from e2e_http import AuthHeaders, NoBody, get_external, unwrap
from e2e_metadata import Domain, Route, Subject, meta
from idp import Identity, Keycloak
from lifecycle import ResourceManager
from models import (
McpOauthCredentials,
McpServerCreateBody,
ObjectPermission,
TeamMemberAddBody,
TeamMemberEntry,
TeamUpdateBody,
)
from oauth_chat_client import ChatMcpClient, InMemoryTokenStorage, OauthToolRun, build_chat_client
from oauth_gateway import OAuthGateway, OAuthObservation, owned_gateway, stored_oauth
from provider_edge import LiveEdge, start_provider_edge
from proxy_client import ProxyClient
from pydantic import BaseModel, ValidationError
pytestmark = [pytest.mark.e2e, pytest.mark.mcp_oauth_live, pytest.mark.provider_live]
class OAuthMetadata(BaseModel):
issuer: str
authorization_endpoint: str
token_endpoint: str
registration_endpoint: str
class LinearTeam(BaseModel):
id: str
name: str
class LinearTeams(BaseModel):
teams: tuple[LinearTeam, ...]
def assert_tool_result(run: OauthToolRun, tool: str) -> None:
assert tool in run.tools
assert run.is_error is False
try:
result: Final = LinearTeams.model_validate_json(run.text)
except ValidationError:
raise AssertionError("list_teams did not return the expected teams payload") from None
assert result.teams, "the test workspace must contain at least one team"
assert all(team.id and team.name for team in result.teams), "team results must contain identifiers and names"
@pytest.fixture(scope="module")
def oauth_gateway(idp: Keycloak, tmp_path_factory: pytest.TempPathFactory) -> Iterator[OAuthGateway]:
assert LINEAR_STORAGE_STATE and Path(LINEAR_STORAGE_STATE).is_file(), (
"E2E_LINEAR_STORAGE_STATE must name a captured Linear login; see mcp/linear_session_capture.py"
)
assert os.environ.get("E2E_FIXTURE_MODE", "live") == "live", "OAuth acceptance cannot use replay"
with ExitStack() as cleanup:
yield owned_gateway(idp, tmp_path_factory.mktemp("mcp-oauth"), cleanup)
@pytest.fixture(scope="module")
def proxy(oauth_gateway: OAuthGateway) -> ProxyClient:
return oauth_gateway.proxy
@pytest.fixture(scope="module")
def client(proxy: ProxyClient) -> ChatMcpClient:
return build_chat_client(proxy)
class TestMcpOauthHappyPath:
@pytest.mark.covers("mcp.list_tools.oauth.succeeds")
@pytest.mark.covers("mcp.call_tool.oauth.succeeds")
@pytest.mark.covers("mcp.call_tool.oauth.persists_across_processes")
@meta(
Subject(
domain=Domain.MCP,
route=Route.MCP,
)
)
@pytest.mark.parametrize("route", ("aggregate_sso", "explicit_header_jwt"))
@pytest.mark.parametrize("observed", (False, True), ids=("direct", "observed"))
def test_consent_list_call_and_cold_restart(
self,
client: ChatMcpClient,
resources: ResourceManager,
jwt_identity: Identity,
idp: Keycloak,
oauth_gateway: OAuthGateway,
route: Literal["aggregate_sso", "explicit_header_jwt"],
observed: bool,
) -> None:
alias: Final = f"e2elinear{unique_marker()}"
tool: Final = f"{alias}-{LINEAR_READONLY_TOOL}"
token: Final = idp.access_token(jwt_identity)
observation: Final = OAuthObservation(gateway_token=token)
edge: Final = (
start_provider_edge(
LiveEdge(observe_request=observation.observe),
mounts=MappingProxyType(
{"linear": "https://mcp.linear.app", ".well-known": "https://mcp.linear.app/.well-known"}
),
)
if observed
else None
)
if edge is not None:
resources.defer(edge.shutdown)
metadata: Final = (
unwrap(
get_external(
"https://mcp.linear.app/.well-known/oauth-authorization-server",
response_type=OAuthMetadata,
)
)
if observed
else None
)
created: Final = client.create_server(
McpServerCreateBody(
alias=alias,
server_name=alias,
url=f"{edge.edge.api_base('linear')}/mcp" if edge is not None else LINEAR_MCP_URL,
transport="http",
allow_all_keys=False,
auth_type="oauth2",
oauth2_flow="authorization_code",
per_server_oauth_discovery=route == "explicit_header_jwt",
issuer=metadata.issuer if metadata else None,
authorization_url=metadata.authorization_endpoint if metadata else None,
token_url=metadata.token_endpoint if metadata else None,
registration_url=metadata.registration_endpoint if metadata else None,
credentials=McpOauthCredentials(upstream_resource=LINEAR_MCP_URL) if observed else None,
)
)
resources.defer(lambda: client.delete_server(created.server_id))
assert client.server_user_credentials(created.server_id) == (), (
"scenario must start without upstream credentials"
)
unwrap(
client.proxy.transport.post(
"/team/member_add",
headers=client.proxy.transport.master,
json=TeamMemberAddBody(
team_id=jwt_identity.group, member=TeamMemberEntry(user_id=jwt_identity.user_id, role="user")
),
response_type=NoBody,
)
)
client.proxy.update_team(
TeamUpdateBody(
team_id=jwt_identity.group,
object_permission=ObjectPermission(mcp_servers=[created.server_id]),
)
)
headers: Final = {"x-litellm-api-key": f"Bearer {token}"} if route == "explicit_header_jwt" else {}
resources.defer(
lambda: client.revoke_user_token(
created.server_id,
AuthHeaders(authorization=f"Bearer {idp.access_token(jwt_identity)}"),
)
)
identity: Final = jwt_identity if route == "aggregate_sso" else None
first: Final = client.list_and_call(
alias,
headers,
InMemoryTokenStorage(),
LINEAR_STORAGE_STATE,
tool,
{},
base_url=oauth_gateway.base_url,
identity=identity,
)
assert_tool_result(first, tool)
credentials: Final = client.server_user_credentials(created.server_id)
assert len(credentials) == 1
assert credentials[0].user_id == jwt_identity.user_id
assert credentials[0].credential_type == "oauth2"
first_stored_oauth: Final = stored_oauth(jwt_identity.user_id, created.server_id)
if observed:
observation.assert_forwarded(first_stored_oauth)
oauth_gateway.restart()
fresh_token: Final = idp.access_token(jwt_identity)
observation.gateway_token = fresh_token
second: Final = client.list_and_call(
alias,
{"Authorization": f"Bearer {fresh_token}"} if identity is None else {},
InMemoryTokenStorage(),
LINEAR_STORAGE_STATE if identity is not None else None,
tool,
{},
base_url=oauth_gateway.base_url,
identity=identity,
allow_upstream_consent=False,
)
assert_tool_result(second, tool)
second_stored_oauth: Final = stored_oauth(jwt_identity.user_id, created.server_id)
if observed:
observation.assert_forwarded(second_stored_oauth)

View file

@ -45,8 +45,6 @@ class KeyLoggingCallbackVars(BaseModel):
langfuse_public_key: str | None = Field(default=None, repr=False)
langfuse_secret_key: str | None = Field(default=None, repr=False)
langfuse_host: str | None = Field(default=None, repr=False)
wandb_api_key: str | None = Field(default=None, repr=False)
weave_project_id: str | None = None
class KeyLoggingCallback(BaseModel):
@ -364,6 +362,7 @@ class ChatBody(BaseModel):
thinking: ThinkingParam | None = None
service_tier: str | None = None
prompt_cache_key: str | None = None
prompt_cache_retention: str | None = None
tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None
tool_choice: str | None = None
guardrails: list[str] | None = None
@ -732,28 +731,14 @@ class McpInfo(BaseModel):
logo_url: str | None = None
class McpOauthCredentials(BaseModel):
upstream_resource: str
class McpServerCreateBody(BaseModel):
"""POST /v1/mcp/server. For a gateway-managed OAuth server, `auth_type` is
`oauth2` and `oauth2_flow` is `authorization_code`; the upstream endpoints
are discovered and registered via DCR when left unset. `allow_all_keys`
false scopes the server to keys granted it through object_permission."""
"""POST /v1/mcp/server. `allow_all_keys` false scopes the server to keys
granted it through object_permission."""
alias: str
url: str
transport: str = "http"
allow_all_keys: bool = True
auth_type: str | None = None
oauth2_flow: Literal["client_credentials", "authorization_code"] | None = None
per_server_oauth_discovery: bool | None = None
issuer: str | None = None
authorization_url: str | None = None
token_url: str | None = None
registration_url: str | None = None
credentials: McpOauthCredentials | None = Field(default=None, repr=False)
server_name: str | None = None
description: str | None = None
mcp_info: McpInfo | None = None
@ -793,26 +778,6 @@ class McpServerListResponse(RootModel[list[McpServerRow]]):
"""GET /v1/mcp/server answers with a bare array of servers."""
class McpServerUserCredentialRow(BaseModel):
user_id: str
credential_type: Literal["oauth2", "byok"]
expires_at: str | None = None
connected_at: str | None = None
updated_at: str
class McpServerUserCredentialListResponse(RootModel[tuple[McpServerUserCredentialRow, ...]]):
"""GET /v1/mcp/server/{server_id}/user-credentials answers with a bare array."""
class McpOauthUserCredentialStatus(BaseModel):
server_id: str
has_credential: bool
expires_at: str | None = None
is_expired: bool = False
connected_at: str | None = None
class ToolsetTool(BaseModel):
server_id: str
tool_name: str
@ -1392,9 +1357,11 @@ class LiteLLMParamsBody(BaseModel):
litellm_credential_name: str | None = None
api_base: str | None = Field(default=None, repr=False)
api_version: str | None = None
responses_api_version: str | None = None
realtime_protocol: str | None = None
allowed_openai_params: list[str] | None = None
drop_params: bool | None = None
thought_signature_in_tool_call_id: bool | None = None
aws_access_key_id: str | None = Field(default=None, repr=False)
aws_secret_access_key: str | None = Field(default=None, repr=False)
aws_region_name: str | None = None
@ -1448,7 +1415,7 @@ class LiteLLMParamsBody(BaseModel):
order: int | None = None
ModelMode = Literal["chat", "batch", "realtime", "image_generation"]
ModelMode = Literal["chat", "embedding", "batch", "realtime", "image_generation"]
class TokenCountModelBody(BaseModel, frozen=True):
@ -1457,6 +1424,7 @@ class TokenCountModelBody(BaseModel, frozen=True):
class ModelInfoBody(BaseModel):
enforce_supported_endpoints: bool | None = None
# id is left unset so the proxy assigns a unique model_id per deployment.
# Pinning it to the model_name made re-registrations of a fixed-name model
# (e.g. the batch suite's openai-batch) collide on the model_id unique

View file

@ -0,0 +1,59 @@
# Admin UI journey latency on the 1TB SpendLogs restore
Run `test_ui_journeys.py` against a dedicated proxy, its Admin UI and a disposable database restored from the metadata-only performance seed. A headless Chromium logs into the dashboard and performs the journeys below, timing each one from the user's click or navigation until the data is on screen. No LLM is called. The test does not look at which routes, SQL or tables served the page, so it stays valid whether the backend gets faster queries or new aggregate tables
## Journeys
| name | what the user does | timer stops when |
| --- | --- | --- |
| `logs_open_24h` | opens Logs (default Last 24 Hours) | request rows are rendered and the refresh control is idle |
| `logs_last_7_days` | switches the time range to Last 7 Days | same |
| `logs_next_page` | clicks the next page control | same |
| `logs_filter_key_hash` | filters by the seeded key hash and applies | same |
| `logs_filter_key_alias` | picks the first key alias in the filter drawer and applies | same |
| `logs_filter_team` | picks the first team in the filter drawer and applies | same |
| `logs_filter_model` | picks the first model in the filter drawer and applies | same |
| `usage_open` | opens Usage | summary tiles show dollar values with no skeletons left |
| `usage_last_30_days` | picks Last 30 days in the Usage date picker | same |
| `usage_team_view` | switches the Usage view to Team Usage | same |
| `usage_customer_view` | switches the Usage view to Customer Usage | same |
| `usage_model_activity_30_days` | on the Model Activity tab, picks Last 30 days | same |
| `cache_open` | opens Caching | the Cache Hits tile is visible and the Cache Hits vs API Requests chart has bars |
| `models_open` | opens Models + Endpoints | model rows are rendered and the loading state is gone |
| `keys_open` | opens Virtual Keys | key rows are rendered and the loading state is gone |
The mix follows what customers report as slow: the Logs page on `LiteLLM_SpendLogs`, the Usage page on the `LiteLLM_Daily*Spend` rollups (30 day ranges, Customer and Model views), and the Models and Keys pages.
## Seeded rows
The snapshot carries the SpendLogs rows and their Daily* rollups. `setup/ui_query_fixture.py` checks that `LiteLLM_DailyUserSpend` and `LiteLLM_DailyTeamSpend` have rows in the week before the anchor, then fills the tables the snapshot does not have so the Keys, Models and filter journeys see realistic counts: 100 teams (`perf-team-N`), 2,000 users (`perf-user-N`), 5,000 keys whose token hashes and `perf-key-N` aliases match the seeded log rows, and 200 DB models pointed at the perf mock provider. Every insert is `ON CONFLICT DO NOTHING`, so reruns on the same restore are no-ops. Nothing here runs outside the disposable restore
Each journey is measured once cold, then `E2E_UI_JOURNEY_REPEATS` more times (default 9) from the same starting state, inside one logged-in tab whose clock is pinned to the seed anchor so relative windows land on seeded rows. The browser timezone is UTC to match the seed
## Safety
The seeded source is never the target. `E2E_UI_PERF_DISPOSABLE=1`, `E2E_RESET_SPEND_LOGS=0` and a hostname matching `db-perf-perf-<id>.litellm.svc` are all required, and in-flight SQL is cancelled only on that disposable restore between journeys (see `setup/ui_query_isolation.py`)
## Running
The componentized perf stack serves the dashboard behind the same router as the proxy, so `LITELLM_PROXY_URL` covers `/ui`. Set `E2E_UI_BASE_URL` only if the UI lives elsewhere. The admin login uses `E2E_UI_USERNAME` (default `admin`) and `E2E_UI_PASSWORD` (default the master key)
```bash
export E2E_PERF_TESTS=1 E2E_UI_PERF_DISPOSABLE=1 E2E_RESET_SPEND_LOGS=0
export DATABASE_URL=postgresql://...@db-perf-perf-<id>.litellm.svc:5432/litellm
export E2E_DATABASE_METRICS_URLS=http://<backend-pod>:4000/metrics
export E2E_ARTIFACT_DIR=/tmp/e2e-artifacts
uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ARTIFACT_DIR/junit.xml" -q
```
`E2E_UI_JOURNEY_TIMEOUT_SECONDS` (default 120) bounds one journey. A journey that never settles is recorded as a `timeout` sample and a journey whose page rendered empty or threw in the browser as a `ui` sample. The test fails when any sample failed, or when the p50 or p90 across all samples is over the journey's `Ceiling` in `UI_JOURNEY_LIMITS` (`setup/ui_journeys.py`, 10s/20s for Logs journeys, 20s/40s for Usage and Caching, 5s/10s for the Models and Keys tables, generous until the first 1TB run sets real numbers). The artifacts are written before the assertion so a failed run still ships its samples
## Artifacts
Each JUnit testcase `test_ui_journey_latency[<name>]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-<name>.json` holds every sample with its on-screen row count and `ui-isolation-<name>.jsonl` the SQL isolation evidence
The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot
To measure a released version on the same snapshot, create the litellm-perf build with `perf_gateway_version=<tag>` (for example `v1.106.1`) next to `litellm_sha`. The gateway, backend and the restore's schema upgrade then come from `ghcr.io/berriai/litellm-{gateway,backend,migrations}:<tag>`, where the migrations image runs `prisma migrate deploy` and builds the request-log indexes the way a customer upgrade does, while the runner and these journeys come from `litellm_sha`. The fixture skips its upgrade-worker wait when `_LiteLLMPerfUpgrade` is absent. The snapshot's schema is v1.105.1 and the restore only migrates forward, so older tags cannot be measured
Each sample in `ui-journey-<name>.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_<status>` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane.

View file

@ -0,0 +1,15 @@
"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions
route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route."""
from __future__ import annotations
from typing import Final
from .direct_vs_bridge import direct_vs_bridge
CASE: Final = direct_vs_bridge(
"chat_completions_2rps",
"chat_completions",
direct="gpt-6-luna",
bridge="gpt-6-luna-responses",
)

View file

@ -0,0 +1,31 @@
"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on
that provider's own API (direct) and half on a deployment it has to translate to another provider
API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as
`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the
translation's cost."""
from __future__ import annotations
from dataclasses import replace
from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey
from .mixed_size_requests_2rps import CASE as MIXED
def _scoped(key: VirtualKey, direct: str) -> VirtualKey:
return replace(key, models=(direct,)) if key.models else key
def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase:
return replace(
MIXED,
name=name,
teams=(
replace(MIXED.team("team-a"), models=(direct, bridge)),
replace(MIXED.team("team-b"), models=(bridge,)),
),
virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys),
model_groups=(direct, bridge),
traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}),
)

View file

@ -0,0 +1,50 @@
"""Where the gateway holds its event loop for 100 ms or more: the mixed case's callers, model groups, guardrails
and body mix (short chats, 300k to 1M-token contexts, tool calls, half streamed) at 1 rps for 120 requests against
a gateway running the stdlib asyncio loop in debug mode, which logs every loop step over 0.1 s with the task that
ran it. No latency, CPU or datastore ceilings: the one check is that no step over 0.1 s comes from a location that
is not on the allowlist below, and that the listed ones do not show up more often than they do today. A stream
that stops yielding (the 46925 fix), a provider handler that re-walks a 1M-token body on the loop (46967), or a
new hook that deep-copies the request inline all land here as a new row. Runs on its own stack: the gateway needs
`PYTHONASYNCIODEBUG=1` and `LITELLM_EVENT_LOOP=asyncio` (uvloop reports nothing and crashed under debug locally),
and the runner `E2E_PERF_EVENT_LOOP=1`, so the nightly build, which sets neither, skips it. The allowlist counts
are twice the highest count any of ten back-to-back local runs showed on that row (rows that showed nothing in
those ten but did in the earlier before/after legs for 46925 and 46967, or in the first scheduled-lane run, get 2),
so each run sits at or under half of every cap and a listed location doubling in frequency is a failure, not a flake. The
`garbage collection` bucket is the gen-2 passes the collector logs in debug mode (0.6 to 0.7 s each, one every
minute or two of this traffic, plus the ramp while the heap fills on a local whole-file run), which asyncio would
otherwise charge to a random task; its count is the signal for a change that makes much more garbage."""
from __future__ import annotations
from dataclasses import replace
from typing import Final
from setup.event_loop import EventLoopStalls, KnownStall
from cases.mixed_size_requests_2rps import CASE as MIXED
CASE: Final = replace(
MIXED,
name="event_loop_stalls",
traffic=replace(MIXED.traffic, rps=1.0, total_requests=120, concurrency=10),
limits=(),
event_loop=EventLoopStalls(
allowlist=(
KnownStall("garbage collection", max_count=4),
KnownStall("LifespanOn.main", max_count=4),
KnownStall("RequestResponseCycle.run_asgi", max_count=2),
KnownStall("with_proxy_compaction_executor", max_count=2),
KnownStall("Logging.async_success_handler", max_count=2),
KnownStall("Logging.dispatch_success_handlers", max_count=2),
KnownStall("_TrackedPrismaEngine", max_count=2),
KnownStall("prisma/actions.py", max_count=2),
KnownStall("SpendCounterReseed", max_count=2),
KnownStall("Subscriber._run", max_count=2),
KnownStall("RedisBatch.flush", max_count=2),
KnownStall("run_coroutine_job", max_count=2),
KnownStall("user_api_key_auth.py", max_count=2),
KnownStall("route_streaming_logging_to_handler", max_count=2),
KnownStall("TaskHandle._run_coro", max_count=2),
),
),
)

View file

@ -0,0 +1,16 @@
"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route,
`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the
Bedrock mock's Converse route."""
from __future__ import annotations
from typing import Final
from .direct_vs_bridge import direct_vs_bridge
CASE: Final = direct_vs_bridge(
"messages_2rps",
"messages",
direct="claude-sonnet-5",
bridge="claude-sonnet-5-bedrock-converse",
)

View file

@ -62,13 +62,24 @@ CASE: Final = PerformanceTestCase(
),
limits=(
Limit("unexpected_status_count", max=3),
Limit("gateway_added_seconds", percentile=50, max=0.58),
Limit("gateway_added_seconds", percentile=99, max=6.1),
Limit("time_to_first_token_seconds", percentile=50, max=2.7),
Limit("time_to_first_token_seconds", percentile=99, max=8.8),
Limit("gateway_added_seconds", percentile=50, max=0.29),
Limit("gateway_added_seconds", percentile=99, max=2.7),
Limit("litellm_overhead_seconds", percentile=50, max=0.052),
Limit("litellm_overhead_seconds", percentile=99, max=1.8),
Limit("litellm_response_seconds", percentile=50, max=0.93),
Limit("litellm_response_seconds", percentile=99, max=2.7),
Limit("prometheus_redis_latency_mean_seconds", max=0.002),
Limit("postgres_active_sessions", max=0.0094),
Limit("postgres_slowest_statement_seconds", max=0.042),
Limit("postgres_peak_connections", max=23),
Limit("time_to_first_token_seconds", percentile=50, max=1.56),
Limit("time_to_first_token_seconds", percentile=99, max=4.5),
Limit("memory_growth_megabytes", max=600),
Limit("cpu_seconds_per_request", max=1.26),
Limit("postgres_calls_per_request", max=3.5),
Limit("redis_calls_per_request", max=21.0),
Limit("cpu_seconds_per_request", max=0.57),
Limit("postgres_calls_per_request", max=2.64),
Limit("postgres_max_statement_per_request", max=0.43),
Limit("postgres_max_background_statement_per_request", max=0.86),
Limit("redis_calls_per_request", max=15.4),
),
background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
)

View file

@ -0,0 +1,24 @@
"""The mixed-size traffic stepped up until the pod stops keeping up: the same callers, guardrails, mocks and body
mix (short chats, 300k to 1M-token contexts, tool calls, half streamed) as `mixed_size_requests_2rps`, 2 rps more
every 90 s from 2 to 14 on the same one 4-worker pod. Where the 2 rps case prices one big body on its own, this one
reports `sustained_rps`, the last step the pod held with big bodies overlapping in every worker, so a handler that
re-walks a 1M-token body on the event loop or a worker that holds several of them in memory at once moves the knee
down and fails the floor."""
from __future__ import annotations
from dataclasses import replace
from typing import Final
from setup.performance_test_case import Limit, Ramp, ramped
from cases.mixed_size_requests_2rps import CASE as MIXED
RAMP: Final = Ramp(first_rps=2.0, last_rps=14.0, step_rps=2.0, step_seconds=90.0, held_p99_seconds=5.0)
CASE: Final = replace(
MIXED,
name="mixed_size_requests_ramp",
traffic=ramped(MIXED.traffic, RAMP, seed=9, concurrency=300),
limits=(Limit("sustained_rps", min=8),),
)

View file

@ -0,0 +1,15 @@
"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route,
`claude-sonnet-5` is bridged to the Anthropic mock's Messages route."""
from __future__ import annotations
from typing import Final
from .direct_vs_bridge import direct_vs_bridge
CASE: Final = direct_vs_bridge(
"responses_2rps",
"responses",
direct="gpt-6-luna",
bridge="claude-sonnet-5",
)

View file

@ -64,11 +64,21 @@ CASE: Final = PerformanceTestCase(
),
limits=(
Limit("unexpected_status_count", max=3),
Limit("gateway_added_seconds", percentile=50, max=0.29),
Limit("time_to_first_token_seconds", percentile=99, max=3.7),
Limit("gateway_added_seconds", percentile=50, max=0.165),
Limit("litellm_overhead_seconds", percentile=50, max=0.006),
Limit("litellm_overhead_seconds", percentile=99, max=0.35),
Limit("litellm_response_seconds", percentile=50, max=0.91),
Limit("litellm_response_seconds", percentile=99, max=1.6),
Limit("prometheus_redis_latency_mean_seconds", max=0.0026),
Limit("postgres_active_sessions", max=0.0052),
Limit("postgres_slowest_statement_seconds", max=0.011),
Limit("postgres_peak_connections", max=23),
Limit("time_to_first_token_seconds", percentile=99, max=2.05),
Limit("memory_growth_megabytes", max=100),
Limit("postgres_calls_per_request", max=1.8),
Limit("redis_calls_per_request", max=18),
Limit("cpu_seconds_per_request", max=0.29),
Limit("postgres_calls_per_request", max=1.31),
Limit("postgres_max_statement_per_request", max=0.27),
Limit("postgres_max_background_statement_per_request", max=0.9),
Limit("redis_calls_per_request", max=13.8),
Limit("cpu_seconds_per_request", max=0.15),
),
)

View file

@ -0,0 +1,24 @@
"""The small-bodies traffic stepped up until the pod stops keeping up: the same callers, guardrails, mocks and
2.5k to 4k-token bodies as `small_bodies_5rps`, 10 rps more every 90 s from 10 to 70 on the same one 4-worker pod.
The 5 rps case prices one request with the pod idle; this one reports `sustained_rps`, the last step the pod
held (gateway added p99 under 5 s, the queue still draining), so a change that moves the knee down, a lock held
across an await, a pool that fills, a flush that stalls the loop while dozens of requests are in flight, fails the
floor instead of hiding under a fixed rate the pod still manages."""
from __future__ import annotations
from dataclasses import replace
from typing import Final
from setup.performance_test_case import Limit, Ramp, ramped
from cases.small_bodies_5rps import CASE as SMALL
RAMP: Final = Ramp(first_rps=10.0, last_rps=70.0, step_rps=10.0, step_seconds=90.0, held_p99_seconds=5.0)
CASE: Final = replace(
SMALL,
name="small_bodies_ramp",
traffic=ramped(SMALL.traffic, RAMP, seed=13, concurrency=400),
limits=(Limit("sustained_rps", min=20),),
)

View file

@ -1,7 +1,10 @@
"""Postgres and Redis work the gateway did during the traffic phase, split into what a request waits on and
what runs after the response. Both come from the servers' own accounting, so nothing here depends on how the
gateway talks to them: Postgres statement counts from `pg_stat_statements` (read before and after the run) and
Redis commands from a `MONITOR` connection held open for the run. `DATABASE_URL` and `REDIS_HOST` are the
Redis commands from a `MONITOR` connection held open for the run, Postgres execution time per statement from the
same `pg_stat_statements` rows (`postgres_active_sessions`, the average-active-sessions number RDS Performance
Insights calls DB load, and the slowest statement's mean) and the connection count from `pg_stat_activity` sampled
once a second (`postgres_peak_connections`). `DATABASE_URL` and `REDIS_HOST` are the
runner's own env, as in the other suites; without them the counters are None and any Limit on them reports as
missing. `E2E_DATABASE_ADMIN_URL`, when set, is used once to create the `pg_stat_statements` extension. Only the
runtime user's own statements are read: other users' rows come back with a NULL queryid and hidden text."""
@ -19,6 +22,8 @@ from typing import TYPE_CHECKING, Final, Literal
from pydantic import TypeAdapter
from setup.series import Recorder, Series
if TYPE_CHECKING:
import redis
@ -26,13 +31,18 @@ Bucket = Literal["awaited", "background", "plumbing", "unclassified"]
_COMMITS_QUERY: Final = "SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()"
_STATEMENTS_QUERY: Final = (
"SELECT queryid, calls, query FROM pg_stat_statements "
"SELECT queryid, calls, query, total_exec_time FROM pg_stat_statements "
"WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) "
"AND userid = (SELECT usesysid FROM pg_user WHERE usename = current_user) AND queryid IS NOT NULL"
)
_CREATE_EXTENSION: Final = "CREATE EXTENSION IF NOT EXISTS pg_stat_statements"
_CONNECTIONS_QUERY: Final = (
"SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() "
"AND backend_type = 'client backend' AND pid <> pg_backend_pid()"
)
_CONNECTIONS_SAMPLE_SECONDS: Final = 1.0
_COMMITS_ROW: Final = TypeAdapter(tuple[int])
_STATEMENT_ROWS: Final = TypeAdapter(list[tuple[int, int, str]])
_STATEMENT_ROWS: Final = TypeAdapter(list[tuple[int, int, str, float]])
_INFO: Final = TypeAdapter(dict[str, object])
_INT: Final = TypeAdapter(int)
_MONITOR_LINE: Final = TypeAdapter(dict[str, object])
@ -64,6 +74,11 @@ class StatementCalls:
queryid: int
query: str
calls: int
total_exec_ms: float = 0.0
@property
def mean_exec_seconds(self) -> float | None:
return self.total_exec_ms / 1000 / self.calls if self.calls > 0 else None
@property
def bucket(self) -> Bucket:
@ -76,6 +91,11 @@ class StatementCalls:
return "plumbing"
return "awaited" if verb == "SELECT" and tables[0] in _AWAITED_TABLES else "background"
@property
def table(self) -> str:
tables: Final = _TABLE.findall(self.query)
return tables[0] if tables else ""
@property
def summary(self) -> str:
return re.sub(r"\s+", " ", self.query)[:110]
@ -131,16 +151,95 @@ def statement_deltas(before: DbCounters | None, after: DbCounters | None) -> tup
"""Calls per statement during the run: `after` minus `before` by queryid, new statements whole."""
if before is None or after is None or before.statements is None or after.statements is None:
return None
earlier: Final = {statement.queryid: statement.calls for statement in before.statements}
earlier: Final = {statement.queryid: statement for statement in before.statements}
return tuple(
StatementCalls(
queryid=statement.queryid, query=statement.query, calls=statement.calls - earlier.get(statement.queryid, 0)
)
_statement_delta(statement, earlier.get(statement.queryid))
for statement in after.statements
if statement.calls > earlier.get(statement.queryid, 0)
if statement.calls > (0 if statement.queryid not in earlier else earlier[statement.queryid].calls)
)
def _statement_delta(statement: StatementCalls, earlier: StatementCalls | None) -> StatementCalls:
return StatementCalls(
queryid=statement.queryid,
query=statement.query,
calls=statement.calls - (earlier.calls if earlier is not None else 0),
total_exec_ms=statement.total_exec_ms - (earlier.total_exec_ms if earlier is not None else 0.0),
)
def active_sessions(statements: tuple[StatementCalls, ...] | None, wall_seconds: float) -> float | None:
"""Seconds Postgres spent executing the run's statements per second of wall time, the average active
sessions figure RDS Performance Insights reports as DB load."""
if statements is None or wall_seconds <= 0:
return None
return sum(statement.total_exec_ms for statement in statements) / 1000 / wall_seconds
def slowest_statement(statements: tuple[StatementCalls, ...] | None) -> StatementCalls | None:
"""The run's statement with the highest mean execution time."""
timed: Final = tuple(statement for statement in statements or () if statement.mean_exec_seconds is not None)
return max(timed, key=lambda statement: statement.mean_exec_seconds or 0.0, default=None)
class ConnectionsSampler:
"""`pg_stat_activity` client backends on the gateway's database once a second on a background thread, minus
this sampler's own connection, so the peak is what the gateway (and the test's own reads) held open.
`series` keeps every count with its time for a ramped run's per-step peaks."""
def __init__(self, url: str) -> None:
self._url: Final = url
self._stop: Final = threading.Event()
self._thread: Final = threading.Thread(target=self._run, name="postgres-connections", daemon=True)
self._counts: Final[Recorder[int]] = Recorder()
@property
def peak(self) -> int | None:
return max((sample.value for sample in self._counts.series.samples), default=None)
@property
def series(self) -> Series[int]:
return self._counts.series
def start(self) -> None:
self._thread.start()
def stop(self) -> None:
self._stop.set()
self._thread.join(timeout=10)
def _run(self) -> None:
import psycopg
with psycopg.connect(self._url, autocommit=True) as conn:
while not self._stop.is_set():
(count,) = _COMMITS_ROW.validate_python(conn.execute(_CONNECTIONS_QUERY).fetchone())
self._counts.record(count)
_ = self._stop.wait(_CONNECTIONS_SAMPLE_SECONDS)
class NoConnectionsSampler:
peak: None = None
series: None = None
def start(self) -> None:
return None
def stop(self) -> None:
return None
@contextmanager
def connections_sampler() -> Generator[ConnectionsSampler | NoConnectionsSampler]:
url: Final = os.environ.get("DATABASE_URL")
sampler: Final = ConnectionsSampler(url) if url is not None else NoConnectionsSampler()
sampler.start()
try:
yield sampler
finally:
sampler.stop()
def calls_in(bucket: Bucket, items: tuple[StatementCalls, ...] | tuple[RedisCommand, ...] | None) -> int | None:
if items is None:
return None
@ -166,7 +265,10 @@ def read_db_counters() -> DbCounters | None:
return DbCounters(commits=commits, statements=None)
return DbCounters(
commits=commits,
statements=tuple(StatementCalls(queryid=queryid, query=query, calls=calls) for queryid, calls, query in rows),
statements=tuple(
StatementCalls(queryid=queryid, query=query, calls=calls, total_exec_ms=total_ms)
for queryid, calls, query, total_ms in rows
),
)

View file

@ -0,0 +1,323 @@
"""Event-loop holds, read from the gateway's own asyncio debug output. A gateway started with
`PYTHONASYNCIODEBUG=1` on the stdlib loop (`LITELLM_EVENT_LOOP=asyncio`, uvloop never reports them) logs every
callback or task step over `slow_callback_duration` (0.1 s) as
`Executing <Task ... coro=<name() running at file:line> ...> took X seconds`. A case with `event_loop` set reads
the gateway log for its traffic window, groups those lines by coroutine and location, and fails on a location that
is not on its allowlist of known holds or shows up more often than the allowlist gives it. asyncio names the task,
not the frame that held the loop, so every request handler lands under `RequestResponseCycle.run_asgi`; the
allowlist caps how often a known location may appear instead of ignoring it. The log comes from the gateway pods
(`pods/log` with the runner's service account, pods picked by `E2E_PERF_GATEWAY_POD_SELECTOR`) or, on a local run,
from the whole file named by `E2E_PERF_GATEWAY_LOG_FILE` (no time window, so startup holds are in it too).
A garbage collection pass pauses the loop too, and asyncio charges it to whatever task was running, so the gateway
logs every pass over the same threshold (`Garbage collection gen=2 collected=N took X seconds`, debug mode only);
a hold that such a pass accounts for is reported under `garbage collection` instead of the task it landed on."""
from __future__ import annotations
import json
import os
import re
from collections.abc import Iterable
from dataclasses import asdict, dataclass, replace
from datetime import datetime
from itertools import groupby
from pathlib import Path
from typing import Final
from urllib.parse import quote
from e2e_http import AuthHeaders, NetworkError, Success, get_external, get_text_external
from pydantic import BaseModel
from setup.podcpu import SELECTOR_ENV, in_cluster
OPT_IN_ENV: Final = "E2E_PERF_EVENT_LOOP"
LOG_FILE_ENV: Final = "E2E_PERF_GATEWAY_LOG_FILE"
_API: Final = "https://kubernetes.default.svc/api/v1/namespaces"
_LINE: Final = re.compile(r"Executing <(?P<desc>.*?)> took (?P<seconds>[0-9.]+) seconds")
_CORO: Final = re.compile(r"coro=<(?P<name>[^ ]+)")
_AT: Final = re.compile(r" at (?P<where>[^ >]+:\d+)")
_GC_LINE: Final = re.compile(
r"Garbage collection gen=(?P<generation>\d+) collected=\d+ took (?P<seconds>[0-9.]+) seconds"
)
GC_BUCKET: Final = "garbage collection"
@dataclass(frozen=True, slots=True)
class KnownStall:
"""A hold the gateway has today: `where` is a substring of the coroutine name or its file:line, `max_count` the
most times it may show up in one run before the case fails."""
where: str
max_count: int
@dataclass(frozen=True, slots=True)
class EventLoopStalls:
allowlist: tuple[KnownStall, ...]
max_hold_seconds: float = 0.1
@dataclass(frozen=True, slots=True)
class GcPass:
generation: int
seconds: float
@dataclass(frozen=True, slots=True)
class Stall:
coroutine: str
where: str
seconds: float
gc_seconds: float = 0.0
gc_generation: int | None = None
@dataclass(frozen=True, slots=True)
class StallRow:
coroutine: str
where: str
count: int
total_seconds: float
max_seconds: float
allowed_by: KnownStall | None
"""Every hold in the row, longest first."""
holds: tuple[float, ...] = ()
@property
def offending(self) -> bool:
return self.allowed_by is None or self.count > self.allowed_by.max_count
@property
def over_allowance(self) -> int:
return self.count if self.allowed_by is None else max(0, self.count - self.allowed_by.max_count)
@property
def note(self) -> str:
allowed: Final = (
f"{self.count} of {self.allowed_by.max_count} allowed" if self.allowed_by else "not on the allowlist"
)
return f"{allowed}, max {self.max_seconds:.2f} s"
def _short(path: str) -> str:
if "site-packages/" in path:
return path.rsplit("site-packages/", 1)[1]
if "/litellm/" in path:
return "litellm/" + path.rsplit("/litellm/", 1)[1]
return path
def _parse_line(line: str) -> Stall | None:
match: Final = _LINE.search(line)
if match is None:
return None
desc: Final = match.group("desc")
coro: Final = _CORO.search(desc)
at: Final = _AT.search(desc)
return Stall(
coroutine=coro.group("name") if coro else desc[:80],
where=_short(at.group("where")) if at else "",
seconds=float(match.group("seconds")),
)
def _parse_event(line: str) -> Stall | GcPass | None:
match: Final = _GC_LINE.search(line)
if match is None:
return _parse_line(line)
return GcPass(generation=int(match.group("generation")), seconds=float(match.group("seconds")))
def _with_passes(stall: Stall, passes: tuple[GcPass, ...]) -> Stall:
fitting: Final = tuple(gc_pass for gc_pass in passes if gc_pass.seconds <= stall.seconds)
if not fitting:
return stall
longest: Final = max(fitting, key=lambda gc_pass: gc_pass.seconds)
return replace(stall, gc_seconds=longest.seconds, gc_generation=longest.generation)
def parse_stalls(lines: Iterable[str]) -> tuple[Stall, ...]:
"""Holds in log order, each carrying the collector passes logged since the previous hold: a pass runs inside the
task asyncio then blames and is only logged when it is itself over the threshold, so its hold line follows it."""
events: Final = tuple(event for event in map(_parse_event, lines) if event is not None)
holds: Final = tuple((index, event) for index, event in enumerate(events) if isinstance(event, Stall))
return tuple(
_with_passes(stall, tuple(event for event in events[previous + 1 : index] if isinstance(event, GcPass)))
for previous, (index, stall) in zip((-1, *(index for index, _ in holds)), holds)
)
def _reported(spec: EventLoopStalls, stall: Stall) -> Stall:
"""A hold the collector accounts for, what is left under the threshold, is the collector's, not the task's."""
if stall.gc_generation is None:
return stall
if stall.seconds - stall.gc_seconds >= spec.max_hold_seconds:
return stall
return Stall(coroutine=GC_BUCKET, where=f"gen={stall.gc_generation}", seconds=stall.seconds)
def _allowed_by(spec: EventLoopStalls, coroutine: str, where: str) -> KnownStall | None:
return next((known for known in spec.allowlist if known.where in coroutine or known.where in where), None)
def summarize(spec: EventLoopStalls, stalls: Iterable[Stall]) -> tuple[StallRow, ...]:
"""One row per (coroutine, location) over the hold threshold, worst total first."""
reported: Final = tuple(_reported(spec, stall) for stall in stalls)
over: Final = sorted(
(stall for stall in reported if stall.seconds >= spec.max_hold_seconds), key=lambda s: (s.coroutine, s.where)
)
rows: Final = tuple(
StallRow(
coroutine=coroutine,
where=where,
count=len(group),
total_seconds=sum(stall.seconds for stall in group),
max_seconds=max(stall.seconds for stall in group),
allowed_by=_allowed_by(spec, coroutine, where),
holds=tuple(sorted((stall.seconds for stall in group), reverse=True)),
)
for (coroutine, where), group in (
(key, tuple(items)) for key, items in groupby(over, key=lambda s: (s.coroutine, s.where))
)
)
return tuple(sorted(rows, key=lambda row: -row.total_seconds))
def offenders(rows: Iterable[StallRow]) -> tuple[str, ...]:
return tuple(
f"{row.coroutine} {row.where}: {row.count}x, max {row.max_seconds:.2f} s"
+ (
f" (allowlist {row.allowed_by.where} permits {row.allowed_by.max_count})"
if row.allowed_by
else " (not on the allowlist)"
)
for row in rows
if row.offending
)
TOP_HOLDS: Final = 10
@dataclass(frozen=True, slots=True)
class Slice:
"""One `_breakdown` row in the shape the other metrics use (label, value, share) plus the allowlist verdict the
dashboard shows under the label and colours by. `share` is None where the rows are not parts of a total."""
label: str
value: float
share: float | None
note: str | None
over: bool
def _label(row: StallRow) -> str:
return f"{row.coroutine} {row.where}".strip()
def _slice(row: StallRow, value: float, total: float) -> Slice:
return Slice(label=_label(row), value=value, share=value / total, note=row.note, over=row.offending)
def _breakdown(name: str, slices: Iterable[Slice]) -> tuple[tuple[str, str], ...]:
rows: Final = tuple(asdict(row) for row in slices)
return ((f"perf_event_loop_{name}_breakdown", json.dumps(rows)),) if rows else ()
def _longest_holds(table: tuple[StallRow, ...]) -> tuple[Slice, ...]:
holds: Final = sorted(((seconds, row) for row in table for seconds in row.holds), key=lambda hold: -hold[0])
return tuple(
Slice(label=_label(row), value=seconds, share=None, note=None, over=row.offending)
for seconds, row in holds[:TOP_HOLDS]
)
def junit_properties(spec: EventLoopStalls, rows: Iterable[StallRow]) -> tuple[tuple[str, float | str], ...]:
"""The stall table as `perf_event_loop_*` properties for the dashboard: every hold over the threshold with the
allowlist's total as its limit, the holds past their allowance (limit 0, the case's one assertion), the longest
and the summed hold, each with a `_breakdown` the dashboard opens on click: holds and seconds per location, the
locations past their cap by how far over they are, and the ten longest single holds with where they landed."""
table: Final = tuple(rows)
holds: Final = sum(row.count for row in table)
offending: Final = sum(row.over_allowance for row in table)
total_seconds: Final = sum(row.total_seconds for row in table)
return (
("perf_event_loop_holds", holds),
("perf_event_loop_holds_limit", sum(known.max_count for known in spec.allowlist)),
("perf_event_loop_offending_holds", offending),
("perf_event_loop_offending_holds_limit", 0),
("perf_event_loop_hold_seconds_max", max((row.max_seconds for row in table), default=0.0)),
("perf_event_loop_hold_seconds_total", total_seconds),
*_breakdown("holds", (_slice(row, row.count, holds) for row in table)),
*_breakdown(
"offending_holds",
(_slice(row, row.over_allowance, offending) for row in table if row.over_allowance),
),
*_breakdown("hold_seconds_total", (_slice(row, row.total_seconds, total_seconds) for row in table)),
*_breakdown("hold_seconds_max", _longest_holds(table)),
)
def render_stalls(spec: EventLoopStalls, rows: Iterable[StallRow]) -> str:
header: Final = (
f" event-loop holds >= {spec.max_hold_seconds:.2f} s in the gateway log (count total_s max_s where)"
)
body: Final = tuple(
f" {'FAIL' if row.offending else 'ok '} {row.count:5d} {row.total_seconds:8.2f} {row.max_seconds:6.2f} "
f"{row.coroutine} {row.where}"
for row in rows
)
return "\n".join((header, *body)) if body else header + "\n none"
class _PodName(BaseModel):
name: str
class _Pod(BaseModel):
metadata: _PodName
class _PodList(BaseModel):
items: list[_Pod]
def _pod_log_lines(url: str, headers: AuthHeaders, ca_bundle: str) -> tuple[str, ...]:
text: Final = get_text_external(url, headers=headers, timeout=60.0, ca_bundle=ca_bundle)
if isinstance(text, NetworkError):
raise RuntimeError(f"gateway pod log unreadable: {text.message}")
return tuple(text.splitlines())
def _pod_logs(since: datetime) -> tuple[str, ...] | None:
cluster: Final = in_cluster()
selector: Final = os.environ.get(SELECTOR_ENV, "")
if cluster is None or not selector:
return None
token, ca_bundle, namespace = cluster
headers: Final = AuthHeaders(authorization=f"Bearer {token}")
pods: Final = get_external(
f"{_API}/{namespace}/pods?labelSelector={quote(selector)}",
response_type=_PodList,
headers=headers,
timeout=60.0,
ca_bundle=ca_bundle,
)
if not isinstance(pods, Success):
raise RuntimeError(f"gateway pod list unreadable: {pods}")
since_param: Final = quote(since.strftime("%Y-%m-%dT%H:%M:%SZ"))
return tuple(
line
for pod in pods.data.items
for line in _pod_log_lines(
f"{_API}/{namespace}/pods/{pod.metadata.name}/log?sinceTime={since_param}", headers, ca_bundle
)
)
def read_gateway_log(since: datetime) -> tuple[str, ...] | None:
"""Gateway log lines for the traffic window, or None when neither a pod nor a local file is reachable."""
local: Final = os.environ.get(LOG_FILE_ENV, "")
if local:
return tuple(Path(local).read_text(errors="replace").splitlines())
return _pod_logs(since)

View file

@ -4,7 +4,8 @@ time to headers, time to first byte, time to first token (streams), total time,
and whether it matched. Runs in its own process (`setup/run.py` starts it), never in the gateway's.
Reads `PERF_PLAN_FILE` (JSON list of wire requests), writes `PERF_RECORDS_FILE` (one JSON object per
line), paces to `PERF_RPS` over `PERF_USERS` users, and quits once the last request is recorded.
line), paces to `PERF_RPS` over `PERF_USERS` users (or, when the plan carries `send_at_s`, sends each
request at its scheduled second), and quits once the last request is recorded.
"""
from __future__ import annotations
@ -19,9 +20,10 @@ from typing import Final, cast
import gevent
from locust import FastHttpUser, task
from locust.contrib.fasthttp import FastResponse, ResponseContextManager
from setup.wire import Endpoint, WireRecord, WireRequest, request_body
from pydantic import TypeAdapter
from setup.wire import Endpoint, PlanProgress, WireRecord, WireRequest, request_body, response_id
_PLAN_ADAPTER: Final = TypeAdapter(list[WireRequest])
_JSON: Final = TypeAdapter[object](object)
_PLAN: Final = _PLAN_ADAPTER.validate_json(Path(os.environ["PERF_PLAN_FILE"]).read_bytes())
@ -33,7 +35,10 @@ _INTERVAL_S: Final = _USERS / _RPS
_JITTER: Final = 0.25
_RNG: Final = random.Random(_PLAN[0].marker if _PLAN else "empty-plan")
_RECORDS: Final = _RECORDS_PATH.open("a", encoding="utf-8")
_written = 0
_PROGRESS: Final = PlanProgress(len(_PLAN))
_SCHEDULED: Final = bool(_PLAN) and _PLAN[0].send_at_s is not None
_RUN_STARTED: Final = time.monotonic()
_STOP_FILE: Final = os.environ.get("PERF_STOP_FILE")
BLOCK_REASON: Final = "perf-test-block"
_TOKEN_MAX_AGE_S: Final = 240.0
_tokens: dict[str, tuple[str, float]] = {}
@ -135,6 +140,15 @@ def _header(response: FastResponse, name: str) -> str | None:
return value if isinstance(value, str) else None
def _ms_header(response: FastResponse | None, name: str) -> float | None:
"""A gateway timing header as a number; the gateway writes the literal `None` when it has no value."""
value: Final = _header(response, name) if response is not None else None
try:
return float(value) if value is not None else None
except ValueError:
return None
def _quit(user: FastHttpUser) -> None:
"""End the run once every record is written. Quitting from a separate greenlet: `runner.quit`
kills the user greenlets, and the user calling it would be killed mid-shutdown."""
@ -155,10 +169,9 @@ def _judge(item: WireRequest, status: int, body_text: str) -> tuple[bool, str |
def _record(record: WireRecord) -> None:
global _written
_ = _RECORDS.write(record.model_dump_json() + "\n")
_RECORDS.flush()
_written += 1
_PROGRESS.completed += 1
def _authorization(user: FastHttpUser, item: WireRequest) -> str:
@ -201,17 +214,21 @@ def _authorization(user: FastHttpUser, item: WireRequest) -> str:
class PlanUser(FastHttpUser):
"""One simulated client; all users share the plan and together hold the planned rate. Each user starts at
a random point of its first interval and jitters every later interval by up to `_JITTER` either way, so the
users never fire in lockstep; the mean interval is unchanged and `rps` stays the total rate."""
users never fire in lockstep; the mean interval is unchanged and `rps` stays the total rate. A scheduled
plan (a ramp) is paced by the plan instead: a user takes the next request and sleeps until its second."""
network_timeout = 120.0
network_timeout = float(os.environ.get("PERF_REQUEST_TIMEOUT", "120"))
connection_timeout = 30.0
def on_start(self) -> None:
self._last_run = 0.0
self._last_wait = 0.0
gevent.sleep(_RNG.uniform(0.0, _INTERVAL_S))
if not _SCHEDULED:
gevent.sleep(_RNG.uniform(0.0, _INTERVAL_S))
def wait_time(self) -> float:
if _SCHEDULED:
return 0.1
target: Final = _INTERVAL_S * _RNG.uniform(1.0 - _JITTER, 1.0 + _JITTER)
now: Final = time.monotonic()
run_s: Final = now - self._last_run - self._last_wait if self._last_run else 0.0
@ -221,11 +238,14 @@ class PlanUser(FastHttpUser):
@task
def send(self) -> None:
item: Final = next(_NEXT, None)
take: Final = _PROGRESS.take(stop=_STOP_FILE is not None and Path(_STOP_FILE).exists())
item: Final = next(_NEXT, None) if take else None
if item is None:
if _written >= len(_PLAN):
if _PROGRESS.drained:
_quit(self)
return
if item.send_at_s is not None:
gevent.sleep(max(0.0, _RUN_STARTED + item.send_at_s - time.monotonic()))
started_at: Final = time.time()
started: Final = time.perf_counter()
first_byte_s: float | None = None
@ -237,6 +257,7 @@ class PlanUser(FastHttpUser):
error: str | None = None
header_s = 0.0
response: FastResponse | None = None
identifier: str | None = None # rebind-ok: the ID arrives in a JSON body or the first SSE event
try:
headers: Final = {"Authorization": _authorization(self, item), "Content-Type": "application/json"}
context: ResponseContextManager
@ -265,6 +286,7 @@ class PlanUser(FastHttpUser):
buffer += line
events, buffer = _events(buffer)
for event in events:
identifier = identifier or response_id(event)
if first_token_s is None and token_in_event(item.endpoint, event):
first_token_s = now
text += delta_of_event(item.endpoint, event)
@ -272,7 +294,9 @@ class PlanUser(FastHttpUser):
body_text = context.text or ""
first_byte_s = time.perf_counter() - started
if status == 200:
text = text_of_json(item.endpoint, _as_dict(_JSON.validate_json(body_text)))
payload: Final = _as_dict(_JSON.validate_json(body_text))
identifier = response_id(payload)
text = text_of_json(item.endpoint, payload)
ok, error = _judge(item, status, body_text)
if ok:
context.success()
@ -302,7 +326,10 @@ class PlanUser(FastHttpUser):
mock_delay_s=float(delay_header) if delay_header else None,
call_id=_header(response, "x-litellm-call-id") if response is not None else None,
error=error,
response_id=identifier,
litellm_overhead_ms=_ms_header(response, "x-litellm-overhead-duration-ms"),
litellm_response_ms=_ms_header(response, "x-litellm-response-duration-ms"),
)
)
if _written >= len(_PLAN):
if _PROGRESS.drained:
_quit(self)

View file

@ -4,27 +4,35 @@ calls, the workers' memory) into the case's metrics, and the case's limits into
from __future__ import annotations
import hashlib
import json
import statistics
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass
from itertools import chain
from typing import Final
from collections import Counter, defaultdict
from collections.abc import Iterable, Mapping
from dataclasses import asdict, dataclass
from itertools import chain, groupby
from types import MappingProxyType
from typing import Final, Literal
from e2e_http import NoBody, get_external, post_external, unwrap
from memory_readings import RssCapture, RssReading, WorkerKey
from pydantic import BaseModel
from typing_extensions import ReadOnly, TypedDict
from setup.datastore import (
DbCounters,
RedisCommand,
RedisCounters,
StatementCalls,
active_sessions,
calls_in,
slowest_statement,
statement_deltas,
)
from setup.mocks.reply_common import Provider
from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase
from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase, Ramp
from setup.ramp import StepReading, render_steps, sustained_rps
from setup.series import Series
from setup.telemetry import GatewayTelemetry
from setup.wire import WireRecord
@ -115,6 +123,20 @@ class Observation:
redis_after: RedisCounters | None = None
redis_commands: tuple[RedisCommand, ...] | None = None
gateway_cpu_seconds: float | None = None
telemetry_before: GatewayTelemetry | None = None
telemetry_after: GatewayTelemetry | None = None
peak_connections: int | None = None
ramp: Ramp | None = None
cpu_series: Series[float] | None = None
connection_series: Series[int] | None = None
telemetry_series: Series[GatewayTelemetry] | None = None
@property
def telemetry(self) -> GatewayTelemetry | None:
"""What the gateway's `/metrics` counted during the run."""
if self.telemetry_before is None or self.telemetry_after is None:
return None
return self.telemetry_after.minus(self.telemetry_before)
def per_request(self, before: int | None, after: int | None) -> float | None:
if before is None or after is None or not self.records:
@ -128,6 +150,21 @@ class Observation:
def statements(self) -> tuple[StatementCalls, ...] | None:
return statement_deltas(self.db_before, self.db_after)
@property
def busiest_awaited_statement(self) -> StatementCalls | None:
"""The awaited statement with the most calls: a lookup added to every request shows up here at
1.0 per request while cache refreshes stay fractional, which the awaited total cannot tell apart."""
awaited: Final = tuple(item for item in self.statements or () if item.bucket == "awaited")
return max(awaited, key=_item_count, default=None)
def busiest_background_statement(self, exempt_tables: frozenset[str]) -> StatementCalls | None:
"""The background statement with the most calls, skipping the case's per-flush exemptions: a rollup
written once per request reads 1.0 here while batched writers stay below it."""
background: Final = tuple(
item for item in self.statements or () if item.bucket == "background" and item.table not in exempt_tables
)
return max(background, key=_item_count, default=None)
@property
def retried_markers(self) -> frozenset[str]:
counts: Final = Counter(seen.marker for seen in self.mock_requests if seen.marker is not None)
@ -156,21 +193,42 @@ class Observation:
"""Requests the plan expected to succeed that came back with any other status (0 is a client timeout)."""
return sum(1 for record in self.records if not record.blocked and record.status != 200)
def latency_pairs(self, metric: LatencyMetric) -> list[tuple[WireRecord, float]]:
"""Each first-attempt request with its value of one latency metric. `gateway_added_seconds` is the
seconds the gateway added (total time minus the mock's own delay for a non-streamed request, time to the
first token minus the mock's delay for a stream), `time_to_first_token_seconds` is the client's wait for
the first token with the mock delay included, and the two `litellm_*` metrics are the gateway's own
`x-litellm-overhead-duration-ms` and `x-litellm-response-duration-ms` headers in seconds."""
if metric == "gateway_added_seconds":
non_stream: Final = [(record, record.total_s - delay) for record, delay in self._clean(stream=False)]
stream: Final = [
(record, record.first_token_s - delay)
for record, delay in self._clean(stream=True)
if record.first_token_s is not None
]
return non_stream + stream
if metric == "time_to_first_token_seconds":
return [(r, r.first_token_s) for r, _ in self._clean(stream=True) if r.first_token_s is not None]
if metric == "litellm_overhead_seconds":
return [
(r, r.litellm_overhead_ms / 1000) for r in self._first_attempts() if r.litellm_overhead_ms is not None
]
return [(r, r.litellm_response_ms / 1000) for r in self._first_attempts() if r.litellm_response_ms is not None]
def _first_attempts(self) -> list[WireRecord]:
return [record for record, _ in self._clean(stream=False)] + [record for record, _ in self._clean(stream=True)]
def gateway_added_seconds(self) -> list[float]:
"""Seconds the gateway added to each first-attempt request: total time minus the mock's own delay for a
non-streamed request, time to the first token minus the mock's delay to its first byte for a stream."""
non_stream: Final = [record.total_s - delay for record, delay in self._clean(stream=False)]
stream: Final = [
record.first_token_s - delay
for record, delay in self._clean(stream=True)
if record.first_token_s is not None
]
return non_stream + stream
return [value for _, value in self.latency_pairs("gateway_added_seconds")]
def litellm_overhead_seconds(self) -> list[float]:
return [value for _, value in self.latency_pairs("litellm_overhead_seconds")]
def litellm_response_seconds(self) -> list[float]:
return [value for _, value in self.latency_pairs("litellm_response_seconds")]
def time_to_first_token_seconds(self) -> list[float]:
"""Seconds from the client sending each first-attempt streamed request to its first token, mock delay
included, comparable across versions because the mocks generate the same replies after the same wait."""
return [record.first_token_s for record, _ in self._clean(stream=True) if record.first_token_s is not None]
return [value for _, value in self.latency_pairs("time_to_first_token_seconds")]
def bodies_off_reply(self) -> int:
"""Served requests whose text is not the reply the mock generated for them (plus the post-call rewrite
@ -189,26 +247,236 @@ class Observation:
post: Final = Counter(call.marker for call in self.guardrail_calls if call.input_type == "response")
return sum(1 for record in self.served if pre[record.marker] != 1 or post[record.marker] != 1)
def steps(self) -> tuple[StepReading, ...]:
"""One reading per step of a ramped run: the gateway's added latency over the step's requests, the rate
at which requests completed inside the step's window, the step's unexpected statuses, and what the pod
was doing in that window (CPU per completed request, Redis latency, Postgres connections)."""
ramp: Final = self.ramp
if ramp is None or not self.records:
return ()
started: Final = min(record.started_at for record in self.records)
added: Final = _by_step(ramp, self.latency_pairs("gateway_added_seconds"))
first_token: Final = _by_step(ramp, self.latency_pairs("time_to_first_token_seconds"))
completed: Final = Counter(
int((record.started_at + record.total_s - started) // ramp.step_seconds) for record in self.records
)
unexpected: Final = Counter(
ramp.step_of(record.seq) for record in self.records if not record.blocked and record.status != 200
)
return tuple(
StepReading(
rps=rate,
planned=count,
completed_rps=completed[index] / ramp.step_seconds,
gateway_added_p99=percentile(added.get(index, ()), 99),
unexpected=unexpected[index],
held_p99_seconds=ramp.held_p99_seconds,
gateway_added_p50=percentile(added.get(index, ()), 50),
time_to_first_token_p99=percentile(first_token.get(index, ()), 99),
**self._window_readings(started + index * ramp.step_seconds, ramp.step_seconds, completed[index]),
)
for index, (rate, count) in enumerate(zip(ramp.rates, ramp.counts, strict=True))
)
def _window_readings(self, start: float, seconds: float, completed: int) -> WindowReadings:
end: Final = start + seconds
cpu: Final = None if self.cpu_series is None else sum(self.cpu_series.between(start, end))
conns: Final = None if self.connection_series is None else self.connection_series.between(start, end)
before: Final = None if self.telemetry_series is None else self.telemetry_series.last_before(start)
after: Final = None if self.telemetry_series is None else self.telemetry_series.last_before(end)
return WindowReadings(
cpu_seconds_per_request=None if cpu is None or completed == 0 else cpu / completed,
redis_latency_mean_seconds=None
if before is None or after is None
else after.minus(before).redis_latency.mean,
postgres_peak_connections=max(conns, default=None) if conns else None,
)
class WindowReadings(TypedDict):
cpu_seconds_per_request: ReadOnly[float | None]
redis_latency_mean_seconds: ReadOnly[float | None]
postgres_peak_connections: ReadOnly[int | None]
def _by_step(ramp: Ramp, pairs: Iterable[tuple[WireRecord, float]]) -> Mapping[int, tuple[float, ...]]:
ordered: Final = sorted(pairs, key=lambda pair: pair[0].seq)
return MappingProxyType(
{
step: tuple(value for _, value in group)
for step, group in groupby(ordered, key=lambda pair: ramp.step_of(pair[0].seq))
}
)
def _text_sha1(text: str, *, intervened: bool) -> str:
original: Final = text.removesuffix(" [post]") if intervened else text
return hashlib.sha1(original.encode()).hexdigest()
def observe(metric: Metric, percentile_rank: int | None, observation: Observation) -> float | None:
LatencyMetric = Literal[
"gateway_added_seconds", "time_to_first_token_seconds", "litellm_overhead_seconds", "litellm_response_seconds"
]
TOP_CONTRIBUTORS: Final = 5
@dataclass(frozen=True, slots=True)
class Contributor:
"""One slice of a metric: a statement, a Redis command, an endpoint, a worker or a status, with its own
value in the metric's unit and, where the metric is a total, its share of that total."""
label: str
value: float
share: float | None = None
def _top(items: Iterable[Contributor]) -> tuple[Contributor, ...]:
return tuple(sorted(items, key=lambda item: item.value, reverse=True)[:TOP_CONTRIBUTORS])
def _endpoint_label(record: WireRecord) -> str:
return f"{record.endpoint}{' stream' if record.stream else ''}"
def _latency_breakdown(metric: LatencyMetric, rank: int, observation: Observation) -> tuple[Contributor, ...]:
pairs: Final = observation.latency_pairs(metric)
by_endpoint: dict[str, list[float]] = defaultdict(list)
for record, value in pairs:
by_endpoint[_endpoint_label(record)].append(value)
return _top(
Contributor(label, percentile(values, rank) or 0.0, len(values) / len(pairs))
for label, values in by_endpoint.items()
)
def _statements_for(
metric: Metric, statements: tuple[StatementCalls, ...], exempt_tables: frozenset[str]
) -> tuple[StatementCalls, ...]:
if metric in ("postgres_calls_per_request", "postgres_max_statement_per_request"):
return tuple(item for item in statements if item.bucket == "awaited")
if metric == "postgres_background_calls_per_request":
return tuple(item for item in statements if item.bucket == "background")
if metric == "postgres_max_background_statement_per_request":
return tuple(item for item in statements if item.bucket == "background" and item.table not in exempt_tables)
return tuple(item for item in statements if item.calls > 0)
def _statement_breakdown(
metric: Metric, observation: Observation, exempt_tables: frozenset[str]
) -> tuple[Contributor, ...]:
chosen: Final = _statements_for(metric, observation.statements or (), exempt_tables)
requests: Final = max(1, len(observation.records))
total_calls: Final = sum(item.calls for item in chosen)
total_ms: Final = sum(item.total_exec_ms for item in chosen)
if metric == "postgres_slowest_statement_seconds":
return _top(
Contributor(
item.summary, item.mean_exec_seconds or 0.0, item.total_exec_ms / total_ms if total_ms else None
)
for item in chosen
)
if metric == "postgres_active_sessions":
return _top(
Contributor(
item.summary, item.total_exec_ms / 1000 / observation.wall_seconds, item.total_exec_ms / total_ms
)
for item in chosen
if total_ms
)
return _top(
Contributor(item.summary, item.calls / requests, item.calls / total_calls) for item in chosen if total_calls
)
def _redis_breakdown(metric: Metric, observation: Observation) -> tuple[Contributor, ...]:
bucket: Final = "awaited" if metric == "redis_calls_per_request" else "background"
chosen: Final = tuple(item for item in observation.redis_commands or () if item.bucket == bucket)
requests: Final = max(1, len(observation.records))
total: Final = sum(item.count for item in chosen)
return _top(
Contributor(f"{item.command} {item.key}".strip(), item.count / requests, item.count / total)
for item in chosen
if total
)
def _memory_breakdown(observation: Observation) -> tuple[Contributor, ...]:
before: Final = {reading.worker: reading.ram_usage_mb for reading in observation.rss_before.readings}
return _top(
Contributor(f"{reading.replica} pid {reading.worker_pid}", reading.ram_usage_mb - before[reading.worker])
for reading in observation.rss_after.readings
if reading.worker in before
)
def _status_breakdown(observation: Observation) -> tuple[Contributor, ...]:
unexpected: Final = [r for r in observation.records if not r.blocked and r.status != 200]
counts: Final = Counter(f"{_endpoint_label(record)} status {record.status}" for record in unexpected)
return _top(Contributor(label, float(count), count / len(unexpected)) for label, count in counts.items())
def breakdown(
metric: Metric, percentile_rank: int | None, observation: Observation, exempt_tables: frozenset[str] = frozenset()
) -> tuple[Contributor, ...]:
"""The top contributors to one metric, so a spike on the chart names its statement, command, endpoint or
worker. Gauges with no slices (`cpu_seconds_per_request`, `prometheus_redis_latency_mean_seconds`,
`postgres_peak_connections`) have none."""
if metric in (
"gateway_added_seconds",
"time_to_first_token_seconds",
"litellm_overhead_seconds",
"litellm_response_seconds",
):
return _latency_breakdown(metric, 50 if percentile_rank is None else percentile_rank, observation)
if metric.startswith("postgres_") and metric != "postgres_peak_connections":
return _statement_breakdown(metric, observation, exempt_tables)
if metric in {"redis_calls_per_request", "redis_background_calls_per_request"}:
return _redis_breakdown(metric, observation)
if metric == "memory_growth_megabytes":
return _memory_breakdown(observation)
if metric == "unexpected_status_count":
return _status_breakdown(observation)
return ()
def observe(
metric: Metric, percentile_rank: int | None, observation: Observation, exempt_tables: frozenset[str] = frozenset()
) -> float | None:
"""The observed value of one metric, at a percentile when the metric is per request."""
per_request: dict[str, list[float]] = {
"gateway_added_seconds": observation.gateway_added_seconds(),
"time_to_first_token_seconds": observation.time_to_first_token_seconds(),
"litellm_overhead_seconds": observation.litellm_overhead_seconds(),
"litellm_response_seconds": observation.litellm_response_seconds(),
}
if metric in per_request:
return percentile(per_request[metric], 50 if percentile_rank is None else percentile_rank)
if metric == "unexpected_status_count":
return float(observation.unexpected_status_count())
if metric == "prometheus_redis_latency_mean_seconds":
counted: Final = observation.telemetry
return None if counted is None else counted.redis_latency.mean
if metric == "postgres_active_sessions":
return active_sessions(observation.statements, observation.wall_seconds)
if metric == "postgres_slowest_statement_seconds":
slowest: Final = slowest_statement(observation.statements)
return None if slowest is None else slowest.mean_exec_seconds
if metric == "postgres_peak_connections":
return None if observation.peak_connections is None else float(observation.peak_connections)
if metric == "postgres_calls_per_request":
return observation.count_per_request(calls_in("awaited", observation.statements))
if metric == "postgres_background_calls_per_request":
return observation.count_per_request(calls_in("background", observation.statements))
if metric == "postgres_max_statement_per_request":
if observation.statements is None:
return None
busiest: Final = observation.busiest_awaited_statement
return observation.count_per_request(0 if busiest is None else busiest.calls)
if metric == "postgres_max_background_statement_per_request":
if observation.statements is None:
return None
busiest_background: Final = observation.busiest_background_statement(exempt_tables)
return observation.count_per_request(0 if busiest_background is None else busiest_background.calls)
if metric == "redis_calls_per_request":
return observation.count_per_request(calls_in("awaited", observation.redis_commands))
if metric == "redis_background_calls_per_request":
@ -216,6 +484,8 @@ def observe(metric: Metric, percentile_rank: int | None, observation: Observatio
if metric == "cpu_seconds_per_request":
cpu: Final = observation.gateway_cpu_seconds
return None if cpu is None or not observation.records else cpu / len(observation.records)
if metric == "sustained_rps":
return None if observation.ramp is None else sustained_rps(observation.steps())
return worst_worker_growth_mb(observation.rss_before, observation.rss_after)
@ -225,6 +495,9 @@ class Row:
percentile: int | None
observed: float | None
limit: float | None
floor: float | None = None
detail: str = ""
breakdown: tuple[Contributor, ...] = ()
@property
def label(self) -> str:
@ -236,22 +509,43 @@ class Row:
@property
def over(self) -> bool:
return self.limit is not None and self.observed is not None and self.observed > self.limit
"""Past the limit: above a max, or below a floor."""
if self.observed is None:
return False
return (self.limit is not None and self.observed > self.limit) or (
self.floor is not None and self.observed < self.floor
)
@property
def missing(self) -> bool:
return self.limit is not None and self.observed is None
return (self.limit is not None or self.floor is not None) and self.observed is None
@property
def bound(self) -> str:
if self.limit is not None:
return f"{self.limit:.4f}"
return "" if self.floor is None else f">= {self.floor:.4f}"
REPORTED: Final[tuple[tuple[Metric, int | None], ...]] = (
("unexpected_status_count", None),
("gateway_added_seconds", 50),
("gateway_added_seconds", 99),
("litellm_overhead_seconds", 50),
("litellm_overhead_seconds", 99),
("litellm_response_seconds", 50),
("litellm_response_seconds", 99),
("prometheus_redis_latency_mean_seconds", None),
("postgres_active_sessions", None),
("postgres_slowest_statement_seconds", None),
("postgres_peak_connections", None),
("time_to_first_token_seconds", 50),
("time_to_first_token_seconds", 99),
("memory_growth_megabytes", None),
("postgres_calls_per_request", None),
("postgres_background_calls_per_request", None),
("postgres_max_statement_per_request", None),
("postgres_max_background_statement_per_request", None),
("redis_calls_per_request", None),
("redis_background_calls_per_request", None),
("cpu_seconds_per_request", None),
@ -265,6 +559,8 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[
(limit.metric, limit.percentile): limit for limit in case.limits
}
keys: list[tuple[Metric, int | None]] = list(REPORTED)
if observation.ramp is not None:
keys.append(("sustained_rps", None))
keys.extend(key for key in limits if key not in keys)
rows: list[Row] = []
for metric, rank in keys:
@ -273,27 +569,53 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[
Row(
metric=metric,
percentile=rank,
observed=observe(metric, rank, observation),
observed=observe(metric, rank, observation, case.background_cap_exempt_tables),
limit=limit.max if limit else None,
floor=limit.min if limit else None,
detail=_detail(metric, observation, case.background_cap_exempt_tables),
breakdown=breakdown(metric, rank, observation, case.background_cap_exempt_tables),
)
)
exceeded: Final = [
f"{row.label}: observed {row.observed:.4f} > limit {row.limit:.4f}"
if row.over and row.observed is not None and row.limit is not None
else f"{row.label}: limit set but nothing observed"
for row in rows
if row.over or row.missing
]
exceeded: Final = [_breach(row) for row in rows if row.over or row.missing]
return rows, exceeded
def _row_properties(row: Row) -> tuple[tuple[str, float], ...]:
def _breach(row: Row) -> str:
if row.observed is None:
return f"{row.label}: limit set but nothing observed"
if row.floor is not None:
return f"{row.label}: observed {row.observed:.4f} < floor {row.floor:.4f}"
return f"{row.label}: observed {row.observed:.4f} > limit {row.bound}{f' ({row.detail})' if row.detail else ''}"
def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[str]) -> str:
if metric == "postgres_max_statement_per_request":
busiest: Final = observation.busiest_awaited_statement
return "" if busiest is None else busiest.summary
if metric == "postgres_max_background_statement_per_request":
busiest_background: Final = observation.busiest_background_statement(exempt_tables)
return "" if busiest_background is None else busiest_background.summary
if metric == "postgres_slowest_statement_seconds":
slowest: Final = slowest_statement(observation.statements)
return "" if slowest is None else slowest.summary
return ""
def _row_properties(row: Row) -> tuple[tuple[str, float | str], ...]:
observed: Final = ((row.property_name, row.observed),) if row.observed is not None else ()
limit: Final = ((f"{row.property_name}_limit", row.limit),) if row.limit is not None else ()
return observed + limit
floor: Final = ((f"{row.property_name}_floor", row.floor),) if row.floor is not None else ()
contributors: Final = (
((f"{row.property_name}_breakdown", json.dumps([asdict(item) for item in row.breakdown])),)
if row.breakdown
else ()
)
return observed + limit + floor + contributors
def junit_properties(rows: Iterable[Row]) -> tuple[tuple[str, float], ...]:
def junit_properties(rows: Iterable[Row]) -> tuple[tuple[str, float | str], ...]:
"""One numeric property per observed metric and per limit, plus a `_breakdown` JSON list of the top
contributors for every metric that has slices, all under the `perf_` prefix the dashboard ingests."""
return tuple(chain.from_iterable(map(_row_properties, rows)))
@ -313,9 +635,17 @@ def render(case: PerformanceTestCase, observation: Observation, rows: list[Row])
]
for row in rows:
observed = "n/a" if row.observed is None else f"{row.observed:.4f}"
limit = "" if row.limit is None else f"{row.limit:.4f}"
flag = " OVER" if row.over else " MISSING" if row.missing else ""
lines.append(f" {row.label:32} {observed:>12} {limit:>12}{flag}")
flag = (
" UNDER"
if row.over and row.floor is not None
else " OVER"
if row.over
else " MISSING"
if row.missing
else ""
)
lines.append(f" {row.label:32} {observed:>12} {row.bound:>12}{flag}")
lines.extend(render_steps(observation.steps()))
lines.extend(_diagnostics(observation))
lines.extend(_datastore_lines(observation))
for record in failures:
@ -332,6 +662,7 @@ def _diagnostics(observation: Observation) -> list[str]:
return [
" diagnostics (not limits):",
f" bodies off reply {observation.bodies_off_reply()}",
f" mock routes {_mock_routes(observation)}",
f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}",
f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}",
f" retried share {len(observation.retried_markers) / count:.4f}",
@ -342,6 +673,12 @@ def _diagnostics(observation: Observation) -> list[str]:
]
def _mock_routes(observation: Observation) -> str:
"""Which provider route each request landed on, so a direct-vs-bridge case shows its split."""
counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests)
return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a"
def _datastore_lines(observation: Observation) -> list[str]:
"""What the per-request Postgres and Redis counts are made of, so a red run names the statement or
command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""

View file

@ -9,10 +9,14 @@ so a case is a literal a reviewer can read top to bottom.
from __future__ import annotations
import bisect
from collections.abc import Mapping
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from itertools import accumulate
from typing import Final, Literal
from setup.event_loop import EventLoopStalls
Endpoint = Literal["chat_completions", "responses", "messages"]
MockProvider = Literal["openai", "anthropic", "bedrock"]
GuardrailMode = Literal["pre_call", "post_call"]
@ -23,9 +27,18 @@ Metric = Literal[
"memory_growth_megabytes",
"postgres_calls_per_request",
"postgres_background_calls_per_request",
"postgres_max_statement_per_request",
"postgres_max_background_statement_per_request",
"cpu_seconds_per_request",
"redis_calls_per_request",
"redis_background_calls_per_request",
"litellm_overhead_seconds",
"litellm_response_seconds",
"prometheus_redis_latency_mean_seconds",
"postgres_active_sessions",
"postgres_slowest_statement_seconds",
"postgres_peak_connections",
"sustained_rps",
]
@ -87,12 +100,50 @@ SHORT: Final = RequestShape("short", input_tokens=(5_000, 100_000), tools=True)
LONG: Final = RequestShape("long", input_tokens=(300_000, 1_000_000), tools=True)
@dataclass(frozen=True, slots=True)
class Ramp:
"""Rate stepped up through the run: `first_rps`, then `step_rps` more every `step_seconds`, up to `last_rps`.
Each step is scored on its own and the run reports `sustained_rps`, the last step the pod held before the
first one it did not: a step holds when its `gateway_added_seconds` p99 stayed under `held_p99_seconds`, it
completed at least `HELD_SHARE` of its rate and no more than `HELD_UNEXPECTED_SHARE` of its requests came back
with an unexpected status (`setup/ramp.py`). Requests are sent on a schedule drawn from the plan's seed, not
from per-user intervals, so the rate changes on the step boundary."""
first_rps: float
last_rps: float
step_rps: float
step_seconds: float
held_p99_seconds: float
@property
def rates(self) -> tuple[float, ...]:
count: Final = round((self.last_rps - self.first_rps) / self.step_rps) + 1
return tuple(self.first_rps + index * self.step_rps for index in range(count))
@property
def counts(self) -> tuple[int, ...]:
return tuple(round(rate * self.step_seconds) for rate in self.rates)
@property
def total_requests(self) -> int:
return sum(self.counts)
@property
def seconds(self) -> float:
return len(self.rates) * self.step_seconds
def step_of(self, seq: int) -> int:
"""Zero-based step of the request with 1-based plan number `seq`."""
return bisect.bisect_left(tuple(accumulate(self.counts)), seq)
@dataclass(frozen=True, slots=True)
class Traffic:
"""The plan: `total_requests` drawn from `seed`, sent at `rps` by `concurrency` simulated
users. Shares are fractions of the requests: `endpoints` and `shapes` must each sum to 1,
`intervene_share` picks non-streaming requests the guardrail rewrites, `block_share` picks
requests it blocks."""
users, or stepped up through a `ramp` (then `rps` is its first step and `total_requests` its
total, see `ramped`). Shares are fractions of the requests: `endpoints` and `shapes` must each
sum to 1, `intervene_share` picks non-streaming requests the guardrail rewrites, `block_share`
picks requests it blocks."""
seed: int
rps: float
@ -103,20 +154,33 @@ class Traffic:
intervene_share: float = 0.0
block_share: float = 0.0
concurrency: int = 20
ramp: Ramp | None = None
@property
def expected_seconds(self) -> float:
return self.total_requests / self.rps
return self.total_requests / self.rps if self.ramp is None else self.ramp.seconds
def ramped(base: Traffic, ramp: Ramp, *, seed: int, concurrency: int) -> Traffic:
"""`base`'s request mix sent along `ramp` instead of at one rate. `concurrency` caps the requests in
flight, so size it past the deepest step the ramp is expected to reach."""
return replace(
base, seed=seed, rps=ramp.first_rps, total_requests=ramp.total_requests, concurrency=concurrency, ramp=ramp
)
@dataclass(frozen=True, slots=True)
class Limit:
"""One pass/fail rule: `metric` (at `percentile`, or the whole-run value) must not exceed `max`.
Metrics without a Limit are still reported."""
"""One pass/fail rule: `metric` (at `percentile`, or the whole-run value) must not exceed `max`, or must
reach `min` for a floor such as `sustained_rps`. Metrics without a Limit are still reported."""
metric: Metric
max: float
max: float | None = None
percentile: int | None = None
min: float | None = None
def __post_init__(self) -> None:
assert (self.max is None) != (self.min is None), f"{self.metric}: a Limit is either a max or a min"
@property
def label(self) -> str:
@ -148,6 +212,10 @@ class PerformanceTestCase:
traffic: Traffic
limits: tuple[Limit, ...]
mock_latency: MockLatency = field(default_factory=lambda: MockLatency())
background_cap_exempt_tables: frozenset[str] = frozenset()
"""Tables whose background write is one upsert per writer flush and, at this case's request rate, lands
above the `postgres_max_background_statement_per_request` cap without being per request."""
event_loop: EventLoopStalls | None = None
def team(self, name: str) -> Team:
return next(team for team in self.teams if team.name == name)
@ -187,6 +255,14 @@ class PerformanceTestCase:
assert 0 <= self.traffic.stream_share <= 1, f"{self.name}: stream_share out of range"
assert self.traffic.intervene_share + self.traffic.block_share <= 1, f"{self.name}: guardrail shares exceed 1"
assert self.traffic.total_requests > 0 and self.traffic.rps > 0, f"{self.name}: empty traffic"
ramp: Final = self.traffic.ramp
assert ramp is None or (ramp.first_rps, ramp.total_requests) == (
self.traffic.rps,
self.traffic.total_requests,
), f"{self.name}: traffic rps and total_requests must be the ramp's, build it with ramped()"
assert ramp is None or (0 < ramp.first_rps <= ramp.last_rps and ramp.step_rps > 0 and ramp.step_seconds > 0), (
f"{self.name}: ramp {ramp} does not step up"
)
assert self.model_groups, f"{self.name}: no model groups"
assert 0 <= self.mock_latency.jitter_seconds <= self.mock_latency.seconds or self.mock_latency.seconds == 0, (
f"{self.name}: mock latency jitter {self.mock_latency.jitter_seconds} s exceeds {self.mock_latency.seconds} s"

View file

@ -12,6 +12,7 @@ from dataclasses import dataclass
from typing import Final, Literal, TypeVar
from setup.performance_test_case import Endpoint, PerformanceTestCase, RequestShape, Team
from setup.ramp import send_offsets
Flag = Literal["n", "i", "b"]
CallerKind = Literal["jwt", "key"]
@ -39,6 +40,7 @@ class PlannedRequest:
flag: Flag
shape: RequestShape
input_tokens: int
send_at_s: float | None = None
@property
def blocked(self) -> bool:
@ -98,6 +100,7 @@ def build_plan(case: PerformanceTestCase) -> tuple[PlannedRequest, ...]:
case.validate()
rng: Final = random.Random(case.traffic.seed)
callers: Final = _callers_by_group(case)
offsets: Final = None if case.traffic.ramp is None else send_offsets(case.traffic.ramp, case.traffic.seed)
planned: list[PlannedRequest] = []
for seq in range(1, case.traffic.total_requests + 1):
endpoint = _pick(rng, case.traffic.endpoints)
@ -117,6 +120,7 @@ def build_plan(case: PerformanceTestCase) -> tuple[PlannedRequest, ...]:
flag=flag,
shape=shape,
input_tokens=rng.randint(low, high),
send_at_s=None if offsets is None else offsets[seq - 1],
)
)
return tuple(planned)

View file

@ -19,6 +19,8 @@ from urllib.parse import quote
from e2e_http import AuthHeaders, Success, get_external
from pydantic import BaseModel
from setup.series import Recorder, Series
SELECTOR_ENV: Final = "E2E_PERF_GATEWAY_POD_SELECTOR"
_SERVICE_ACCOUNT: Final = Path("/var/run/secrets/kubernetes.io/serviceaccount")
_INTERVAL_S: Final = 5.0
@ -49,7 +51,8 @@ def cores(quantity: str) -> float:
class GatewayCpuSampler:
"""Integrates the matching pods' core use over the time it is running; `seconds` is CPU seconds."""
"""Integrates the matching pods' core use over the time it is running; `seconds` is CPU seconds and
`series` the CPU seconds of each sampling interval, stamped when the interval ended."""
def __init__(self, selector: str, namespace: str, token: str, ca_bundle: str) -> None:
self._url: Final = (
@ -60,12 +63,15 @@ class GatewayCpuSampler:
self._ca_bundle: Final = ca_bundle
self._stop: Final = threading.Event()
self._thread: Final = threading.Thread(target=self._run, name="gateway-cpu-sampler", daemon=True)
self._cpu_seconds = 0.0
self._samples = 0
self._intervals: Final[Recorder[float]] = Recorder()
@property
def seconds(self) -> float | None:
return self._cpu_seconds if self._samples > 0 else None
return sum(sample.value for sample in self._intervals.series.samples) if len(self._intervals) else None
@property
def series(self) -> Series[float]:
return self._intervals.series
def _read_cores(self) -> float | None:
result: Final = get_external(
@ -81,8 +87,7 @@ class GatewayCpuSampler:
now: float = time.monotonic()
current: float | None = self._read_cores()
if current is not None:
self._cpu_seconds += current * (now - last)
self._samples += 1
self._intervals.record(current * (now - last))
last = now # rebind-ok: the sampling clock advances every tick
def start(self) -> None:
@ -95,6 +100,7 @@ class GatewayCpuSampler:
class NoSampler:
seconds: Final[None] = None
series: Final[None] = None
def start(self) -> None:
return None
@ -103,7 +109,7 @@ class NoSampler:
return None
def _in_cluster() -> tuple[str, str, str] | None:
def in_cluster() -> tuple[str, str, str] | None:
token: Final = _SERVICE_ACCOUNT / "token"
ca: Final = _SERVICE_ACCOUNT / "ca.crt"
namespace: Final = _SERVICE_ACCOUNT / "namespace"
@ -115,7 +121,7 @@ def _in_cluster() -> tuple[str, str, str] | None:
@contextmanager
def gateway_cpu_sampler() -> Generator[GatewayCpuSampler | NoSampler]:
selector: Final = os.environ.get(SELECTOR_ENV, "")
cluster: Final = _in_cluster()
cluster: Final = in_cluster()
sampler: Final = (
GatewayCpuSampler(selector, cluster[2], cluster[0], cluster[1]) if selector and cluster else NoSampler()
)

View file

@ -0,0 +1,116 @@
"""A ramped run step by step: the send schedule its plan carries (`send_offsets`), one `StepReading` per step
and `sustained_rps`, the last step the pod held before the first it did not. `Ramp` itself sits in
`performance_test_case.py` with the rest of the case schema."""
from __future__ import annotations
import random
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import chain, takewhile
from typing import Final
from setup.performance_test_case import Ramp
HELD_SHARE: Final = 0.9
"""A step that completed fewer than this share of its rate is a queue that stopped draining."""
HELD_UNEXPECTED_SHARE: Final = 0.01
"""A step with more than this share of unexpected statuses broke, whatever its latency."""
_JITTER: Final = 0.25
def _step_offsets(ramp: Ramp, index: int, count: int, rng: random.Random) -> Iterator[float]:
for slot in range(count):
yield (index + (slot + 0.5 + rng.uniform(-_JITTER, _JITTER)) / count) * ramp.step_seconds
def send_offsets(ramp: Ramp, seed: int) -> tuple[float, ...]:
"""Seconds after the run starts to send each request of the plan, in plan order: a step's requests spread
evenly over its window, each moved by up to a quarter of a slot either way, drawn from `seed` so a rerun
sends at the same seconds."""
rng: Final = random.Random(f"ramp-{seed}")
return tuple(chain.from_iterable(_step_offsets(ramp, index, count, rng) for index, count in enumerate(ramp.counts)))
@dataclass(frozen=True, slots=True)
class StepReading:
rps: float
planned: int
completed_rps: float
gateway_added_p99: float | None
unexpected: int
held_p99_seconds: float
gateway_added_p50: float | None = None
time_to_first_token_p99: float | None = None
cpu_seconds_per_request: float | None = None
redis_latency_mean_seconds: float | None = None
postgres_peak_connections: int | None = None
"""The hold verdict reads the first six. The rest say what the pod was doing on that step: CPU per request
from the pod sampler, Redis latency from the gateway's `/metrics`, connections from `pg_stat_activity`,
all cut to the step's window, so a latency jump can be told apart as CPU, waiting or neither."""
@property
def held(self) -> bool:
return (
self.gateway_added_p99 is not None
and self.gateway_added_p99 <= self.held_p99_seconds
and self.completed_rps >= HELD_SHARE * self.rps
and self.unexpected <= HELD_UNEXPECTED_SHARE * self.planned
)
def sustained_rps(steps: Iterable[StepReading]) -> float:
"""The rate of the last step that held before the first that did not, 0 when the first step did not hold."""
held: Final = tuple(takewhile(lambda step: step.held, steps))
return held[-1].rps if held else 0.0
def step_properties(steps: Iterable[StepReading]) -> tuple[tuple[str, float], ...]:
"""The per-step readings as JUnit properties (`perf_ramp_step_<rps>_<reading>`, every reading the step has)
plus the hold bound, so the dashboard can draw the run's curves next to the sustained rate."""
rows: Final = tuple(steps)
hold: Final = (("perf_ramp_hold_p99_seconds", rows[0].held_p99_seconds),) if rows else ()
return (*hold, *chain.from_iterable(_step_properties(step) for step in rows))
def _step_properties(step: StepReading) -> Iterator[tuple[str, float]]:
prefix: Final = f"perf_ramp_step_{step.rps:g}"
readings: Final[tuple[tuple[str, float | None], ...]] = (
("gateway_added_p99", step.gateway_added_p99),
("completed_rps", step.completed_rps),
("unexpected", float(step.unexpected)),
("gateway_added_p50", step.gateway_added_p50),
("time_to_first_token_p99", step.time_to_first_token_p99),
("cpu_seconds_per_request", step.cpu_seconds_per_request),
("redis_latency_mean_seconds", step.redis_latency_mean_seconds),
(
"postgres_peak_connections",
None if step.postgres_peak_connections is None else float(step.postgres_peak_connections),
),
)
return ((f"{prefix}_{name}", value) for name, value in readings if value is not None)
def render_steps(steps: Iterable[StepReading]) -> tuple[str, ...]:
rows: Final = tuple(steps)
if not rows:
return ()
header: Final = (
f" steps: {'rps':>6} {'planned':>8} {'completed rps':>14} {'gateway added p50':>18} {'p99':>8}"
f" {'ttft p99':>9} {'unexpected':>11} {'cpu s/req':>10} {'redis mean':>11} {'pg conns':>9}"
)
return (header, *(_step_line(step) for step in rows))
def _seconds(value: float | None) -> str:
return "n/a" if value is None else f"{value:.4f}"
def _step_line(step: StepReading) -> str:
conns: Final = "n/a" if step.postgres_peak_connections is None else str(step.postgres_peak_connections)
return (
f" {step.rps:6.1f} {step.planned:8d} {step.completed_rps:14.2f} {_seconds(step.gateway_added_p50):>18}"
f" {_seconds(step.gateway_added_p99):>8} {_seconds(step.time_to_first_token_p99):>9} {step.unexpected:11d}"
f" {_seconds(step.cpu_seconds_per_request):>10} {_seconds(step.redis_latency_mean_seconds):>11} {conns:>9}"
f" {'held' if step.held else 'broke'}"
)

View file

@ -9,9 +9,11 @@ import shutil
import subprocess
import sys
import tempfile
import time
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Final
from typing import BinaryIO, Final
from e2e_config import PROXY_BASE_URL
from pydantic import TypeAdapter
@ -61,6 +63,7 @@ def wire_request(item: PlannedRequest, setup: CaseSetup) -> WireRequest:
token_source=setup.token_source(item.caller),
stream=item.stream,
blocked=item.blocked,
send_at_s=item.send_at_s,
)
@ -73,7 +76,59 @@ def _keep_records_artifact(records_path: Path) -> None:
_ = shutil.copyfile(records_path, Path(artifact_dir) / "perf-records.jsonl")
def run_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...], setup: CaseSetup) -> RunOutcome:
class RecordStream:
def __init__(self, source: BinaryIO) -> None:
self.source: Final = source
self.partial = b""
def read(self) -> tuple[WireRecord, ...]:
chunks: Final = (self.partial + self.source.read()).split(b"\n")
self.partial = chunks[-1]
return tuple(_RECORD_ADAPTER.validate_json(line) for line in chunks[:-1] if line)
def run_until(
command: list[str],
env: dict[str, str],
records_path: Path,
stop_path: Path,
stop_when: Callable[[tuple[WireRecord, ...]], bool],
timeout: int,
) -> subprocess.CompletedProcess[str]:
with records_path.open("rb") as records, records_path.with_suffix(".log").open("w+") as output:
with subprocess.Popen(command, env=env, stdout=output, stderr=subprocess.STDOUT, text=True) as process:
deadline: Final = time.monotonic() + timeout
reader: Final = RecordStream(records)
try:
while process.poll() is None:
if stop_when(reader.read()):
stop_path.touch()
if time.monotonic() >= deadline:
raise subprocess.TimeoutExpired(command, timeout)
try:
_ = process.wait(timeout=1)
except subprocess.TimeoutExpired:
pass
finally:
stop_path.touch()
try:
_ = process.wait(timeout=35)
except subprocess.TimeoutExpired:
process.kill()
_ = process.wait(timeout=10)
_keep_records_artifact(records_path)
_ = output.seek(0)
return subprocess.CompletedProcess(command, process.returncode, output.read(), "")
def run_plan(
case: PerformanceTestCase,
plan: tuple[PlannedRequest, ...],
setup: CaseSetup,
*,
stop_when: Callable[[tuple[WireRecord, ...]], bool] | None = None,
request_timeout: float = 120.0,
) -> RunOutcome:
"""Replay `plan` with Locust in a child process and collect its records. The run-time ceiling
is well past the planned duration so Locust's own clock never cuts a slow run short; the
locustfile quits itself once the last record is written."""
@ -82,6 +137,7 @@ def run_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...], setup:
with tempfile.TemporaryDirectory(prefix="e2e-perf-") as directory:
plan_path: Final = Path(directory) / "plan.json"
records_path: Final = Path(directory) / "records.jsonl"
stop_path: Final = Path(directory) / "stop"
_ = plan_path.write_bytes(_PLAN_ADAPTER.dump_json(wire))
records_path.touch()
env: Final = {
@ -91,33 +147,33 @@ def run_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...], setup:
"PERF_RECORDS_FILE": str(records_path),
"PERF_RPS": str(case.traffic.rps),
"PERF_USERS": str(case.traffic.concurrency),
"PERF_REQUEST_TIMEOUT": str(request_timeout),
**({"PERF_STOP_FILE": str(stop_path)} if stop_when is not None else {}),
}
completed: Final = subprocess.run(
[
sys.executable,
"-P",
"-m",
"locust",
"--headless",
"--only-summary",
"--locustfile",
str(_LOCUSTFILE),
"--host",
PROXY_BASE_URL,
"--users",
str(case.traffic.concurrency),
"--spawn-rate",
str(case.traffic.concurrency),
"--run-time",
f"{ceiling_s}s",
"--exit-code-on-error",
"0",
],
env=env,
capture_output=True,
text=True,
timeout=ceiling_s + 120,
check=False,
command: Final = [
sys.executable,
"-P",
"-m",
"locust",
"--headless",
"--only-summary",
"--locustfile",
str(_LOCUSTFILE),
"--host",
PROXY_BASE_URL,
"--users",
str(case.traffic.concurrency),
"--spawn-rate",
str(case.traffic.concurrency),
"--run-time",
f"{ceiling_s}s",
"--exit-code-on-error",
"0",
]
completed: Final = (
run_until(command, env, records_path, stop_path, stop_when, ceiling_s + 120)
if stop_when is not None
else subprocess.run(command, env=env, capture_output=True, text=True, timeout=ceiling_s + 120, check=False)
)
records: Final = tuple(
_RECORD_ADAPTER.validate_json(line)

View file

@ -0,0 +1,48 @@
"""Readings a background sampler takes while the traffic runs, each stamped with the wall clock
(`time.time()`, the clock the Locust records carry in `started_at`), so a ramped run can cut them into its
steps' windows after the fact."""
from __future__ import annotations
import time
from dataclasses import dataclass
from typing import Final, Generic, TypeVar
T = TypeVar("T")
@dataclass(frozen=True, slots=True)
class Sample(Generic[T]):
at: float
value: T
@dataclass(frozen=True, slots=True)
class Series(Generic[T]):
samples: tuple[Sample[T], ...]
def between(self, start: float, end: float) -> tuple[T, ...]:
"""The values stamped inside `[start, end)`, in time order."""
return tuple(sample.value for sample in self.samples if start <= sample.at < end)
def last_before(self, at: float) -> T | None:
"""The newest value stamped before `at`, None when the series starts later."""
earlier: Final = tuple(sample.value for sample in self.samples if sample.at < at)
return earlier[-1] if earlier else None
class Recorder(Generic[T]):
"""What a sampler thread appends to; `series` is a frozen copy for the observation."""
def __init__(self) -> None:
self._samples: list[Sample[T]] = []
def record(self, value: T) -> None:
self._samples.append(Sample(at=time.time(), value=value))
@property
def series(self) -> Series[T]:
return Series(samples=tuple(self._samples))
def __len__(self) -> int:
return len(self._samples)

View file

@ -0,0 +1,110 @@
"""The gateway's own `/metrics`, read before and after a run (and every few seconds during a ramped run, see
`telemetry_sampler`) for the numbers nothing outside the gateway can see. Today that is `litellm_redis_latency`, the histogram customers' Grafana panels show when Redis
slows the gateway down. The gateway's latency and request counts are read per request from its response
headers instead (`setup/wire.py`), which gives percentiles where a histogram delta only gives a mean.
RSS and CPU are not here: a multi-worker gateway serves `/metrics` through the prometheus multiprocess
collector, which drops `process_resident_memory_bytes` and `process_cpu_seconds_total`, so those stay
on psutil (`proxy_usage.py`)."""
from __future__ import annotations
import math
import threading
from collections.abc import Generator, Iterable
from contextlib import contextmanager
from dataclasses import dataclass
from itertools import chain
from typing import Final
from e2e_http import NoBody
from prometheus_client.parser import text_string_to_metric_families
from prometheus_client.samples import Sample
from proxy_client import ProxyClient
from setup.series import Recorder, Series
_SAMPLE_SECONDS: Final = 10.0
@dataclass(frozen=True, slots=True)
class HistogramTotals:
"""`_sum` and `_count` of one histogram across every label set, enough for a mean over a window."""
total: float
count: float
def minus(self, earlier: HistogramTotals) -> HistogramTotals:
return HistogramTotals(total=self.total - earlier.total, count=self.count - earlier.count)
@property
def mean(self) -> float | None:
return self.total / self.count if self.count > 0 else None
@dataclass(frozen=True, slots=True)
class GatewayTelemetry:
redis_latency: HistogramTotals
"""`litellm_redis_latency`, seconds per Redis call the gateway made."""
def minus(self, earlier: GatewayTelemetry) -> GatewayTelemetry:
return GatewayTelemetry(redis_latency=self.redis_latency.minus(earlier.redis_latency))
def _value(samples: Iterable[Sample], name: str) -> float:
return sum(sample.value for sample in samples if sample.name == name and math.isfinite(sample.value))
def _histogram(samples: tuple[Sample, ...], family: str) -> HistogramTotals:
return HistogramTotals(total=_value(samples, f"{family}_sum"), count=_value(samples, f"{family}_count"))
def parse_gateway_telemetry(text: str) -> GatewayTelemetry:
samples: Final = tuple(chain.from_iterable(family.samples for family in text_string_to_metric_families(text)))
return GatewayTelemetry(redis_latency=_histogram(samples, "litellm_redis_latency"))
def read_gateway_telemetry(proxy: ProxyClient) -> GatewayTelemetry | None:
"""One `/metrics` scrape, or None when the gateway does not serve it (no `prometheus` callback), in which
case every limit on a gateway-reported metric fails as missing rather than reading a silent zero."""
scrape: Final = proxy.probe("/metrics", params=NoBody())
return parse_gateway_telemetry(scrape.body) if scrape.status_code == 200 else None
class TelemetrySampler:
"""`/metrics` every `_SAMPLE_SECONDS` on a background thread while traffic runs, so a ramped run can read
the Redis latency of each step's window as the difference between the scrapes around it."""
def __init__(self, proxy: ProxyClient) -> None:
self._proxy: Final = proxy
self._stop: Final = threading.Event()
self._thread: Final = threading.Thread(target=self._run, name="gateway-telemetry", daemon=True)
self._scrapes: Final[Recorder[GatewayTelemetry]] = Recorder()
@property
def series(self) -> Series[GatewayTelemetry]:
return self._scrapes.series
def _run(self) -> None:
while not self._stop.is_set():
scrape: GatewayTelemetry | None = read_gateway_telemetry(self._proxy)
if scrape is not None:
self._scrapes.record(scrape)
_ = self._stop.wait(_SAMPLE_SECONDS)
def start(self) -> None:
self._thread.start()
def stop(self) -> None:
self._stop.set()
self._thread.join(timeout=30)
@contextmanager
def telemetry_sampler(proxy: ProxyClient) -> Generator[TelemetrySampler]:
sampler: Final = TelemetrySampler(proxy)
sampler.start()
try:
yield sampler
finally:
sampler.stop()

View file

@ -0,0 +1,430 @@
from __future__ import annotations
import math
import re
import statistics
import time
from collections.abc import Callable, Generator
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import Final
from urllib.parse import urlsplit
from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import Locator, Page, Response, expect, sync_playwright
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from prometheus_client.parser import text_string_to_metric_families
from pydantic import BaseModel, ConfigDict
MONEY: Final = re.compile(r"^\$[\d,]+\.\d{2}$")
LOADING_LOGS: Final = "Loading request logs…"
LOADING_MODELS: Final = "Loading models…"
LOADING_KEYS: Final = "Loading keys..."
class Sample(BaseModel):
model_config = ConfigDict(frozen=True)
seconds: float
rows: int = 0
error: str = ""
status: int = 0
request: str = ""
class Timings(BaseModel):
model_config = ConfigDict(frozen=True)
first_seconds: float | None
repeated_samples: int
repeated_p50_seconds: float | None
repeated_p95_seconds: float | None
repeated_p99_seconds: float | None
repeated_max_seconds: float | None
@dataclass(frozen=True, slots=True)
class Journey:
"""One dashboard interaction. ``start`` brings the page to the state the user acts from (untimed),
``act`` is the click or navigation being measured and ``settled`` blocks until the data the user
was waiting for is on screen, returning the number of rows or tiles that rendered."""
name: str
surface: str
start: Callable[[Page, float], None]
act: Callable[[Page], None]
settled: Callable[[Page, float], int]
@dataclass(frozen=True, slots=True)
class Dashboard:
base_url: str
username: str
password: str
anchor: datetime
def summarize(samples: tuple[Sample, ...]) -> Timings:
assert samples, "No UI journeys were measured"
repeated: Final = tuple(sorted(sample.seconds for sample in samples[1:] if not sample.error))
return Timings(
first_seconds=samples[0].seconds if not samples[0].error else None,
repeated_samples=len(repeated),
repeated_p50_seconds=statistics.median(repeated) if repeated else None,
repeated_p95_seconds=repeated[math.ceil(len(repeated) * 0.95) - 1] if len(repeated) >= 20 else None,
repeated_p99_seconds=repeated[math.ceil(len(repeated) * 0.99) - 1] if len(repeated) >= 100 else None,
repeated_max_seconds=max(repeated) if repeated else None,
)
def error_code(sample: Sample) -> str:
if not sample.error:
return ""
if sample.status >= 400:
return f"http_{sample.status}"
if sample.error.startswith("wall_clock"):
return "wall_clock"
return "timeout" if sample.error == "timeout" else "ui"
def ui_junit_properties(samples: tuple[Sample, ...]) -> tuple[tuple[str, float], ...]:
codes: Final = tuple(error_code(sample) for sample in samples if sample.error)
timings: Final = summarize(samples)
successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error))
latency: Final = (
("seconds_p50", statistics.median(successful) if successful else None),
("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None),
("seconds_max", max(successful) if successful else None),
("first_seconds", timings.first_seconds),
("repeated_seconds_p50", timings.repeated_p50_seconds),
("repeated_seconds_p95", timings.repeated_p95_seconds),
("repeated_seconds_p99", timings.repeated_p99_seconds),
("repeated_seconds_max", timings.repeated_max_seconds),
)
return (
("perf_ui_journeys", len(samples)),
("perf_ui_failures", len(codes)),
("perf_ui_samples", len(successful)),
("perf_ui_repeated_samples", timings.repeated_samples),
*((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))),
*((f"perf_ui_{name}", value) for name, value in latency if value is not None),
)
@dataclass(frozen=True, slots=True)
class Ceiling:
p50_seconds: float
p90_seconds: float
LOGS_CEILING: Final = Ceiling(p50_seconds=10, p90_seconds=20)
CHART_CEILING: Final = Ceiling(p50_seconds=20, p90_seconds=40)
TABLE_CEILING: Final = Ceiling(p50_seconds=5, p90_seconds=10)
UI_JOURNEY_LIMITS: Final = MappingProxyType(
{
"logs_open_24h": LOGS_CEILING,
"logs_last_7_days": LOGS_CEILING,
"logs_next_page": LOGS_CEILING,
"logs_filter_key_hash": LOGS_CEILING,
"logs_filter_key_alias": LOGS_CEILING,
"logs_filter_team": LOGS_CEILING,
"logs_filter_model": LOGS_CEILING,
"usage_open": CHART_CEILING,
"usage_last_30_days": CHART_CEILING,
"usage_team_view": CHART_CEILING,
"usage_customer_view": CHART_CEILING,
"usage_model_activity_30_days": CHART_CEILING,
"cache_open": CHART_CEILING,
"models_open": TABLE_CEILING,
"keys_open": TABLE_CEILING,
}
)
def assert_within_ceiling(journey_name: str, samples: tuple[Sample, ...]) -> None:
failed: Final = tuple(sample.error for sample in samples if sample.error)
assert not failed, f"{len(failed)} of {len(samples)} {journey_name} samples failed: {failed[0]}"
metrics: Final = dict(ui_junit_properties(samples))
ceiling: Final = UI_JOURNEY_LIMITS[journey_name]
assert metrics["perf_ui_seconds_p50"] <= ceiling.p50_seconds, (
f"{journey_name} p50 {metrics['perf_ui_seconds_p50']:.1f}s over {ceiling.p50_seconds}s"
)
assert metrics["perf_ui_seconds_p90"] <= ceiling.p90_seconds, (
f"{journey_name} p90 {metrics['perf_ui_seconds_p90']:.1f}s over {ceiling.p90_seconds}s"
)
@contextmanager
def dashboard_page(dashboard: Dashboard, timeout: float) -> Generator[Page]:
"""A logged-in admin tab whose clock reads the seed anchor, so relative windows such as
Last 24 Hours land on seeded rows. Live Tail is off so its 15 second refetch cannot land
inside a sample. Playwright keeps a UTC timezone to match the seed."""
with sync_playwright() as playwright:
browser: Final = playwright.chromium.launch()
context: Final = browser.new_context(
base_url=dashboard.base_url, timezone_id="UTC", viewport={"width": 1600, "height": 1000}
)
context.set_default_timeout(timeout * 1000)
context.add_init_script('sessionStorage.setItem("isLiveTail", "false")')
page: Final = context.new_page()
page.clock.set_fixed_time(dashboard.anchor)
_login(page, dashboard)
try:
yield page
finally:
context.close()
browser.close()
def _login(page: Page, dashboard: Dashboard) -> None:
_ = page.goto("/ui/login")
page.get_by_placeholder("Enter your username").fill(dashboard.username)
page.get_by_placeholder("Enter your password").fill(dashboard.password)
page.get_by_role("button", name="Login", exact=True).click()
page.wait_for_url(lambda url: "/ui" in url and "/login" not in url)
expect(page.get_by_label("LiteLLM home")).to_be_visible()
def _is_data_fetch(response: Response) -> bool:
"""API calls only. The Admin UI's own assets and Next.js prefetches live under /ui/ and /_next/."""
path: Final = urlsplit(response.url).path
return response.request.resource_type in ("fetch", "xhr") and not path.startswith(("/ui/", "/_next/"))
def _worst(responses: list[Response]) -> Response | None:
return next((response for response in responses if response.status >= 400), responses[-1] if responses else None)
def _with_response(sample: Sample, responses: list[Response]) -> Sample:
worst: Final = _worst(responses)
if worst is None:
return sample
return sample.model_copy(
update={"status": worst.status, "request": f"{worst.request.method} {urlsplit(worst.url).path}"[:120]}
)
def measure(page: Page, journey: Journey, timeout: float) -> Sample:
"""Wall time from the user's action to the data being on screen. Every sample starts from the
same pre-action state so repeats measure the same journey, not a warmed client cache."""
try:
journey.start(page, timeout)
except (PlaywrightError, AssertionError) as error:
return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300])
responses: Final[list[Response]] = [] # mutable-ok: filled by the playwright response listener
def record(response: Response) -> None:
if _is_data_fetch(response):
responses.append(response)
page.on("response", record)
started: Final = time.monotonic()
try:
with page.expect_response(_is_data_fetch, timeout=timeout * 1000):
journey.act(page)
rows: Final = journey.settled(page, timeout)
except PlaywrightTimeoutError:
return _with_response(Sample(seconds=time.monotonic() - started, error="timeout"), responses)
except (PlaywrightError, AssertionError) as error:
return _with_response(
Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]), responses
)
finally:
page.remove_listener("response", record)
return _with_response(Sample(seconds=time.monotonic() - started, rows=rows), responses)
def _visible(page: Page, test_id: str) -> Locator:
return page.get_by_test_id(test_id).filter(visible=True).first
def _logs_rows(page: Page) -> Locator:
return page.locator("table").filter(visible=True).first.locator("tbody tr")
def _logs_settled(page: Page, timeout: float) -> int:
millis: Final = timeout * 1000
rows: Final = _logs_rows(page).filter(has_not_text="No results")
expect(rows.first).to_be_visible(timeout=millis)
expect(_visible(page, "datatable-refresh")).to_be_enabled(timeout=millis)
expect(page.get_by_text(LOADING_LOGS)).to_have_count(0, timeout=millis)
return rows.count()
def _no_skeletons(page: Page, timeout: float) -> None:
expect(page.locator('[data-slot="skeleton"]')).to_have_count(0, timeout=timeout * 1000)
def _usage_settled(page: Page, timeout: float) -> int:
millis: Final = timeout * 1000
expect(page.get_by_text("Total Spend", exact=True).first).to_be_visible(timeout=millis)
_no_skeletons(page, timeout)
money: Final = page.get_by_text(MONEY)
expect(money.first).to_be_visible(timeout=millis)
return money.count()
def _cache_settled(page: Page, timeout: float) -> int:
millis: Final = timeout * 1000
expect(page.get_by_text("Cache Hits", exact=True)).to_be_visible(timeout=millis)
bars: Final = page.locator(".recharts-bar-rectangle")
expect(bars.first).to_be_visible(timeout=millis)
return bars.count()
def _table_settled(loading: str) -> Callable[[Page, float], int]:
def settled(page: Page, timeout: float) -> int:
millis: Final = timeout * 1000
rows: Final = page.locator("[data-row-id]")
expect(rows.first).to_be_visible(timeout=millis)
expect(page.get_by_text(loading)).to_have_count(0, timeout=millis)
return rows.count()
return settled
def _open_logs(page: Page, timeout: float) -> None:
_ = page.goto("/ui/logs")
_ = _logs_settled(page, timeout)
def _open_usage(page: Page, timeout: float) -> None:
_ = page.goto("/ui/usage")
_ = _usage_settled(page, timeout)
def _filters_drawer(page: Page) -> Locator:
_visible(page, "datatable-filters-trigger").click()
drawer: Final = page.get_by_role("dialog", name="Filters")
expect(drawer).to_be_visible()
return drawer
def _apply_filters(page: Page) -> None:
page.get_by_role("dialog", name="Filters").get_by_role("button", name="Apply Filters").click()
def _pick_first_option(page: Page, combobox: str) -> None:
_filters_drawer(page).get_by_role("combobox", name=combobox).click()
page.get_by_role("option").first.click()
def _open_time_range(page: Page, timeout: float) -> None:
_open_logs(page, timeout)
page.get_by_role("button", name="Last 24 Hours").click()
def _open_usage_range(page: Page, timeout: float) -> None:
_open_usage(page, timeout)
page.locator('[data-slot="advanced-date-picker-trigger"]').click()
def _model_activity_settled(page: Page, timeout: float) -> int:
millis: Final = timeout * 1000
expect(page.locator('[role="tab"][data-active]', has_text="Model Activity")).to_be_visible(timeout=millis)
_no_skeletons(page, timeout)
money: Final = page.get_by_text(MONEY).filter(visible=True)
expect(money.first).to_be_visible(timeout=millis)
return money.count()
def _open_model_activity_range(page: Page, timeout: float) -> None:
_open_usage(page, timeout)
page.get_by_role("tab", name="Model Activity").click()
_ = _model_activity_settled(page, timeout)
page.locator('[data-slot="advanced-date-picker-trigger"]').click()
def _pick_usage_preset(preset: str) -> Callable[[Page], None]:
def act(page: Page) -> None:
page.locator('[data-slot="advanced-date-picker-preset"]', has_text=preset).click()
return act
def _select_usage_view(page: Page, view: str) -> None:
page.locator('[data-slot="select-trigger"]').filter(has_text="Usage").first.click()
page.get_by_role("option", name=re.compile(rf"^{re.escape(view)}")).click()
def _nothing(page: Page, timeout: float) -> None:
return None
def _goto(path: str) -> Callable[[Page], None]:
def act(page: Page) -> None:
_ = page.goto(path)
return act
def _logs_filter(name: str, prepare: Callable[[Page], None]) -> Journey:
def start(page: Page, timeout: float) -> None:
_open_logs(page, timeout)
prepare(page)
return Journey(name, "logs", start, _apply_filters, _logs_settled)
def journeys(api_key: str) -> tuple[Journey, ...]:
return (
Journey("logs_open_24h", "logs", _nothing, _goto("/ui/logs"), _logs_settled),
Journey(
"logs_last_7_days",
"logs",
_open_time_range,
lambda page: page.get_by_role("button", name="Last 7 Days", exact=True).click(),
_logs_settled,
),
Journey(
"logs_next_page", "logs", _open_logs, lambda page: _visible(page, "pagination-next").click(), _logs_settled
),
_logs_filter(
"logs_filter_key_hash",
lambda page: _filters_drawer(page).get_by_placeholder("Enter key hash…").fill(api_key),
),
_logs_filter("logs_filter_key_alias", lambda page: _pick_first_option(page, "Search a key alias")),
_logs_filter("logs_filter_team", lambda page: _pick_first_option(page, "Search or select a team")),
_logs_filter("logs_filter_model", lambda page: _pick_first_option(page, "Search a model")),
Journey("usage_open", "usage", _nothing, _goto("/ui/usage"), _usage_settled),
Journey("usage_last_30_days", "usage", _open_usage_range, _pick_usage_preset("Last 30 days"), _usage_settled),
Journey(
"usage_team_view", "usage", _open_usage, lambda page: _select_usage_view(page, "Team Usage"), _usage_settled
),
Journey(
"usage_customer_view",
"usage",
_open_usage,
lambda page: _select_usage_view(page, "Customer Usage"),
_usage_settled,
),
Journey(
"usage_model_activity_30_days",
"usage",
_open_model_activity_range,
_pick_usage_preset("Last 30 days"),
_model_activity_settled,
),
Journey("cache_open", "caching", _nothing, _goto("/ui/caching"), _cache_settled),
Journey("models_open", "models", _nothing, _goto("/ui/models-and-endpoints"), _table_settled(LOADING_MODELS)),
Journey("keys_open", "keys", _nothing, _goto("/ui/api-keys"), _table_settled(LOADING_KEYS)),
)
def database_failures(text: str) -> float:
families: Final = tuple(text_string_to_metric_families(text))
required: Final = frozenset(("litellm_postgres_failed_requests", "litellm_batch_write_to_db_failed_requests"))
missing: Final = required - frozenset(family.name for family in families)
all_samples: Final = tuple(chain.from_iterable(family.samples for family in families))
has_activity: Final = any(
sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0
for sample in all_samples
)
assert not missing or has_activity, f"Missing PostgreSQL failure counters {missing}"
names: Final = frozenset(f"{name}_total" for name in required)
values: Final = tuple(sample.value for sample in all_samples if sample.name in names)
assert all(math.isfinite(value) and value >= 0 for value in values), "Invalid PostgreSQL failure counter"
return sum(values)

View file

@ -0,0 +1,186 @@
from __future__ import annotations
import hashlib
import os
import time
from collections.abc import Iterable
from datetime import datetime, timedelta
from itertools import chain
from typing import Final
import psycopg
from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter
from setup.ui_query_isolation import validate_target
from setup.upgrade_worker import read_state, release_indexes
class UiFixture(BaseModel):
model_config = ConfigDict(frozen=True)
anchor: AwareDatetime
api_key: str
"""`perf_ui_db_*` JUnit properties naming how much data the journeys ran against."""
volume: tuple[tuple[str, int], ...] = ()
class TableVolume(BaseModel):
model_config = ConfigDict(frozen=True)
table: str
rows: int
bytes: int
QUERIED_TABLES: Final = (
"LiteLLM_SpendLogs",
"LiteLLM_DailyUserSpend",
"LiteLLM_DailyTeamSpend",
"LiteLLM_DailyTagSpend",
"LiteLLM_VerificationToken",
"LiteLLM_TeamTable",
"LiteLLM_UserTable",
"LiteLLM_ProxyModelTable",
)
_VOLUME_SQL: Final = """
SELECT c.relname, GREATEST(c.reltuples, 0)::bigint, pg_total_relation_size(c.oid)
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relname = ANY(%(tables)s)
"""
SEEDED_TEAMS: Final = 100
SEEDED_USERS: Final = 2000
SEEDED_KEYS: Final = 5000
SEEDED_MODELS: Final = 200
DAILY_TABLES: Final = ("LiteLLM_DailyUserSpend", "LiteLLM_DailyTeamSpend")
_SEED_SQL: Final = (
"""
INSERT INTO "LiteLLM_TeamTable" (team_id, team_alias, members_with_roles, metadata, model_spend, model_max_budget)
SELECT 'perf-team-' || n, 'perf-team-' || n, '[]', '{}', '{}', '{}' FROM generate_series(0, %(teams)s - 1) AS n
ON CONFLICT (team_id) DO NOTHING
""",
"""
INSERT INTO "LiteLLM_UserTable" (user_id, user_alias, user_email, user_role, team_id, metadata, model_spend, model_max_budget)
SELECT 'perf-user-' || n, 'perf-user-' || n, 'perf-user-' || n || '@example.com', 'internal_user',
'perf-team-' || (n %% %(teams)s), '{}', '{}', '{}'
FROM generate_series(0, %(users)s - 1) AS n
ON CONFLICT (user_id) DO NOTHING
""",
"""
INSERT INTO "LiteLLM_VerificationToken" (token, key_name, key_alias, team_id, user_id, aliases, config, permissions,
metadata, model_spend, model_max_budget, created_at, updated_at)
SELECT md5('perf-key:' || n) || md5('perf-key-tail:' || n), 'sk-...perf' || n, 'perf-key-' || n,
'perf-team-' || (n %% %(teams)s), 'perf-user-' || (n %% %(users)s), '{}', '{}', '{}', '{}', '{}', '{}',
%(anchor)s - make_interval(days => n %% 365), %(anchor)s
FROM generate_series(0, %(keys)s - 1) AS n
ON CONFLICT (token) DO NOTHING
""",
"""
INSERT INTO "LiteLLM_ProxyModelTable" (model_id, model_name, litellm_params, model_info, created_by, updated_by)
SELECT 'perf-model-' || n, 'perf-chat-' || (n %% 20),
jsonb_build_object('model', 'openai/perf-chat', 'api_key', 'perf-seed',
'api_base', 'os.environ/E2E_PERF_MOCK_OPENAI_URL'),
jsonb_build_object('id', 'perf-model-' || n, 'db_model', true), 'perf-seed', 'perf-seed'
FROM generate_series(0, %(models)s - 1) AS n
ON CONFLICT (model_id) DO NOTHING
""",
)
def volume_properties(volumes: Iterable[TableVolume]) -> tuple[tuple[str, int], ...]:
"""One `perf_ui_db_rows_<table>` and one `perf_ui_db_bytes_<table>` property per table the journeys read,
so the dashboard can say what the latencies were measured against."""
return tuple(
chain.from_iterable(
((f"perf_ui_db_rows_{v.table}", v.rows), (f"perf_ui_db_bytes_{v.table}", v.bytes)) for v in volumes
)
)
def table_volumes(connection: psycopg.Connection, spend_log_rows: int) -> tuple[TableVolume, ...]:
"""Planner row estimates and on-disk size (indexes included) of the queried tables. The restore's
`LiteLLM_SpendLogs` count comes from the seed checkpoint, exact where the estimate can be stale."""
adapter: Final = TypeAdapter(tuple[str, int, int])
found: Final = tuple(
adapter.validate_python(row)
for row in connection.execute(_VOLUME_SQL, {"tables": list(QUERIED_TABLES)}).fetchall()
)
return tuple(
TableVolume(table=table, rows=spend_log_rows if table == "LiteLLM_SpendLogs" else rows, bytes=size)
for table, rows, size in sorted(found)
)
def prepare_fixture() -> UiFixture:
validate_target(os.environ["DATABASE_URL"])
_finish_upgrade()
with psycopg.connect(os.environ["DATABASE_URL"], connect_timeout=10) as connection:
_ = connection.execute("SET statement_timeout='30s'")
row: Final = connection.execute("SELECT spec->>'anchor', rows FROM litellm_perf_seed.checkpoint").fetchone()
assert row is not None and TypeAdapter(int).validate_python(row[1]) > 0, "Missing completed seed rows"
seed_ui_tables(connection, anchor=TypeAdapter(datetime).validate_python(row[0]))
fixture: Final = UiFixture(
anchor=TypeAdapter(datetime).validate_python(row[0]),
api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest()
+ hashlib.md5(b"perf-key-tail:0", usedforsecurity=False).hexdigest(),
volume=volume_properties(table_volumes(connection, TypeAdapter(int).validate_python(row[1]))),
)
return fixture
def seed_ui_tables(connection: psycopg.Connection, anchor: datetime) -> None:
"""Give the Keys, Teams, Users and Models pages realistic row counts on the disposable restore.
Names follow the SpendLogs seed (perf-key-N aliases, perf-team-N, perf-user-N) so the Logs filters and
Team Usage line up with real spend. The Daily* rollups come from the snapshot itself and are only checked.
"""
window: Final = {
"since": (anchor - timedelta(days=7)).strftime("%Y-%m-%d"),
"until": anchor.strftime("%Y-%m-%d"),
}
for table in DAILY_TABLES:
row = connection.execute(f'SELECT count(*) FROM "{table}" WHERE date BETWEEN %(since)s AND %(until)s', window)
assert _count(row.fetchone()) > 0, f"{table} has no rows in the week before the anchor"
params: Final = {
"teams": SEEDED_TEAMS,
"users": SEEDED_USERS,
"keys": SEEDED_KEYS,
"models": SEEDED_MODELS,
"anchor": anchor,
}
for statement in _SEED_SQL:
_ = connection.execute(statement, params)
connection.commit()
def _count(row: tuple[object, ...] | None) -> int:
assert row is not None
return TypeAdapter(int).validate_python(row[0])
def _released_schema(connection: psycopg.Connection) -> bool:
"""A released tag's migrations image upgrades the restore (schema and request-log indexes) before the
stack starts, so there is no upgrade worker state to wait on."""
row: Final = connection.execute("""SELECT to_regclass('public."_LiteLLMPerfUpgrade"') IS NULL""").fetchone()
return row is None or TypeAdapter(bool).validate_python(row[0])
def _finish_upgrade() -> None:
if os.environ.get("E2E_DATABASE_UPGRADE") != "1":
return
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True, connect_timeout=10) as connection:
if _released_schema(connection):
return
release_indexes(connection)
deadline: Final = time.monotonic() + 6 * 3600
while time.monotonic() < deadline:
match read_state(connection).phase:
case "failed":
raise AssertionError("Snapshot upgrade failed")
case "complete":
return
case _:
pass
time.sleep(5)
raise AssertionError("Snapshot indexes did not finish before UI measurements")

View file

@ -0,0 +1,112 @@
from __future__ import annotations
import json
import os
import re
import time
from collections.abc import Callable, Generator
from contextlib import contextmanager
from pathlib import Path
from typing import Final
from urllib.parse import urlparse
import psycopg
import pytest
from psycopg.rows import dict_row
from pydantic import BaseModel, ConfigDict, TypeAdapter
class ActiveQuery(BaseModel):
model_config = ConfigDict(frozen=True)
pid: int
leader_pid: int | None
application_name: str
state: str | None
wait_event_type: str | None
wait_event: str | None
seconds: float | None
query: str
def validate_target(url: str) -> None:
assert os.environ.get("E2E_UI_PERF_DISPOSABLE") == "1", "Disposable database confirmation required"
assert os.environ.get("E2E_RESET_SPEND_LOGS") != "1", "Never truncate the snapshot baseline"
assert re.fullmatch(r"db-perf-perf-[a-z0-9-]+\.litellm\.svc", urlparse(url).hostname or ""), (
"SQL cancellation is restricted to the disposable perf-lane restore"
)
def active(connection: psycopg.Connection[dict[str, object]]) -> tuple[ActiveQuery, ...]:
rows: Final = connection.execute("""SELECT pid, leader_pid, application_name, state,
wait_event_type, wait_event, extract(epoch from clock_timestamp()-query_start)::float AS seconds,
left(query, 1500) AS query FROM pg_stat_activity
WHERE datname = current_database() AND pid <> pg_backend_pid()
AND backend_type IN ('client backend', 'parallel worker')
AND state IS DISTINCT FROM 'idle' ORDER BY pid""").fetchall()
return TypeAdapter(tuple[ActiveQuery, ...]).validate_python(rows)
def wait_idle(connection: psycopg.Connection[dict[str, object]], evidence: Path, *, cancel: bool) -> None:
consecutive = 0
for queries in (active(connection) for _ in range(30)):
with evidence.open("a", encoding="utf-8") as output:
_ = output.write(
json.dumps(
{"time": time.time(), "cancel": cancel, "active": tuple(query.model_dump() for query in queries)}
)
+ "\n"
)
if queries:
consecutive = 0 # rebind-ok: consecutive idle observations reset on activity
for query in queries:
if cancel and query.leader_pid is None:
_ = connection.execute(
"SELECT pg_cancel_backend(%s)"
if query.state == "active"
else "SELECT pg_terminate_backend(%s)",
(query.pid,),
)
else:
consecutive += 1 # rebind-ok: consecutive idle observations across polling iterations
if consecutive == 3:
return
time.sleep(1)
pytest.exit("Database did not become idle. No further UI measurements will run", returncode=2)
def settled_counter(read: Callable[[], float], evidence: Path) -> float:
previous: float | None = None
consecutive = 0
for value in (read() for _ in range(10)):
with evidence.open("a", encoding="utf-8") as output:
_ = output.write(json.dumps({"time": time.time(), "database_failures": value}) + "\n")
consecutive = consecutive + 1 if value == previous else 1 # rebind-ok: polling state
if consecutive == 3:
return value
previous = value # rebind-ok: last observed counter for stabilization
time.sleep(1)
pytest.exit("Database counters did not settle. No further UI measurements will run", returncode=2)
@contextmanager
def isolated_database(name: str) -> Generator[None]:
url: Final = os.environ["DATABASE_URL"]
validate_target(url)
evidence: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) / f"ui-isolation-{name}.jsonl"
with psycopg.Connection[dict[str, object]].connect(
url,
autocommit=True,
connect_timeout=10,
application_name="ui-isolation-monitor",
options="-c statement_timeout=5000",
row_factory=dict_row,
) as connection:
try:
wait_idle(connection, evidence, cancel=False)
yield
finally:
try:
wait_idle(connection, evidence, cancel=True)
except Exception as error:
pytest.exit(f"SQL cleanup failed ({type(error).__name__}). Stopping UI measurements", returncode=2)

View file

@ -0,0 +1,223 @@
from __future__ import annotations
import math
import time
from collections.abc import Callable
from dataclasses import replace
from typing import Final
import psycopg
from cases.small_bodies_5rps import CASE
from e2e_config import MASTER_KEY
from e2e_http import send_text_external
from prometheus_client.parser import text_string_to_metric_families
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
from setup.performance_test_case import PerformanceTestCase
from setup.upgrade_worker import UpgradeState, fingerprints, missing_indexes, read_state, release_indexes
from setup.wire import WireRecord
MAX_TRAFFIC_SECONDS: Final = 6 * 3600
_ROWS: Final = TypeAdapter(tuple[tuple[str, str | None], ...])
_ACTIVITY: Final = TypeAdapter(tuple[list[int], int])
_COUNT: Final = TypeAdapter(tuple[int])
_SERVICES: Final = ("postgres", "batch_write_to_db")
def upgrade_case() -> PerformanceTestCase:
return replace(
CASE,
name="database_upgrade_5rps",
traffic=replace(CASE.traffic, total_requests=int(CASE.traffic.rps * MAX_TRAFFIC_SECONDS)),
limits=tuple(
limit
for limit in CASE.limits
if limit.metric
in {
"unexpected_status_count",
"gateway_added_seconds",
"time_to_first_token_seconds",
"memory_growth_megabytes",
}
),
)
class MetricsHeaders(BaseModel):
model_config = ConfigDict(frozen=True)
authorization: str = Field(default=f"Bearer {MASTER_KEY}", alias="Authorization")
def postgres_failures(text: str) -> float:
families: Final = tuple(text_string_to_metric_families(text))
names: Final = frozenset(family.name for family in families)
required: Final = frozenset(f"litellm_{service}_failed_requests" for service in _SERVICES)
samples: Final = tuple(sample for family in families for sample in family.samples)
has_database_activity: Final = any(
sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0
for sample in samples
)
assert required <= names or has_database_activity, f"Missing database service counters: {required - names}"
failures: Final = tuple(sample.value for sample in samples if sample.name in {f"{name}_total" for name in required})
assert all(math.isfinite(value) and value >= 0 for value in failures), "Invalid database failure counter"
return sum(failures)
def assert_database_metrics(urls: tuple[str, ...]) -> None:
assert urls, "E2E_DATABASE_METRICS_URLS must name every gateway/backend /metrics endpoint"
for url in urls:
assert_database_metrics_url(url)
def assert_database_metrics_url(url: str) -> None:
result: Final = send_text_external("GET", url, headers=MetricsHeaders(), timeout=10)
assert result.status_code == 200, f"Database metrics unavailable at {url}: {result.status_code}"
assert postgres_failures(result.body) == 0, f"Failed PostgreSQL operations reported at {url}"
def persisted(connection: psycopg.Connection[tuple[object, ...]], records: tuple[WireRecord, ...]) -> frozenset[str]:
ids: Final = tuple(record.call_id for record in records if record.call_id)
if not ids:
return frozenset()
rows: Final = _ROWS.validate_python(
connection.execute(
'SELECT request_id, litellm_call_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=ANY(%s)', (list(ids),)
).fetchall()
)
return matched_response_ids(records, rows)
def matched_response_ids(records: tuple[WireRecord, ...], rows: tuple[tuple[str, str | None], ...]) -> frozenset[str]:
call_ids: Final = frozenset(call_id for _, call_id in rows if call_id)
return frozenset(record.response_id for record in records if record.response_id and record.call_id in call_ids)
def activity(connection: psycopg.Connection[tuple[object, ...]]) -> tuple[list[int], int]:
return _ACTIVITY.validate_python(
connection.execute("""SELECT
ARRAY(SELECT pid FROM pg_stat_progress_create_index WHERE datname=current_database()),
(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock')
""").fetchone()
)
def assert_finished(connection: psycopg.Connection[tuple[object, ...]], state: UpgradeState) -> None:
assert state.phase == "complete", f"Upgrade did not finish: {state.phase}"
assert not missing_indexes(connection, state.required_indexes), "Required target indexes are missing or invalid"
(missing,) = _COUNT.validate_python(
connection.execute(
"""
SELECT count(*) FROM unnest(%s::text[]) name WHERE NOT EXISTS (
SELECT 1 FROM "_prisma_migrations" WHERE migration_name=name
AND finished_at IS NOT NULL AND rolled_back_at IS NULL)""",
(list(state.required_migrations),),
).fetchone()
)
assert missing == 0, f"{missing} target migrations were not applied"
(unfinished,) = _COUNT.validate_python(
connection.execute("""
SELECT count(*) FROM "_prisma_migrations" WHERE finished_at IS NULL AND rolled_back_at IS NULL
""").fetchone()
)
assert unfinished == 0, f"{unfinished} unfinished migrations"
assert fingerprints(connection, tuple(state.baseline_rows)) == state.baseline_rows, (
"Baseline rows changed or disappeared"
)
class UpgradeMonitor:
def __init__(
self,
*,
state: Callable[[], UpgradeState],
check_metrics: Callable[[], None],
read_activity: Callable[[], tuple[list[int], int]],
read_persisted: Callable[[tuple[WireRecord, ...]], frozenset[str]],
start_indexes: Callable[[], None],
clock: Callable[[], float] = time.monotonic,
wall_clock: Callable[[], float] = time.time,
) -> None:
self.state: Final = state
self.check_metrics: Final = check_metrics
self.read_activity: Final = read_activity
self.read_persisted: Final = read_persisted
self.start_indexes: Final = start_indexes
self.clock: Final = clock
self.wall_clock: Final = wall_clock
self.started: Final = clock()
self.status_limit: Final = next(
limit.max for limit in CASE.limits if limit.metric == "unexpected_status_count" and limit.max is not None
)
self.pending: tuple[WireRecord, ...] = ()
self.processed = 0
self.successes = 0
self.persisted_count = 0
self.overlap_writes = 0
self.max_lock_waiters = 0
self.previous_builds: frozenset[int] = frozenset()
self.build_seen_at: float | None = None
self.released = False
self.completed_at: float | None = None
def poll(self, records: tuple[WireRecord, ...]) -> bool:
self.check_metrics()
current: Final = self.state()
assert current.phase != "failed", "Target migration/index worker failed"
assert self.clock() - self.started < MAX_TRAFFIC_SECONDS, "Upgrade exceeded its six-hour operational timeout"
successful: Final = tuple(record for record in records if record.status == 200)
self.processed += len(records)
self.successes += len(successful)
assert self.processed - self.successes <= self.status_limit, "Too many unexpected non-200 responses"
assert all(record.ok and record.text for record in successful), "Failed or empty inference response"
assert all(record.response_id and record.call_id for record in successful), (
"Response missing spend-log identifiers"
)
builds, lock_waiters = self.read_activity()
if not self.previous_builds.intersection(builds):
self.build_seen_at = self.wall_clock() if builds else None
self.max_lock_waiters = max(self.max_lock_waiters, lock_waiters)
self.pending += successful
found: Final = self.read_persisted(self.pending)
self.persisted_count += len(found)
still_building, _ = self.read_activity()
since: Final = self.build_seen_at
if since is not None and frozenset(builds).intersection(still_building):
self.overlap_writes += sum(
1 for record in self.pending if record.response_id in found and record.started_at >= since
)
self.previous_builds = frozenset(builds)
self.pending = tuple(record for record in self.pending if record.response_id not in found)
if self.successes and not self.released:
self.start_indexes()
self.released = True
if current.phase == "complete" and self.completed_at is None:
self.completed_at = self.clock()
return self.completed_at is not None and self.clock() - self.completed_at >= 60 and self.successes >= 300
def assert_complete(self, state: UpgradeState) -> None:
assert self.successes >= 300, f"Only {self.successes} requests completed"
assert not self.pending, f"{len(self.pending)} successful requests have no matching spend log"
assert self.persisted_count == self.successes, "Spend-log count does not match successful requests"
assert not state.pending_indexes or self.overlap_writes > 0, "No persisted requests observed during index work"
def upgrade_properties(state: UpgradeState, monitor: UpgradeMonitor) -> tuple[tuple[str, float], ...]:
"""The upgrade's own numbers under the `perf_` prefix the dashboard ingests, next to the perf rows."""
return (
("perf_upgrade_schema_seconds", state.schema_seconds),
("perf_upgrade_index_seconds", state.index_seconds),
("perf_upgrade_overlap_writes", float(monitor.overlap_writes)),
("perf_upgrade_max_lock_waiters", float(monitor.max_lock_waiters)),
("perf_upgrade_pending_indexes", float(len(state.pending_indexes))),
("perf_upgrade_unmatched_requests", float(len(monitor.pending))),
)
def monitor_for(connection: psycopg.Connection[tuple[object, ...]], urls: tuple[str, ...]) -> UpgradeMonitor:
return UpgradeMonitor(
state=lambda: read_state(connection),
check_metrics=lambda: assert_database_metrics(urls),
read_activity=lambda: activity(connection),
read_persisted=lambda records: persisted(connection, records),
start_indexes=lambda: release_indexes(connection),
)

View file

@ -0,0 +1,146 @@
from __future__ import annotations
import json
import os
import sys
import time
from pathlib import Path
from typing import Final, Literal
import psycopg
from pydantic import BaseModel, ConfigDict, TypeAdapter
class UpgradeState(BaseModel):
model_config = ConfigDict(frozen=True)
phase: Literal["schema", "schema_ready", "indexes", "complete", "failed"]
started_at: float
schema_seconds: float = 0
index_seconds: float = 0
required_indexes: tuple[str, ...]
required_migrations: tuple[str, ...]
pending_indexes: tuple[str, ...]
baseline_rows: dict[str, str]
_STATE: Final = TypeAdapter(tuple[UpgradeState])
_BOOL: Final = TypeAdapter(tuple[bool])
_STRINGS: Final = TypeAdapter(tuple[tuple[str], ...])
_FINGERPRINTS: Final = TypeAdapter(tuple[tuple[str, str], ...])
_FINGERPRINT: Final = """md5(ROW(request_id, api_key, spend, total_tokens, prompt_tokens,
completion_tokens, model, "startTime", "endTime")::text)"""
def connect() -> psycopg.Connection[tuple[object, ...]]:
return psycopg.connect(
os.environ["DATABASE_URL"],
autocommit=True,
connect_timeout=10,
options="-c statement_timeout=10000 -c timezone=UTC",
)
def fingerprints(connection: psycopg.Connection[tuple[object, ...]], ids: tuple[str, ...]) -> dict[str, str]:
rows: Final = connection.execute(
f'SELECT request_id, {_FINGERPRINT} FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s)', (list(ids),)
).fetchall()
return dict(_FINGERPRINTS.validate_python(rows))
def missing_indexes(connection: psycopg.Connection[tuple[object, ...]], names: tuple[str, ...]) -> tuple[str, ...]:
rows: Final = connection.execute(
"""SELECT name FROM unnest(%s::text[]) AS name WHERE NOT EXISTS (
SELECT 1 FROM pg_index WHERE indexrelid = to_regclass(format('public.%%I', name))
AND indisvalid AND indisready)""",
(list(names),),
).fetchall()
return tuple(row[0] for row in _STRINGS.validate_python(rows))
def read_state(connection: psycopg.Connection[tuple[object, ...]]) -> UpgradeState:
(state,) = _STATE.validate_python(
connection.execute('SELECT state FROM "_LiteLLMPerfUpgrade" WHERE id=1').fetchone()
)
return state
def write_state(connection: psycopg.Connection[tuple[object, ...]], state: UpgradeState) -> None:
_ = connection.execute(
"""INSERT INTO "_LiteLLMPerfUpgrade" (id, state) VALUES (1, %s::jsonb)
ON CONFLICT (id) DO UPDATE SET state = EXCLUDED.state""",
(state.model_dump_json(),),
)
def release_indexes(connection: psycopg.Connection[tuple[object, ...]]) -> None:
_ = connection.execute('UPDATE "_LiteLLMPerfUpgrade" SET traffic_ready=true WHERE id=1')
def wait_for_traffic(connection: psycopg.Connection[tuple[object, ...]], deadline: float) -> bool:
while time.monotonic() < deadline:
(ready,) = _BOOL.validate_python(
connection.execute('SELECT traffic_ready FROM "_LiteLLMPerfUpgrade" WHERE id=1').fetchone()
)
if ready:
return True
time.sleep(1)
return False
def main() -> int:
assert os.environ.get("E2E_DATABASE_UPGRADE") == "1", "Only run against a disposable restored baseline"
assert os.environ.get("DATABASE_PERF_CONFIRM_DISPOSABLE") == "1", "Disposable database confirmation required"
from litellm_proxy_extras import request_log_indexes
from litellm_proxy_extras.utils import ProxyExtrasDBManager
with connect() as connection:
_ = connection.execute("""CREATE TABLE "_LiteLLMPerfUpgrade" (
id integer PRIMARY KEY CHECK (id=1), state jsonb NOT NULL, traffic_ready boolean NOT NULL DEFAULT false)""")
sample: Final = connection.execute("""
(SELECT request_id FROM "LiteLLM_SpendLogs" ORDER BY request_id LIMIT 32)
UNION (SELECT request_id FROM "LiteLLM_SpendLogs" ORDER BY request_id DESC LIMIT 32)""").fetchall()
ids: Final = tuple(row[0] for row in _STRINGS.validate_python(sample))
assert ids, "Restored baseline has no spend logs"
names: Final = tuple(index.name for index in request_log_indexes.REQUEST_LOG_INDEXES)
migrations: Final = Path(request_log_indexes.__file__).parent / "migrations"
initial: Final = UpgradeState(
phase="schema",
started_at=time.time(),
required_indexes=names,
required_migrations=tuple(sorted(path.parent.name for path in migrations.glob("*/migration.sql"))),
pending_indexes=missing_indexes(connection, names),
baseline_rows=fingerprints(connection, ids),
)
assert initial.required_migrations and initial.required_indexes, "Target schema requirements are missing"
write_state(connection, initial)
try:
assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True), (
"Schema migration failed"
)
ready: Final = initial.model_copy(
update={
"phase": "schema_ready",
"schema_seconds": time.time() - initial.started_at,
"pending_indexes": missing_indexes(connection, names),
}
)
write_state(connection, ready)
assert wait_for_traffic(connection, time.monotonic() + 1800), "Traffic did not start within 30 minutes"
write_state(connection, ready.model_copy(update={"phase": "indexes"}))
index_start: Final = time.monotonic()
assert ProxyExtrasDBManager.build_request_log_indexes(), "Request-log index build failed"
complete: Final = ready.model_copy(
update={"phase": "complete", "index_seconds": time.monotonic() - index_start}
)
write_state(connection, complete)
_ = sys.stdout.write(complete.model_dump_json() + "\n")
return 0
except Exception as exc:
write_state(connection, read_state(connection).model_copy(update={"phase": "failed"}))
_ = sys.stderr.write(json.dumps({"error": type(exc).__name__}) + "\n")
return 1
if __name__ == "__main__":
raise SystemExit(main())

View file

@ -9,12 +9,13 @@ from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
from setup.corpus.session import Endpoint as CorpusEndpoint
from setup.corpus.session import request_body as corpus_body
Endpoint = Literal["chat_completions", "responses", "messages"]
_PAYLOAD: Final = TypeAdapter(dict[str, object])
class TokenSource(BaseModel):
@ -50,6 +51,8 @@ class WireRequest(BaseModel):
token_source: TokenSource | None = None
stream: bool
blocked: bool
send_at_s: float | None = None
"""Seconds after the run starts to send this request, set when the case ramps; otherwise the users pace."""
class WireRecord(BaseModel):
@ -70,6 +73,35 @@ class WireRecord(BaseModel):
mock_delay_s: float | None
call_id: str | None
error: str | None
response_id: str | None = None
litellm_overhead_ms: float | None = None
litellm_response_ms: float | None = None
def response_id(payload: Mapping[str, object]) -> str | None:
nested: Final = payload.get("response") or payload.get("message")
source: Final = _PAYLOAD.validate_python(nested) if isinstance(nested, Mapping) else payload
identifier: Final = source.get("id")
return identifier if isinstance(identifier, str) else None
class PlanProgress:
def __init__(self, total: int) -> None:
self.total: Final = total
self.issued = 0
self.completed = 0
self.stopped = False
def take(self, *, stop: bool) -> bool:
self.stopped = self.stopped or stop
if self.stopped or self.issued >= self.total:
return False
self.issued += 1
return True
@property
def drained(self) -> bool:
return (self.stopped or self.issued == self.total) and self.completed == self.issued
_CORPUS_ENDPOINT: Final[Mapping[Endpoint, CorpusEndpoint]] = MappingProxyType(

View file

@ -0,0 +1,102 @@
from __future__ import annotations
import os
import sys
import time
from pathlib import Path
from typing import Final
import pytest
from idp import Keycloak
from lifecycle import ResourceManager
from memory_readings import read_rss_everywhere
from proxy_client import ProxyClient
from setup.datastore import read_db_counters, read_redis_counters
from setup.observe import (
Observation,
evaluate,
junit_properties,
read_mock_requests,
render,
reset_mock,
set_mock_latency,
)
from setup.plan import build_plan
from setup.podcpu import gateway_cpu_sampler
from setup.provision import MOCK_PROVIDERS, build_setup, mock_url
from setup.run import run_plan
from setup.upgrade import assert_finished, monitor_for, upgrade_case, upgrade_properties
from setup.upgrade_worker import connect, read_state
pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack]
@pytest.mark.skipif(os.environ.get("E2E_DATABASE_UPGRADE") != "1", reason="Dedicated disposable snapshot lane only")
def test_database_upgrade(
proxy: ProxyClient, idp: Keycloak, resources: ResourceManager, request: pytest.FixtureRequest
) -> None:
case: Final = upgrade_case()
urls: Final = tuple(mock_url(provider) for provider in MOCK_PROVIDERS)
for url in urls:
_ = reset_mock(url)
_ = set_mock_latency(url, case.mock_latency)
setup: Final = build_setup(case, proxy=proxy, idp=idp, resources=resources)
metrics: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip())
with connect() as connection:
assert read_state(connection).phase == "schema_ready", "Expected the target schema ready, index work waiting"
rss_before: Final = read_rss_everywhere(proxy)
db_before: Final = read_db_counters()
redis_before: Final = read_redis_counters()
monitor: Final = monitor_for(connection, metrics)
with gateway_cpu_sampler() as cpu:
outcome: Final = run_plan(case, build_plan(case), setup, stop_when=monitor.poll, request_timeout=30)
assert outcome.locust_exit == 0, outcome.locust_output
remaining: Final = outcome.records[monitor.processed :]
_ = monitor.poll(remaining)
deadline: Final = time.monotonic() + 120
while monitor.pending and time.monotonic() < deadline:
time.sleep(1)
_ = monitor.poll(())
state: Final = read_state(connection)
_ = sys.stdout.write(
f"successful_requests={monitor.successes} persisted_requests={monitor.persisted_count} "
f"unmatched_requests={len(monitor.pending)}\n"
)
rss_after: Final = read_rss_everywhere(proxy)
observed: Final = Observation(
records=outcome.records,
mock_requests=tuple(seen for url in urls for seen in read_mock_requests(url)),
guardrail_calls=(),
rss_before=rss_before,
rss_after=rss_after,
wall_seconds=outcome.wall_seconds,
spend_logged_share=1,
db_before=db_before,
db_after=read_db_counters(),
redis_before=redis_before,
redis_after=read_redis_counters(),
gateway_cpu_seconds=cpu.seconds,
)
rows, exceeded = evaluate(case, observed)
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
assert isinstance(item, pytest.Item)
item.user_properties[:] = [
entry for entry in item.user_properties if not entry[0].startswith("perf_")
] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set
item.user_properties.extend(junit_properties(rows))
item.user_properties.extend(upgrade_properties(state, monitor))
_ = sys.stdout.write(render(case, observed, rows) + "\n")
_ = sys.stdout.write(
f"schema_seconds={state.schema_seconds:.3f} index_seconds={state.index_seconds:.3f} "
f"overlap_writes={monitor.overlap_writes} max_lock_waiters={monitor.max_lock_waiters} "
f"pending_indexes={state.pending_indexes}\n"
)
artifact: Final = os.environ.get("E2E_ARTIFACT_DIR")
if artifact:
_ = (Path(artifact) / "database-upgrade.json").write_text(state.model_dump_json(), encoding="utf-8")
monitor.assert_complete(state)
assert_finished(connection, state)
assert {(r.replica, r.hostname) for r in rss_before.readings} == {
(r.replica, r.hostname) for r in rss_after.readings
}, "Gateway pods replaced during the upgrade"
assert not exceeded, "\n".join(exceeded)

View file

@ -5,7 +5,10 @@ clients saw, what the mocks received and how the workers' memory moved."""
from __future__ import annotations
import os
import time
from collections.abc import Mapping
from datetime import UTC, datetime
from typing import Final
import pytest
@ -15,7 +18,18 @@ from lifecycle import ResourceManager
from memory_readings import read_rss_everywhere
from models import SpendLogsParams
from proxy_client import ProxyClient
from setup.datastore import read_db_counters, read_redis_counters, redis_monitor
from setup.datastore import connections_sampler, read_db_counters, read_redis_counters, redis_monitor
from setup.event_loop import (
LOG_FILE_ENV,
OPT_IN_ENV,
StallRow,
offenders,
parse_stalls,
read_gateway_log,
render_stalls,
summarize,
)
from setup.event_loop import junit_properties as stall_properties
from setup.observe import (
Observation,
evaluate,
@ -26,14 +40,17 @@ from setup.observe import (
reset_mock,
set_mock_latency,
)
from setup.performance_test_case import PerformanceTestCase
from setup.plan import build_plan
from setup.performance_test_case import MockLatency, PerformanceTestCase
from setup.plan import PlannedRequest, build_plan
from setup.podcpu import gateway_cpu_sampler
from setup.provision import MOCK_PROVIDERS, build_setup, guardrail_url, mock_url
from setup.ramp import step_properties
from setup.run import run_plan
from setup.telemetry import read_gateway_telemetry, telemetry_sampler
pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack]
_SPEND_SAMPLE: Final = 40
_WARMUP_SECONDS: Final = 60
def _spend_logged_share(proxy: ProxyClient, call_ids: tuple[str, ...]) -> float | None:
@ -48,6 +65,31 @@ def _spend_logged_share(proxy: ProxyClient, call_ids: tuple[str, ...]) -> float
return found / len(call_ids)
def _warmup_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...]) -> tuple[PlannedRequest, ...]:
"""The plan's first minute of traffic. It is sent before the memory baseline is read so a worker
has shed its startup allocations and `memory_growth_megabytes` is what the measured traffic added,
not startup churn (a lightly loaded case read negative growth)."""
return plan[: max(1, round(case.traffic.rps * _WARMUP_SECONDS))]
def _arm_mocks(mock_urls: Mapping[str, str], latency: MockLatency) -> None:
for url in (*mock_urls.values(), guardrail_url()):
_ = reset_mock(url)
for url in mock_urls.values():
_ = set_mock_latency(url, latency)
def _event_loop_rows(case: PerformanceTestCase, since: datetime) -> tuple[StallRow, ...]:
"""The gateway's asyncio debug holds for the traffic window; a case that asks for them cannot pass without a log."""
assert case.event_loop is not None
lines: Final = read_gateway_log(since)
assert lines is not None, (
f"{case.name} needs the gateway log: run in the perf lane (pods/log with the runner service account) or set "
f"{LOG_FILE_ENV} to the gateway's log file"
)
return summarize(case.event_loop, parse_stalls(lines))
@pytest.mark.parametrize("case", CASES, ids=[case.name for case in CASES])
def test_perf_case(
case: PerformanceTestCase,
@ -56,24 +98,36 @@ def test_perf_case(
resources: ResourceManager,
request: pytest.FixtureRequest,
) -> None:
if case.event_loop is not None and not os.environ.get(OPT_IN_ENV):
pytest.skip(
f"{case.name} reads asyncio debug output: deploy the gateway with PYTHONASYNCIODEBUG=1 and "
f"LITELLM_EVENT_LOOP=asyncio, then set {OPT_IN_ENV}=1"
)
plan: Final = build_plan(case)
mock_urls: Final = {provider: mock_url(provider) for provider in MOCK_PROVIDERS}
for url in (*mock_urls.values(), guardrail_url()):
_ = reset_mock(url)
for url in mock_urls.values():
_ = set_mock_latency(url, case.mock_latency)
_arm_mocks(mock_urls, case.mock_latency)
setup_started: Final = time.monotonic()
setup: Final = build_setup(case, proxy=proxy, idp=idp, resources=resources)
_ = run_plan(case, _warmup_plan(case, plan), setup)
_arm_mocks(mock_urls, case.mock_latency)
rss_before: Final = read_rss_everywhere(proxy)
telemetry_before: Final = read_gateway_telemetry(proxy)
db_before: Final = read_db_counters()
redis_before: Final = read_redis_counters()
run_started: Final = time.monotonic()
with gateway_cpu_sampler() as cpu, redis_monitor() as monitor:
traffic_started_at: Final = datetime.now(tz=UTC)
with (
gateway_cpu_sampler() as cpu,
redis_monitor() as monitor,
connections_sampler() as connections,
telemetry_sampler(proxy) as telemetry,
):
outcome: Final = run_plan(case, plan, setup)
run_finished: Final = time.monotonic()
db_after: Final = read_db_counters()
redis_after: Final = read_redis_counters()
rss_after: Final = read_rss_everywhere(proxy)
telemetry_after: Final = read_gateway_telemetry(proxy)
rss_read: Final = time.monotonic()
assert outcome.records, f"locust wrote no records (exit {outcome.locust_exit}):\n{outcome.locust_output}"
served_call_ids: Final = tuple(
@ -100,15 +154,28 @@ def test_perf_case(
redis_after=redis_after,
redis_commands=monitor.commands,
gateway_cpu_seconds=cpu.seconds,
telemetry_before=telemetry_before,
telemetry_after=telemetry_after,
peak_connections=connections.peak,
ramp=case.traffic.ramp,
cpu_series=cpu.series,
connection_series=connections.series,
telemetry_series=telemetry.series,
)
rows, exceeded_limits = evaluate(case, observation)
stall_rows: Final = _event_loop_rows(case, traffic_started_at) if case.event_loop is not None else ()
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
assert isinstance(item, pytest.Item)
item.user_properties[:] = [
entry for entry in item.user_properties if not entry[0].startswith("perf_")
] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set
item.user_properties.extend(junit_properties(rows))
item.user_properties.extend(
stall_properties(case.event_loop, stall_rows) if case.event_loop is not None else junit_properties(rows)
)
item.user_properties.extend(step_properties(observation.steps()))
print(render(case, observation, rows))
if case.event_loop is not None:
print(render_stalls(case.event_loop, stall_rows))
print(
f" phases: setup {run_started - setup_started:.0f}s, locust {run_finished - run_started:.0f}s, "
f"observe {time.monotonic() - run_finished:.0f}s (rss {rss_read - run_finished:.0f}s, spend sample "
@ -118,3 +185,4 @@ def test_perf_case(
f"{len(plan)} planned, {len(outcome.records)} recorded; locust said:\n{outcome.locust_output}"
)
assert not exceeded_limits, "limits exceeded (observed > max):\n" + "\n".join(exceeded_limits)
assert not offenders(stall_rows), "event-loop holds over the threshold:\n" + "\n".join(offenders(stall_rows))

View file

@ -0,0 +1,137 @@
from __future__ import annotations
import json
import os
import signal
import time
from collections.abc import Generator
from datetime import UTC, datetime
from pathlib import Path
from types import FrameType
from typing import Final
import pytest
from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME
from e2e_http import send_text_external, without_retries
from playwright.sync_api import Page
from setup.ui_journeys import (
Dashboard,
Journey,
Sample,
assert_within_ceiling,
dashboard_page,
database_failures,
error_code,
journeys,
measure,
summarize,
ui_junit_properties,
)
from setup.ui_query_fixture import UiFixture, prepare_fixture
from setup.ui_query_isolation import isolated_database, settled_counter
from transport import HttpTransport
pytestmark = [
pytest.mark.perf,
pytest.mark.quiet_stack,
pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"),
]
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if "journey_name" in metafunc.fixturenames:
metafunc.parametrize("journey_name", [journey.name for journey in journeys("")])
@pytest.fixture(scope="module")
def ui_fixture() -> UiFixture:
return prepare_fixture()
def _metrics(transport: HttpTransport) -> float:
urls: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip())
try:
assert urls, "Provide a metrics URL for every proxy worker/pod"
readings: Final = tuple(send_text_external("GET", url, headers=transport.master, timeout=10) for url in urls)
assert all(reading.status_code == 200 for reading in readings), "Cannot read database failure metrics"
return sum(database_failures(reading.body) for reading in readings)
except Exception as error:
pytest.exit(f"Database metrics unavailable ({type(error).__name__}). Stopping UI measurements", returncode=2)
def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]:
with dashboard_page(dashboard, timeout) as page:
return tuple(_within_budget(page, journey, timeout, repeats))
def _within_budget(page: Page, journey: Journey, timeout: float, repeats: int) -> Generator[Sample]:
"""One hanging browser call must not eat the lane: SIGALRM ends the journey at timeout x (repeats + 2),
the budget every sample needs (start plus act plus settle) with one spare sample of slack."""
budget: Final = timeout * (repeats + 2)
def expire(signum: int, frame: FrameType | None) -> None:
raise TimeoutError(f"{journey.name} exceeded its {budget:.0f}s wall clock")
previous: Final = signal.signal(signal.SIGALRM, expire)
started: Final = time.monotonic()
try:
_ = signal.setitimer(signal.ITIMER_REAL, budget)
for _ in range(repeats + 1):
yield measure(page, journey, timeout)
except Exception as error:
if time.monotonic() - started < budget:
raise
yield Sample(seconds=time.monotonic() - started, error=f"wall_clock {type(error).__name__}: {error}"[:300])
finally:
_ = signal.setitimer(signal.ITIMER_REAL, 0)
_ = signal.signal(signal.SIGALRM, previous)
def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None:
journey: Final = next(journey for journey in journeys(ui_fixture.api_key) if journey.name == journey_name)
transport: Final = HttpTransport(base_url=UI_BASE_URL, master_key=MASTER_KEY)
timeout: Final = float(os.environ.get("E2E_UI_JOURNEY_TIMEOUT_SECONDS", "120"))
repeats: Final = int(os.environ.get("E2E_UI_JOURNEY_REPEATS", "9"))
assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings"
target: Final = Path(os.environ["E2E_ARTIFACT_DIR"])
assert target.is_dir(), "E2E_ARTIFACT_DIR must exist"
evidence: Final = target / f"ui-isolation-{journey.name}.jsonl"
dashboard: Final = Dashboard(
base_url=UI_BASE_URL, username=UI_USERNAME, password=UI_PASSWORD, anchor=ui_fixture.anchor
)
with without_retries():
with isolated_database(journey.name):
before: Final = settled_counter(lambda: _metrics(transport), evidence)
samples: Final = _samples(dashboard, journey, timeout, repeats)
with evidence.open("a", encoding="utf-8") as sample_output:
_ = sample_output.write(
json.dumps({"samples": tuple(sample.model_dump() for sample in samples)}) + "\n"
)
after_requests: Final = _metrics(transport)
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
assert isinstance(item, pytest.Item)
item.user_properties.extend(ui_junit_properties(samples))
item.user_properties.extend(ui_fixture.volume)
after: Final = settled_counter(lambda: _metrics(transport), evidence)
item.user_properties.extend(
(
("perf_ui_database_failures", after_requests - before),
("perf_ui_cleanup_database_failures", after - after_requests),
)
)
artifact: Final = {
"name": journey.name,
"surface": journey.surface,
"measured_at": datetime.now(UTC).isoformat(),
"samples": [sample.model_dump() for sample in samples],
"error_codes": [error_code(sample) for sample in samples],
"timings": summarize(samples).model_dump(),
"timeouts": sum(sample.error == "timeout" for sample in samples),
"failures": sum(bool(sample.error) for sample in samples),
"database_failures_before": before,
"database_failures_after_requests": after_requests,
"database_failures_after_cleanup": after,
}
output: Final = target / f"ui-journey-{journey.name}.json"
_ = output.write_text(json.dumps(artifact, indent=2), encoding="utf-8")
assert_within_ceiling(journey.name, samples)

View file

@ -14,7 +14,6 @@ markers =
cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set
redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_PERF_TESTS is set
quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; every other test waits for it to finish
mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set
provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set
owned_gateway: boots its own proxy from source against the stack's Postgres, so it needs DATABASE_URL on the pytest host; deselected unless E2E_OWNED_GATEWAY is set
otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set

View file

@ -32,7 +32,6 @@ export const menuLabelToPage: Record<string, Page> = {
Budgets: Page.Budgets,
"API Playground": Page.TransformRequest,
"Tag Management": Page.TagManagement,
"Old Usage": Page.Usage,
// Tools submenu items
"MCP Servers": Page.McpServers,
"Vector Stores": Page.VectorStores,

View file

@ -118,12 +118,6 @@ export const MIGRATED_E2E_PAGES: Readonly<Record<string, MigratedPage>> = {
content: { role: "heading", name: "AI Hub" },
},
new_usage: { segment: "usage", linkName: "Usage", content: { role: "heading", name: "Usage" } },
usage: {
segment: "old-usage",
linkName: "Old Usage",
group: "Experimental",
content: { role: "tab", name: "All Up" },
},
agents: { segment: "agents", linkName: "Agents", group: "Agentic", content: { role: "heading", name: "Agents" } },
"router-settings": {
segment: "router-settings",

View file

@ -30,5 +30,4 @@ export enum Page {
TagManagement = "tag-management",
VectorStores = "vector-stores",
NewUsage = "new_usage",
Usage = "usage",
}

View file

@ -2,7 +2,7 @@ import { chromium, expect, request } from "@playwright/test";
import { users, Role, STORAGE_PATHS } from "./fixtures/users";
import { ARTIFACT_DIR, UI_BASE_URL } from "./constants";
import { expectUnrestrictedDashboard, setInvitedUserPassword } from "./helpers/userOnboarding";
import { hideLiteAdmin } from "./helpers/navigation";
import { setAriaHidden } from "./helpers/navigation";
import { masterKey as getMasterKey } from "./helpers/traffic";
import * as fs from "fs";
import * as path from "path";
@ -78,7 +78,7 @@ async function globalSetup() {
await dismiss.click();
}
if (role === Role.ProxyAdmin) {
await hideLiteAdmin(page);
await setAriaHidden(page, true);
}
// The login flow stores a post-login return URL in the litellm_return_url
// cookie. If the snapshot captures it before the app consumes it, every

View file

@ -0,0 +1,20 @@
import { Page as PlaywrightPage, expect } from "@playwright/test";
import { Page } from "../fixtures/pages";
import { dismissFeedbackPopup, navigateToPage } from "./navigation";
export async function deleteGuardrailFromTable(page: PlaywrightPage, guardrailName: string): Promise<void> {
await navigateToPage(page, Page.Guardrails);
await dismissFeedbackPopup(page);
const row = page.getByRole("row").filter({ hasText: guardrailName });
await expect(row).toHaveCount(1, { timeout: 15_000 });
await row.getByRole("button", { name: "Open guardrail actions" }).click();
await page.getByRole("menuitem", { name: "Delete" }).click();
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
await deleteModal.getByPlaceholder(guardrailName, { exact: true }).fill(guardrailName);
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 });
await expect(row).toHaveCount(0, { timeout: 15_000 });
}

View file

@ -62,18 +62,29 @@ export async function dismissFeedbackPopup(page: PlaywrightPage): Promise<void>
}
}
export async function hideLiteAdmin(page: PlaywrightPage): Promise<void> {
export async function setAriaHidden(page: PlaywrightPage, hidden: boolean): Promise<void> {
await page.getByRole("button", { name: /Account menu/i }).click();
const panel = page.getByTestId("sidebar-account-menu-panel");
await expect(panel).toBeVisible({ timeout: 5_000 });
const toggle = panel.getByRole("switch", { name: "Toggle hide LiteAdmin" });
if ((await toggle.getAttribute("aria-checked")) !== "true") {
await toggle.click();
const premium = await panel.getByText("Premium", { exact: true }).isVisible();
const toggle = panel.getByRole("switch", { name: "Toggle hide Aria Agent", exact: true });
if (premium) {
await expect(toggle).toBeVisible();
if ((await toggle.getAttribute("aria-checked")) !== String(hidden)) {
await toggle.click();
}
await expect(toggle).toHaveAttribute("aria-checked", String(hidden));
} else {
await expect(toggle).toHaveCount(0);
}
await expect(toggle).toHaveAttribute("aria-checked", "true");
await page.keyboard.press("Escape");
await expect(panel).toBeHidden();
await expect(page.getByRole("button", { name: "LiteAdmin", exact: true })).toBeHidden();
const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true });
if (premium && hidden) {
await expect(aria).toBeHidden();
} else {
await expect(aria).toBeVisible();
}
}
/**

View file

@ -2,42 +2,57 @@ import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/t
import { navigateToPage, dismissFeedbackPopup } from "./navigation";
import { Page } from "../fixtures/pages";
/** Controls for the Test Key / Playground page, shared with the router-fallback specs. */
/**
* The configuration panel is rendered twice, docked and overlay, with one visible at a time.
* Every control is narrowed to the visible copy or it trips strict mode against its hidden twin.
*/
export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first();
/** The model combobox, addressed by the placeholder its search input shows before selection. */
export const modelSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByPlaceholder("Select a Model"));
export const composer = (page: PlaywrightPage): Locator =>
onlyVisible(page.getByRole("group").filter({ has: page.getByRole("textbox", { name: "Message" }) }));
export const modelSelect = (page: PlaywrightPage): Locator =>
onlyVisible(page.getByRole("combobox", { name: "Model", exact: true }));
export const sendButton = (page: PlaywrightPage): Locator =>
onlyVisible(page.getByRole("button", { name: "Send message" }));
composer(page).getByRole("button", { name: "Send message" });
/** The Virtual Key Source dropdown, addressed by the accessible name on its trigger. */
export const keySourceSelect = (page: PlaywrightPage): Locator =>
onlyVisible(page.getByLabel("Virtual Key Source"));
export const keySourceSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByLabel("Send as"));
export async function openPlayground(page: PlaywrightPage): Promise<void> {
await navigateToPage(page, Page.LlmPlayground);
await dismissFeedbackPopup(page);
await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({
timeout: 20_000,
});
await expect(modelSelect(page)).toBeVisible({ timeout: 20_000 });
const searchHint = page.getByRole("button", { name: "Dismiss search hint" });
if (await searchHint.isVisible()) {
await searchHint.click();
}
}
export async function openRequestSettings(page: PlaywrightPage): Promise<void> {
await composer(page).getByRole("button", { name: "Settings", exact: true }).click();
await expect(keySourceSelect(page)).toBeVisible({ timeout: 10_000 });
}
export async function enterVirtualKey(page: PlaywrightPage, key: string): Promise<void> {
await openRequestSettings(page);
await keySourceSelect(page).click();
await onlyVisible(page.getByRole("option", { name: "Virtual key", exact: true })).click({ timeout: 15_000 });
const keyInput = onlyVisible(page.getByLabel("Virtual key", { exact: true }));
await expect(keyInput).toBeVisible({ timeout: 10_000 });
await keyInput.fill(key);
await page.keyboard.press("Escape");
}
export async function searchModels(page: PlaywrightPage, query: string): Promise<void> {
await modelSelect(page).click();
await onlyVisible(page.getByPlaceholder("Search models")).fill(query);
}
export async function selectModel(page: PlaywrightPage, model: string): Promise<void> {
const select = modelSelect(page);
await select.click();
// Virtualized: options outside the rendered window are absent from the DOM, so search first.
await select.fill(model);
await searchModels(page, model);
await onlyVisible(page.getByRole("option", { name: model, exact: true })).click({ timeout: 15_000 });
await expect(modelSelect(page)).toContainText(model);
}
export async function sendMessage(page: PlaywrightPage, message: string): Promise<void> {
const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
const input = composer(page).getByRole("textbox", { name: "Message" });
await expect(input).toBeVisible({ timeout: 15_000 });
await input.fill(message);
await sendButton(page).click();

View file

@ -60,13 +60,11 @@ test.describe("Budgets", () => {
await expect(modal).toBeVisible({ timeout: 10_000 });
await modal.getByRole("textbox", { name: "Budget ID" }).fill(budgetId);
await modal.getByRole("spinbutton", { name: "Max Tokens per minute" }).fill("5000");
await modal.getByRole("spinbutton", { name: "Max Requests per minute" }).fill("60");
await modal.getByRole("button", { name: "Optional Settings" }).click();
await modal.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("25.5");
await modal.getByRole("combobox", { name: "Reset Budget" }).click();
await page.getByRole("option", { name: "weekly" }).click();
await modal.getByRole("spinbutton", { name: "Tokens per minute" }).fill("5000");
await modal.getByRole("spinbutton", { name: "Requests per minute" }).fill("60");
await modal.getByRole("spinbutton", { name: "Spend limit" }).fill("25.5");
await modal.getByRole("combobox", { name: "Reset period" }).click();
await page.getByRole("option", { name: "per week" }).click();
await modal.getByRole("button", { name: "Create Budget" }).click();
await expect(modal).not.toBeVisible({ timeout: 10_000 });
@ -100,15 +98,12 @@ test.describe("Budgets", () => {
const modal = page.getByRole("dialog", { name: "Edit Budget" });
await expect(modal).toBeVisible({ timeout: 10_000 });
await modal.getByRole("button", { name: "Optional Settings" }).click();
await modal.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("99");
await modal.getByRole("spinbutton", { name: "Spend limit" }).fill("99");
await modal.getByRole("button", { name: "Save", exact: true }).click();
await expect(modal).not.toBeVisible({ timeout: 10_000 });
await expect(page.getByRole("row").filter({ hasText: budgetId })).toContainText("$99.00", { timeout: 10_000 });
// Not hypothetical: the edit form posts the whole budget, so a field it fails to
// seed from the existing row goes to the server as null and silently clears.
const stored = await findBudget(page, budgetId);
expect(stored?.max_budget, "spend cap raised").toBe(99);
expect(stored?.tpm_limit, "TPM limit untouched by a spend-cap edit").toBe(1000);

View file

@ -2,6 +2,7 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants";
import { Page } from "../../fixtures/pages";
import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
import { deleteGuardrailFromTable } from "../../helpers/guardrails";
import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic";
interface StoredGuardrail {
@ -183,21 +184,9 @@ test.describe("Guardrails", () => {
test("Delete a guardrail", async ({ page }) => {
const stamp = Date.now();
const guardrailName = `e2e-guardrail-delete-${stamp}`;
const guardrailId = await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
await navigateToPage(page, Page.Guardrails);
await dismissFeedbackPopup(page);
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 });
await page.getByTestId(`guardrail-actions-${guardrailId}`).click();
await page.getByTestId("guardrail-action-delete").click();
const modal = page.getByRole("dialog");
await expect(modal).toBeVisible({ timeout: 5_000 });
await modal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(0, { timeout: 15_000 });
await deleteGuardrailFromTable(page, guardrailName);
// The RC checklist deletes then reloads, because a row vanishing from the table has
// fooled us before; assert against the route the reload would read.
@ -274,19 +263,7 @@ test.describe("Guardrails", () => {
await expect(page.getByRole("option", { name: guardrailName })).toBeVisible({ timeout: 10_000 });
await page.keyboard.press("Escape");
await navigateToPage(page, Page.Guardrails);
await expect(row).toHaveCount(1, { timeout: 15_000 });
await row.getByRole("button", { name: "Open guardrail actions" }).click();
await page.getByRole("menuitem", { name: "Delete" }).click();
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({
timeout: 10_000,
});
await expect(row).toHaveCount(0, { timeout: 15_000 });
await deleteGuardrailFromTable(page, guardrailName);
await page.reload();
await expect(page.getByRole("button", { name: /Add New Guardrail/i })).toBeVisible({ timeout: 20_000 });

View file

@ -1,9 +1,10 @@
import { test as base, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL, PROPAGATION_TIMEOUT_MS } from "../../constants";
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
import { deleteGuardrailFromTable } from "../../helpers/guardrails";
import { Page } from "../../fixtures/pages";
import { CHAT_MODEL_A, masterKey, rootPath, uniqueSuffix, waitForSpendLog } from "../../helpers/traffic";
import { openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground";
import { composer, openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground";
const RAW_EMAIL = "jane.doe@example.com";
const RAW_PHONE = "555-867-5309";
@ -49,19 +50,6 @@ async function createPresidioGuardrail(page: PlaywrightPage, guardrailName: stri
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(1, { timeout: 15_000 });
}
async function deleteGuardrail(page: PlaywrightPage, guardrailName: string): Promise<void> {
await navigateToPage(page, Page.Guardrails);
await dismissFeedbackPopup(page);
const row = page.getByRole("row").filter({ hasText: guardrailName });
await expect(row).toHaveCount(1, { timeout: 15_000 });
await row.getByRole("button", { name: "Open guardrail actions" }).click();
await page.getByRole("menuitem", { name: "Delete" }).click();
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 });
}
const test = base.extend<{ guardrailName: string }>({
guardrailName: async ({ page }, use) => {
const name = `e2e-presidio-story-${uniqueSuffix()}`;
@ -69,7 +57,7 @@ const test = base.extend<{ guardrailName: string }>({
try {
await use(name);
} finally {
await deleteGuardrail(page, name);
await deleteGuardrailFromTable(page, name);
}
},
});
@ -123,14 +111,12 @@ test.describe("Presidio PII guardrail, end to end from the dashboard", () => {
await openPlayground(page);
await selectModel(page, CHAT_MODEL_A);
const guardrailSelect = onlyVisible(page.getByPlaceholder("Select guardrails"));
await expect(guardrailSelect).toBeVisible({ timeout: 20_000 });
await guardrailSelect.click();
await guardrailSelect.fill(guardrailName);
await composer(page).getByRole("combobox", { name: "Guardrails" }).click();
await onlyVisible(page.getByPlaceholder("Search guardrails")).fill(guardrailName);
await onlyVisible(page.getByRole("option", { name: guardrailName, exact: true })).click({ timeout: 20_000 });
await page.keyboard.press("Escape");
const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
const input = composer(page).getByRole("textbox", { name: "Message" });
await expect(input).toBeVisible({ timeout: 15_000 });
await input.fill(prompt);

View file

@ -9,7 +9,7 @@ import {
import { Page } from "../../fixtures/pages";
import { navigateToPage, clickTeamId } from "../../helpers/navigation";
import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic";
import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground";
import { openPlayground, selectModel, sendMessage, enterVirtualKey } from "../../helpers/playground";
test.describe("Internal User", () => {
test.use({ storageState: INTERNAL_USER_STORAGE_PATH });
@ -67,18 +67,13 @@ test.describe("Internal User", () => {
await page.getByRole("button", { name: "Create Key", exact: true }).click();
await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim();
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
expect(apiKey).toMatch(/^sk-/);
await page.keyboard.press("Escape");
try {
await openPlayground(page);
await keySourceSelect(page).click();
await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 });
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
await expect(keyInput).toBeVisible({ timeout: 10_000 });
await keyInput.fill(apiKey);
await enterVirtualKey(page, apiKey);
await selectModel(page, CHAT_MODEL_A);
await sendMessage(page, `internal user team key ping ${keyName}`);

View file

@ -194,17 +194,6 @@ test.describe("Logs page", () => {
await expect(row, `no logs row for call id ${callId}`).toHaveCount(1, { timeout: 30_000 });
await expect(row, "the row itself shows only the request id").not.toContainText(callId);
await row.getByText(requestId).hover();
const tooltip = page.locator("[data-slot='tooltip-content']");
await expect(tooltip, "hovering the Request ID cell does not list the x-litellm-call-id").toContainText(
`x-litellm-call-id: ${callId}`,
{ timeout: 10_000 },
);
await tooltip.getByRole("button", { name: "Copy x-litellm-call-id" }).click();
if (await page.evaluate(() => window.isSecureContext)) {
await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(callId);
}
await row.click();
const drawer = page.getByRole("dialog").first();
await expect(drawer.getByText("Request & Response")).toBeVisible({ timeout: 20_000 });

View file

@ -1,8 +1,14 @@
import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
import { navigateToPage, dismissFeedbackPopup, setAriaHidden } from "../../helpers/navigation";
import { Page } from "../../fixtures/pages";
import { CHAT_MODEL_A, createVirtualKey, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic";
import {
CHAT_MODEL_A,
createVirtualKey,
deleteVirtualKey,
sendChatCompletion,
waitForSpendLog,
} from "../../helpers/traffic";
/**
* Session-grouped pagination (#38060): a page of N rows must render exactly N session rows, a
@ -45,6 +51,52 @@ async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"):
await page.getByRole("option", { name: size, exact: true }).click();
}
test.describe("Logs pagination with Aria in the header", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("keeps pagination reachable and Aria available at 1440px", async ({ page, request }) => {
await page.setViewportSize({ width: 1440, height: 900 });
const alias = `e2e-aria-header-${uniqueSuffix()}`;
const key = await createVirtualKey(request, { key_alias: alias });
try {
const ids = await Promise.all(
Array.from({ length: 26 }, (_, index) =>
sendChatCompletion(request, {
model: CHAT_MODEL_A,
prompt: `${alias}-${index}`,
apiKey: key.key,
}),
),
);
await Promise.all(ids.map((id) => waitForSpendLog(request, id)));
await openLogs(page);
await setAriaHidden(page, false);
await applyKeyAliasFilter(page, await openFilterDrawer(page), alias);
await setRowsPerPage(page, "25");
await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26");
await visibleTestId(page, "pagination-next").click();
await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-26 of 26");
const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true });
await aria.click();
await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible();
await page.keyboard.press("Escape");
await expect(aria).toBeFocused();
await visibleTestId(page, "pagination-first").click();
await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26");
for (const destination of [Page.NewUsage, Page.LlmPlayground, Page.ApiKeys]) {
await navigateToPage(page, destination);
await aria.click();
await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible();
await page.keyboard.press("Escape");
await expect(aria).toBeFocused();
}
} finally {
await deleteVirtualKey(request, key.token);
}
});
});
test.describe("Logs page session-grouped pagination", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });

View file

@ -1,52 +0,0 @@
import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { navigateToPage } from "../../helpers/navigation";
import { Page } from "../../fixtures/pages";
/**
* Opens Add Auto Router and returns the Template select's trigger, which is the
* shallowest real page that renders SelectContent with tall multi-line options.
*/
async function openTemplateSelect(page: PlaywrightPage) {
await navigateToPage(page, Page.Models);
await page.getByRole("tab", { name: "Auto-Routers" }).click();
await page.getByRole("button", { name: "Add Auto Router" }).click();
const trigger = page.getByTestId("template-selector");
await expect(trigger).toBeVisible();
return trigger;
}
async function boxes(trigger: Locator, options: Locator) {
const triggerBox = await trigger.boundingBox();
const optionsBox = await options.boundingBox();
return triggerBox && optionsBox ? { triggerBox, optionsBox } : null;
}
const clippedPopup = (page: PlaywrightPage) => page.locator('[data-slot="select-content"]');
function pollOptionsCoverTrigger(trigger: Locator, options: Locator) {
return expect.poll(async () => {
const box = await boxes(trigger, options);
return (
box &&
box.optionsBox.y < box.triggerBox.y + box.triggerBox.height &&
box.optionsBox.y + box.optionsBox.height > box.triggerBox.y
);
});
}
test.describe("Auto Router template select anchoring", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
test("keeps the trigger uncovered when the options open with no room below it", async ({ page }) => {
await page.setViewportSize({ width: 1280, height: 560 });
const trigger = await openTemplateSelect(page);
await trigger.scrollIntoViewIfNeeded();
await trigger.click();
await expect(page.getByRole("listbox")).toBeVisible();
await pollOptionsCoverTrigger(trigger, clippedPopup(page)).toBe(false);
});
});

View file

@ -0,0 +1,146 @@
import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { Role, users } from "../../fixtures/users";
import { Page } from "../../fixtures/pages";
import { navigateToPage } from "../../helpers/navigation";
import { captureRequestBody, readBack } from "../../helpers/roundTrip";
const FIVE_MINUTE_PER_MILLION = 3.75;
const ONE_HOUR_PER_MILLION = 7.5;
const perToken = (perMillion: number): number => perMillion / 1_000_000;
const ONE_HOUR_LABEL = /^Cache Write Cost \(1 hour, per 1M tokens\)/;
type Deployment = {
model_name?: string;
model_info?: { id?: string };
litellm_params: {
cache_creation_input_token_cost?: number;
cache_creation_input_token_cost_above_1hr?: number;
};
};
const masterKey = users[Role.ProxyAdmin].password;
const auth = { Authorization: `Bearer ${masterKey}` };
async function deleteModel(page: PlaywrightPage, id: string) {
await page.request.post("/model/delete", { headers: auth, data: { id } });
}
async function readDeployment(page: PlaywrightPage, modelId: string): Promise<Deployment | undefined> {
const body = await readBack<{ data: Deployment[] }>(page, `/model/info?litellm_model_id=${modelId}`);
return body.data[0];
}
async function selectProvider(page: PlaywrightPage, providerName: string) {
const providerDropdown = page.getByRole("combobox", { name: "Provider", exact: true });
await providerDropdown.click();
await providerDropdown.fill(providerName);
await page.getByRole("option").filter({ hasText: providerName }).first().click();
await expect(providerDropdown).toHaveValue(providerName);
}
test.describe("Cache write cost tiers", () => {
test.use({ storageState: ADMIN_STORAGE_PATH });
let createdModelId: string | null = null;
test.afterEach(async ({ page }) => {
if (createdModelId) {
await deleteModel(page, createdModelId);
createdModelId = null;
}
});
test("the edit form saves the 1 hour rate next to the 5 minute rate and the detail view shows both", async ({
page,
}) => {
const modelName = `e2e-cache-tiers-edit-${Date.now()}`;
const created = await page.request.post("/model/new", {
headers: auth,
data: {
model_name: modelName,
litellm_params: {
model: "anthropic/claude-sonnet-4-6",
api_key: "sk-e2e-not-used",
cache_creation_input_token_cost: perToken(FIVE_MINUTE_PER_MILLION),
},
model_info: {},
},
});
expect(created.ok(), `POST /model/new for ${modelName}`).toBe(true);
createdModelId = (await created.json()).model_info?.id;
expect(createdModelId, "model id from /model/new").toBeTruthy();
await page.goto("/ui");
await page.getByText("Models + Endpoints").click();
const modelIdCell = page.getByTestId(`model-id-${createdModelId}`);
await expect(modelIdCell).toBeVisible({ timeout: 15_000 });
await modelIdCell.click();
await expect(page.getByText("Back to Models").first()).toBeVisible({ timeout: 10_000 });
await page.getByRole("button", { name: "Edit Settings" }).click();
const oneHour = page.getByLabel(ONE_HOUR_LABEL);
await oneHour.waitFor({ timeout: 15_000 });
await oneHour.fill(String(ONE_HOUR_PER_MILLION));
const patchBody = await captureRequestBody(
page,
{ method: "PATCH", urlIncludes: `/model/${createdModelId}/update` },
async () => {
await page.getByRole("button", { name: "Save Changes" }).click();
},
);
expect(patchBody.litellm_params.cache_creation_input_token_cost_above_1hr, "1 hour rate sent per token").toBe(
perToken(ONE_HOUR_PER_MILLION),
);
await expect(page.getByText("Model settings updated successfully")).toBeVisible({ timeout: 10_000 });
const stored = await readDeployment(page, createdModelId!);
expect(stored?.litellm_params?.cache_creation_input_token_cost_above_1hr, "1 hour rate persisted").toBe(
perToken(ONE_HOUR_PER_MILLION),
);
expect(stored?.litellm_params?.cache_creation_input_token_cost, "5 minute rate untouched").toBe(
perToken(FIVE_MINUTE_PER_MILLION),
);
await expect(page.getByText(ONE_HOUR_LABEL)).toBeVisible();
await expect(page.getByText(ONE_HOUR_PER_MILLION.toFixed(4))).toBeVisible({ timeout: 10_000 });
await expect(page.getByText(FIVE_MINUTE_PER_MILLION.toFixed(4))).toBeVisible();
});
test("Add Model sends both cache write tiers per token", async ({ page }) => {
const publicName = `e2e-cache-tiers-add-${Date.now()}`;
await navigateToPage(page, Page.Models);
await page.getByRole("tab", { name: "Add Model" }).click();
await selectProvider(page, "Anthropic");
await page.getByRole("combobox", { name: "Select models" }).click();
await page.getByRole("option", { name: "Custom Model Name (Enter below)" }).click();
await page.keyboard.press("Escape");
await page.getByPlaceholder("Enter custom model name").fill(publicName);
await page.locator("#api_key").fill("sk-e2e-not-used");
await page.getByText("Advanced Settings").click();
await page.getByRole("switch", { name: "Custom Pricing" }).click();
await page.getByLabel(/^Input Cost \(per 1M tokens\)/).fill("3");
await page.getByLabel(/^Output Cost \(per 1M tokens\)/).fill("15");
await page.getByLabel(/^Cache Write Cost \(per 1M tokens\)/).fill(String(FIVE_MINUTE_PER_MILLION));
await page.getByLabel(ONE_HOUR_LABEL).fill(String(ONE_HOUR_PER_MILLION));
const body = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
await page.getByTestId("add-model-btn").click();
});
await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 });
const listed = await readBack<{ data: Deployment[] }>(page, "/v2/model/info");
createdModelId = listed.data.find((row) => row.model_name === publicName)?.model_info?.id ?? null;
expect(body.model_name).toBe(publicName);
expect(body.litellm_params.cache_creation_input_token_cost, "5 minute rate sent per token").toBe(
perToken(FIVE_MINUTE_PER_MILLION),
);
expect(body.litellm_params.cache_creation_input_token_cost_above_1hr, "1 hour rate sent per token").toBe(
perToken(ONE_HOUR_PER_MILLION),
);
expect(createdModelId, "the model landed in the database").toBeTruthy();
});
});

View file

@ -87,7 +87,7 @@ test.describe("Clear custom pricing on a deployment", () => {
// Both cache fields share the same placeholder ("Defaults to Input Cost if blank"),
// so disambiguate via their labels.
const cacheReadCost = page.getByLabel(/Cache Read Cost/);
const cacheWriteCost = page.getByLabel(/Cache Write Cost/);
const cacheWriteCost = page.getByLabel("Cache Write Cost (per 1M tokens)", { exact: true });
await inputCost.waitFor({ timeout: 15_000 });
for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) {
await field.click({ clickCount: 3 });

View file

@ -1,7 +1,14 @@
import { test, expect } from "@playwright/test";
import { ADMIN_STORAGE_PATH } from "../../constants";
import { CHAT_MODEL_A, CHAT_MODEL_B, MOCK_RESPONSE_TEXT, createVirtualKey } from "../../helpers/traffic";
import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground";
import {
keySourceSelect,
openPlayground,
openRequestSettings,
selectModel,
sendMessage,
enterVirtualKey,
} from "../../helpers/playground";
/**
* The one flow that exercises the dashboard's own LLM call path rather than an admin CRUD endpoint,
@ -14,8 +21,9 @@ test.describe("Playground", () => {
test(`chats with ${model} using the current UI session`, async ({ page }) => {
await openPlayground(page);
// "Current UI Session" is the default: the logged-in admin's key, nothing pasted.
await expect(keySourceSelect(page)).toContainText("Current UI Session");
await openRequestSettings(page);
await expect(keySourceSelect(page)).toContainText("Current UI session");
await page.keyboard.press("Escape");
await selectModel(page, model);
const prompt = `playground ping for ${model}`;
@ -34,14 +42,7 @@ test.describe("Playground", () => {
await openPlayground(page);
// Switch the source to "Virtual Key" and paste the key we just minted.
await keySourceSelect(page).click();
await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 });
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
await expect(keyInput).toBeVisible({ timeout: 10_000 });
await keyInput.fill(key);
await enterVirtualKey(page, key);
await selectModel(page, CHAT_MODEL_A);
await sendMessage(page, "playground ping via virtual key");

View file

@ -254,7 +254,7 @@ test.describe("Proxy Admin - Keys", () => {
// verify it can call /chat/completions for the model it was scoped to.
// The mock LLM server (fixtures/mock_llm_server/server.py) replies with
// a fixed "This is a mock response." body.
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim();
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
expect(apiKey).toMatch(/^sk-/);
const response = await page.request.post("/chat/completions", {

View file

@ -69,7 +69,7 @@ test.describe("Second proxy admin", () => {
await page.getByRole("button", { name: "Create Key", exact: true }).click();
await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim();
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
expect(apiKey).toMatch(/^sk-/);
await page.keyboard.press("Escape");

View file

@ -130,7 +130,7 @@ test.describe("Team Admin - Member permissions", () => {
await memberPage.getByRole("button", { name: "Create Key", exact: true }).click();
const saveDialog = memberPage.getByRole("dialog", { name: "Save your Key" });
await expect(saveDialog).toBeVisible({ timeout: 15_000 });
const apiKey = (await saveDialog.locator("pre").innerText()).trim();
const apiKey = (await saveDialog.locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
expect(apiKey).toMatch(/^sk-/);
createdKeys.push(apiKey);
await memberPage.keyboard.press("Escape");

View file

@ -10,7 +10,7 @@ import { Page } from "../../fixtures/pages";
import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
import { captureRequestBody, readBack } from "../../helpers/roundTrip";
import { CHAT_MODEL_A, masterKey } from "../../helpers/traffic";
import { keySourceSelect, modelSelect, onlyVisible, openPlayground } from "../../helpers/playground";
import { onlyVisible, openPlayground, searchModels, enterVirtualKey } from "../../helpers/playground";
/**
* Every identifier a roster is addressable by. Which of user_id / user_email is populated depends on
@ -228,21 +228,14 @@ test.describe("Team Admin", () => {
.toBe(true);
await openPlayground(page);
await keySourceSelect(page).click();
await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 });
await enterVirtualKey(page, teamKey);
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
await expect(keyInput).toBeVisible({ timeout: 10_000 });
await keyInput.fill(teamKey);
const select = modelSelect(page);
await select.click();
await select.fill(teamModelName);
await searchModels(page, teamModelName);
await expect(onlyVisible(page.getByRole("option", { name: teamModelName }))).toBeVisible({
timeout: 15_000,
});
await select.fill(CHAT_MODEL_A);
await onlyVisible(page.getByPlaceholder("Search models")).fill(CHAT_MODEL_A);
await expect(onlyVisible(page.getByRole("option", { name: CHAT_MODEL_A }))).toBeVisible({
timeout: 15_000,
});

View file

@ -11,7 +11,7 @@ import {
waitForSpendLog,
} from "../../helpers/traffic";
/** Covers /ui/usage. The legacy /ui/old-usage view is deprecated and deliberately not covered. */
/** Covers /ui/usage. */
const topKeysCard = (page: PlaywrightPage): Locator =>
page.locator("section").filter({ has: page.getByRole("heading", { name: "Top Virtual Keys", exact: true }) });

View file

@ -0,0 +1,182 @@
from __future__ import annotations
import math
from typing import Final
from pydantic import BaseModel, ConfigDict, TypeAdapter
from setup.event_loop import EventLoopStalls, KnownStall, junit_properties, offenders, parse_stalls, summarize
_HANDLER: Final = (
"Executing <Task pending name='Task-9' coro=<RequestResponseCycle.run_asgi() running at "
"/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py:416> wait_for=<Future pending "
"cb=[Task.task_wakeup()]> created at /usr/local/lib/python3.12/asyncio/base_events.py:448> took 0.731 seconds"
)
_STREAM: Final = (
"Executing <Task pending name='Task-31' coro=<TaskHandle._run_coro() running at "
"/usr/local/lib/python3.12/site-packages/anyio/_core/_tasks.py:275> created at "
"/usr/local/lib/python3.12/asyncio/tasks.py:420> took 0.412 seconds"
)
_STARTUP: Final = (
"Executing <Task pending name='Task-2' coro=<LifespanOn.main() running at "
"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py:86> created at "
"/usr/local/lib/python3.12/asyncio/base_events.py:448> took 0.302 seconds"
)
_GC_PASS: Final = (
"2026-10-17 04:46:02,101 - LiteLLM Proxy - WARNING - Garbage collection gen=2 collected=98608 took 0.711 seconds"
)
_SPEC: Final = EventLoopStalls(
allowlist=(
KnownStall("LifespanOn.main", max_count=3),
KnownStall("RequestResponseCycle.run_asgi", max_count=2),
KnownStall("garbage collection", max_count=1),
)
)
class _Slice(BaseModel):
model_config = ConfigDict(frozen=True)
label: str
value: float
share: float | None
note: str | None
over: bool
def _breakdown(properties: dict[str, float | str], name: str) -> list[_Slice]:
return TypeAdapter(list[_Slice]).validate_json(str(properties[f"perf_event_loop_{name}_breakdown"]))
def test_parse_keeps_the_coroutine_its_location_and_the_hold() -> None:
(stall,) = parse_stalls(("INFO: plain log line", _HANDLER))
assert stall.coroutine == "RequestResponseCycle.run_asgi()"
assert stall.where == "uvicorn/protocols/http/h11_impl.py:416"
assert stall.seconds == 0.731
def test_a_location_off_the_allowlist_offends_once_however_often_it_appears() -> None:
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER)))
assert [(row.coroutine, row.count, row.offending) for row in rows] == [
("TaskHandle._run_coro()", 2, True),
("RequestResponseCycle.run_asgi()", 1, False),
("LifespanOn.main()", 1, False),
]
assert offenders(rows) == (
"TaskHandle._run_coro() anyio/_core/_tasks.py:275: 2x, max 0.41 s (not on the allowlist)",
)
def test_a_known_location_offends_only_past_its_count() -> None:
within: Final = summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER)))
assert offenders(within) == ()
past: Final = summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER, _HANDLER)))
assert offenders(past) == (
"RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416: 3x, max 0.73 s "
"(allowlist RequestResponseCycle.run_asgi permits 2)",
)
def test_holds_under_the_threshold_are_not_rows() -> None:
short: Final = _HANDLER.replace("took 0.731 seconds", "took 0.099 seconds")
assert summarize(_SPEC, parse_stalls((short,))) == ()
def test_a_hold_the_collector_pass_before_it_accounts_for_is_the_collectors_not_the_tasks() -> None:
rows: Final = summarize(_SPEC, parse_stalls((_GC_PASS, _STREAM.replace("0.412", "0.731"))))
assert [(row.coroutine, row.where, row.max_seconds, row.offending) for row in rows] == [
("garbage collection", "gen=2", 0.731, False)
]
def test_a_hold_well_past_its_collector_pass_stays_on_the_task() -> None:
short_pass: Final = _GC_PASS.replace("0.711", "0.200")
rows: Final = summarize(_SPEC, parse_stalls((short_pass, _STREAM.replace("0.412", "0.731"))))
assert [(row.coroutine, row.offending) for row in rows] == [("TaskHandle._run_coro()", True)]
def test_a_hold_is_charged_to_the_longest_collector_pass_that_fits_it_not_to_their_sum() -> None:
thread_side_pass: Final = _GC_PASS.replace("0.711", "0.627")
rows: Final = summarize(
_SPEC, parse_stalls((thread_side_pass, _GC_PASS.replace("0.711", "0.605"), _STREAM.replace("0.412", "0.614")))
)
assert [(row.coroutine, row.where, row.max_seconds, row.offending) for row in rows] == [
("garbage collection", "gen=2", 0.614, False)
]
def test_a_collector_pass_explains_only_the_next_hold_and_only_one_at_least_as_long() -> None:
rows: Final = summarize(_SPEC, parse_stalls((_GC_PASS, _STARTUP, _HANDLER)))
assert sorted((row.coroutine, row.offending) for row in rows) == [
("LifespanOn.main()", False),
("RequestResponseCycle.run_asgi()", False),
]
assert summarize(_SPEC, parse_stalls((_GC_PASS,))) == ()
def test_junit_properties_count_every_hold_and_only_the_holds_past_their_allowance_as_offending() -> None:
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER)))
properties: Final = dict(junit_properties(_SPEC, rows))
assert properties["perf_event_loop_holds"] == 6
assert properties["perf_event_loop_holds_limit"] == 6
assert properties["perf_event_loop_offending_holds"] == 3
assert properties["perf_event_loop_offending_holds_limit"] == 0
assert properties["perf_event_loop_hold_seconds_max"] == 0.731
assert math.isclose(float(properties["perf_event_loop_hold_seconds_total"]), 0.302 + 2 * 0.412 + 3 * 0.731)
breakdown: Final = _breakdown(properties, "holds")
assert [(slice.label, slice.value, slice.note, slice.over) for slice in breakdown] == [
(
"RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416",
3,
"3 of 2 allowed, max 0.73 s",
True,
),
("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, "not on the allowlist, max 0.41 s", True),
("LifespanOn.main() uvicorn/lifespan/on.py:86", 1, "1 of 3 allowed, max 0.30 s", False),
]
assert math.isclose(sum(slice.share or 0.0 for slice in breakdown), 1.0)
def test_the_offending_breakdown_lists_only_the_locations_past_their_cap_by_how_far_over_they_are() -> None:
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER)))
offending: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "offending_holds")
assert [(slice.label, slice.value, slice.share, slice.over) for slice in offending] == [
("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 1, 1 / 3, True),
("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, 2 / 3, True),
]
clean: Final = dict(junit_properties(_SPEC, summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER)))))
assert clean["perf_event_loop_offending_holds"] == 0
assert "perf_event_loop_offending_holds_breakdown" not in clean
def test_the_seconds_breakdowns_carry_the_time_per_location_and_the_longest_single_holds() -> None:
long_handler: Final = _HANDLER.replace("0.731", "0.950")
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, long_handler, _HANDLER)))
properties: Final = dict(junit_properties(_SPEC, rows))
per_location: Final = _breakdown(properties, "hold_seconds_total")
assert [(slice.label.split()[0], slice.value) for slice in per_location] == [
("RequestResponseCycle.run_asgi()", 0.950 + 0.731),
("TaskHandle._run_coro()", 0.412),
("LifespanOn.main()", 0.302),
]
assert math.isclose(sum(slice.share or 0.0 for slice in per_location), 1.0)
longest: Final = _breakdown(properties, "hold_seconds_max")
assert [(slice.label.split()[0], slice.value, slice.share, slice.over) for slice in longest] == [
("RequestResponseCycle.run_asgi()", 0.950, None, False),
("RequestResponseCycle.run_asgi()", 0.731, None, False),
("TaskHandle._run_coro()", 0.412, None, True),
("LifespanOn.main()", 0.302, None, False),
]
def test_the_longest_holds_breakdown_stops_at_ten() -> None:
rows: Final = summarize(_SPEC, parse_stalls(tuple(_STARTUP for _ in range(12))))
longest: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "hold_seconds_max")
assert len(longest) == 10
def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None:
properties: Final = dict(junit_properties(_SPEC, ()))
assert properties["perf_event_loop_holds"] == 0
assert properties["perf_event_loop_offending_holds"] == 0
assert properties["perf_event_loop_hold_seconds_max"] == 0
assert not [name for name in properties if name.endswith("_breakdown")]

View file

@ -6,7 +6,7 @@ from typing import Final
from cases.small_bodies_5rps import CASE
from memory_readings import RssCapture
from setup.datastore import DbCounters, StatementCalls
from setup.observe import Observation, evaluate, junit_properties
from setup.observe import Contributor, Observation, breakdown, evaluate, junit_properties
from setup.performance_test_case import Limit
from setup.wire import WireRecord
@ -36,7 +36,12 @@ def served(seq: int, total_s: float) -> WireRecord:
def _key_lookup(calls: int) -> StatementCalls:
return StatementCalls(queryid=1, query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1', calls=calls)
return StatementCalls(
queryid=1,
query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1',
calls=calls,
total_exec_ms=calls * 2.0,
)
def test_junit_properties_carry_each_observed_metric_and_each_limit_including_unobserved_ones() -> None:
@ -62,12 +67,163 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un
properties: Final = junit_properties(rows)
assert len({name for name, _ in properties}) == len(properties)
assert dict(properties) == {
assert {name: value for name, value in properties if not name.endswith("_breakdown")} == {
"perf_unexpected_status_count": 0.0,
"perf_gateway_added_seconds_p50": 0.5,
"perf_gateway_added_seconds_p50_limit": 0.625,
"perf_gateway_added_seconds_p99": 0.75,
"perf_postgres_calls_per_request": 10.0,
"perf_postgres_background_calls_per_request": 0.0,
"perf_postgres_max_statement_per_request": 10.0,
"perf_postgres_max_background_statement_per_request": 0.0,
"perf_postgres_active_sessions": 0.02,
"perf_postgres_slowest_statement_seconds": 0.002,
"perf_redis_calls_per_request_limit": 35.0,
}
def _statement(queryid: int, query: str, calls: int) -> StatementCalls:
return StatementCalls(queryid=queryid, query=query, calls=calls)
def _observation_with(statements: tuple[StatementCalls, ...]) -> Observation:
before: Final = tuple(replace(item, calls=0) for item in statements)
return Observation(
records=tuple(served(seq, 1.0) for seq in range(1000)),
mock_requests=(),
guardrail_calls=(),
rss_before=NO_RSS,
rss_after=NO_RSS,
wall_seconds=600.0,
db_before=DbCounters(commits=0, statements=before),
db_after=DbCounters(commits=0, statements=statements),
)
MAX_STATEMENT_CASE: Final = replace(CASE, limits=(Limit("postgres_max_statement_per_request", max=0.5),))
BACKGROUND_CAP_CASE: Final = replace(
CASE,
limits=(Limit("postgres_max_background_statement_per_request", max=0.9),),
background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
)
GUARDRAIL_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyGuardrailMetrics" ("guardrail_id","date") VALUES ($1,$2)'
MODEL_USAGE_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyModelUsage" ("date","model") VALUES ($1,$2)'
TEAM_REFRESH: Final = 'SELECT * FROM "LiteLLM_TeamMembership" WHERE user_id = $1'
USER_LOOKUP: Final = 'SELECT * FROM "LiteLLM_UserTable" WHERE user_id = $1'
def test_cache_refreshes_whose_sum_exceeds_the_cap_pass_the_per_statement_limit() -> None:
statements: Final = (
_statement(1, TEAM_REFRESH, calls=357),
_statement(2, USER_LOOKUP, calls=245),
_statement(3, 'INSERT INTO "LiteLLM_DailyModelUsage" VALUES ($1)', calls=1000),
)
rows, exceeded = evaluate(MAX_STATEMENT_CASE, _observation_with(statements))
assert exceeded == []
assert dict(junit_properties(rows))["perf_postgres_max_statement_per_request"] == 0.357
assert dict(junit_properties(rows))["perf_postgres_calls_per_request"] == 0.602
def test_a_lookup_on_every_request_fails_the_per_statement_limit_and_names_the_statement() -> None:
statements: Final = (_statement(1, TEAM_REFRESH, calls=357), _statement(2, USER_LOOKUP, calls=1000))
_, exceeded = evaluate(MAX_STATEMENT_CASE, _observation_with(statements))
assert exceeded == [f"postgres_max_statement_per_request: observed 1.0000 > limit 0.5000 ({USER_LOOKUP})"], exceeded
def test_per_statement_limit_without_db_counters_reports_nothing_observed() -> None:
observation: Final = replace(_observation_with(()), db_before=None, db_after=None)
_, exceeded = evaluate(MAX_STATEMENT_CASE, observation)
assert exceeded == ["postgres_max_statement_per_request: limit set but nothing observed"]
def test_per_flush_rollups_pass_the_background_cap_and_the_exempt_table_is_skipped() -> None:
statements: Final = (
_statement(1, GUARDRAIL_UPSERT, calls=1150),
_statement(2, MODEL_USAGE_UPSERT, calls=810),
_statement(3, USER_LOOKUP, calls=1000),
)
rows, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements))
assert exceeded == []
assert dict(junit_properties(rows))["perf_postgres_max_background_statement_per_request"] == 0.81
def test_a_rollup_written_on_every_request_fails_the_background_cap_and_names_the_statement() -> None:
statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1050), _statement(2, MODEL_USAGE_UPSERT, calls=1000))
_, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements))
assert exceeded == [
f"postgres_max_background_statement_per_request: observed 1.0000 > limit 0.9000 ({MODEL_USAGE_UPSERT})"
], exceeded
def test_background_cap_without_exemptions_counts_the_guardrail_upsert() -> None:
case: Final = replace(BACKGROUND_CAP_CASE, background_cap_exempt_tables=frozenset())
statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1150), _statement(2, MODEL_USAGE_UPSERT, calls=810))
_, exceeded = evaluate(case, _observation_with(statements))
assert exceeded == [
f"postgres_max_background_statement_per_request: observed 1.1500 > limit 0.9000 ({GUARDRAIL_UPSERT})"
], exceeded
def _guardrail_rollup(calls: int) -> StatementCalls:
return StatementCalls(
queryid=2,
query='INSERT INTO "LiteLLM_DailyGuardrailMetrics" ("guardrail_id","date") VALUES ($1,$2) ON CONFLICT DO UPDATE',
calls=calls,
total_exec_ms=calls * 1.0,
)
def _model_rollup(calls: int) -> StatementCalls:
return StatementCalls(
queryid=3,
query='INSERT INTO "LiteLLM_DailyModelUsage" ("model","date") VALUES ($1,$2) ON CONFLICT DO UPDATE',
calls=calls,
total_exec_ms=calls * 1.0,
)
def test_breakdown_names_the_slices_behind_each_metric_and_skips_plain_gauges() -> None:
failed: Final = served(3, 1.0).model_copy(update={"status": 500, "ok": False})
observation: Final = Observation(
records=(served(0, 1.25), served(1, 1.5), served(2, 1.75), failed),
mock_requests=(),
guardrail_calls=(),
rss_before=NO_RSS,
rss_after=NO_RSS,
wall_seconds=4.0,
db_before=DbCounters(
commits=100, statements=(_key_lookup(calls=100), _guardrail_rollup(calls=100), _model_rollup(calls=100))
),
db_after=DbCounters(
commits=130, statements=(_key_lookup(calls=130), _guardrail_rollup(calls=124), _model_rollup(calls=112))
),
gateway_cpu_seconds=2.0,
)
busiest: Final = breakdown("postgres_max_background_statement_per_request", None, observation)
assert [(item.value, item.share) for item in busiest] == [(6.0, 24 / 36), (3.0, 12 / 36)]
assert "LiteLLM_DailyGuardrailMetrics" in busiest[0].label
assert breakdown("gateway_added_seconds", 50, observation) == (Contributor("chat_completions", 0.5, 1.0),)
assert breakdown("unexpected_status_count", None, observation) == (
Contributor("chat_completions status 500", 1.0, 1.0),
)
assert breakdown("cpu_seconds_per_request", None, observation) == ()
rows, _ = evaluate(CASE, observation)
properties: Final = dict(junit_properties(rows))
assert "perf_cpu_seconds_per_request_breakdown" not in properties
assert properties["perf_unexpected_status_count_breakdown"] == (
'[{"label": "chat_completions status 500", "value": 1.0, "share": 1.0}]'
)

View file

@ -0,0 +1,168 @@
from __future__ import annotations
import math
from dataclasses import replace
from typing import Final
from cases.small_bodies_5rps import CASE
from memory_readings import RssCapture
from setup.observe import Observation, evaluate, junit_properties
from setup.performance_test_case import Limit, Ramp, ramped
from setup.plan import build_plan
from setup.ramp import StepReading, send_offsets, step_properties, sustained_rps
from setup.series import Sample, Series
from setup.telemetry import GatewayTelemetry, HistogramTotals
from setup.wire import WireRecord
NO_RSS: Final = RssCapture(readings=(), failures=())
def test_send_offsets_put_each_step_s_requests_inside_its_window_in_plan_order() -> None:
ramp: Final = Ramp(first_rps=10.0, last_rps=30.0, step_rps=10.0, step_seconds=10.0, held_p99_seconds=5.0)
offsets: Final = send_offsets(ramp, seed=13)
assert ramp.counts == (100, 200, 300)
assert len(offsets) == ramp.total_requests
assert offsets == tuple(sorted(offsets))
for seq, offset in enumerate(offsets, start=1):
step = ramp.step_of(seq)
assert step * 10.0 <= offset < (step + 1) * 10.0, f"request {seq} at {offset:.2f}s is outside step {step}"
assert send_offsets(ramp, seed=13) == offsets
assert send_offsets(ramp, seed=14) != offsets
def test_build_plan_carries_the_schedule_only_for_a_ramped_case() -> None:
ramp: Final = Ramp(first_rps=2.0, last_rps=4.0, step_rps=2.0, step_seconds=5.0, held_p99_seconds=5.0)
ramped_case: Final = replace(CASE, traffic=ramped(CASE.traffic, ramp, seed=3, concurrency=20))
assert ramped_case.traffic.expected_seconds == 10.0
assert [item.send_at_s for item in build_plan(ramped_case)] == list(send_offsets(ramp, seed=3))
assert all(item.send_at_s is None for item in build_plan(CASE))
def _step(rps: float, p99: float, completed_rps: float | None = None, unexpected: int = 0) -> StepReading:
return StepReading(
rps=rps,
planned=int(rps * 90),
completed_rps=rps if completed_rps is None else completed_rps,
gateway_added_p99=p99,
unexpected=unexpected,
held_p99_seconds=5.0,
)
def test_sustained_rps_is_the_last_step_held_before_the_first_that_broke() -> None:
assert sustained_rps((_step(10, 0.5), _step(20, 1.0), _step(30, 9.0), _step(40, 0.7))) == 20
assert sustained_rps((_step(10, 0.5), _step(20, 1.0, completed_rps=17.0), _step(30, 1.0))) == 10
assert sustained_rps((_step(10, 0.5), _step(20, 1.0, unexpected=19), _step(30, 1.0))) == 10
assert sustained_rps((_step(10, 0.5), _step(20, 1.0, unexpected=18), _step(30, 1.0))) == 30
assert sustained_rps((_step(10, 6.0), _step(20, 1.0))) == 0
def _record(seq: int, started_at: float, total_s: float) -> WireRecord:
return WireRecord(
seq=seq,
marker=f"perfreq-{seq:06d}-n",
caller="key-a1",
endpoint="chat_completions",
stream=False,
blocked=False,
started_at=started_at,
status=200,
header_s=total_s,
first_byte_s=total_s,
first_token_s=None,
total_s=total_s,
text="",
ok=True,
mock_delay_s=1.0,
call_id=None,
error=None,
)
def test_a_floor_fails_when_the_second_step_s_latency_breaks_it_and_the_properties_carry_the_floor() -> None:
ramp: Final = Ramp(first_rps=10.0, last_rps=20.0, step_rps=10.0, step_seconds=100.0, held_p99_seconds=2.0)
case: Final = replace(
CASE,
traffic=ramped(CASE.traffic, ramp, seed=1, concurrency=50),
limits=(Limit("sustained_rps", min=20),),
)
offsets: Final = send_offsets(ramp, seed=1)
records: Final = tuple(
_record(seq, 1000.0 + offset, 1.5 if ramp.step_of(seq) == 0 else 4.0)
for seq, offset in enumerate(offsets, start=1)
)
observation: Final = Observation(
records=records,
mock_requests=(),
guardrail_calls=(),
rss_before=NO_RSS,
rss_after=NO_RSS,
wall_seconds=200.0,
ramp=ramp,
)
first, second = observation.steps()
assert first.held and first.gateway_added_p99 == 0.5 and first.completed_rps > 9.5
assert not second.held and second.gateway_added_p99 == 3.0
rows, exceeded = evaluate(case, observation)
assert exceeded == ["sustained_rps: observed 10.0000 < floor 20.0000"]
assert {("perf_sustained_rps", 10.0), ("perf_sustained_rps_floor", 20.0)} <= set(junit_properties(rows))
assert dict(step_properties((first, second))) == {
"perf_ramp_hold_p99_seconds": 2.0,
"perf_ramp_step_10_gateway_added_p99": 0.5,
"perf_ramp_step_10_completed_rps": first.completed_rps,
"perf_ramp_step_10_unexpected": 0.0,
"perf_ramp_step_10_gateway_added_p50": 0.5,
"perf_ramp_step_20_gateway_added_p99": 3.0,
"perf_ramp_step_20_completed_rps": second.completed_rps,
"perf_ramp_step_20_unexpected": 0.0,
"perf_ramp_step_20_gateway_added_p50": 3.0,
}
assert not evaluate(replace(case, limits=(Limit("sustained_rps", min=10),)), observation)[1]
def test_each_step_reads_the_pod_s_cpu_redis_latency_and_connections_from_its_own_window() -> None:
ramp: Final = Ramp(first_rps=10.0, last_rps=20.0, step_rps=10.0, step_seconds=100.0, held_p99_seconds=2.0)
records: Final = tuple(
_record(seq, 1000.0 + offset, 1.5) for seq, offset in enumerate(send_offsets(ramp, seed=1), start=1)
)
started: Final = min(record.started_at for record in records)
redis: Final = tuple(
Sample(at=started + at, value=GatewayTelemetry(redis_latency=HistogramTotals(total=total, count=count)))
for at, total, count in ((-1.0, 0.0, 0.0), (50.0, 1.0, 100.0), (99.0, 2.0, 200.0), (199.0, 42.0, 300.0))
)
observation: Final = Observation(
records=records,
mock_requests=(),
guardrail_calls=(),
rss_before=NO_RSS,
rss_after=NO_RSS,
wall_seconds=200.0,
ramp=ramp,
cpu_series=Series(samples=tuple(Sample(at=started + at, value=5.0) for at in (10.0, 60.0, 110.0, 160.0))),
connection_series=Series(
samples=tuple(
Sample(at=started + at, value=n) for at, n in ((1.0, 12), (50.0, 18), (150.0, 25), (199.0, 20))
)
),
telemetry_series=Series(samples=redis),
)
first, second = observation.steps()
first_completed: Final = round(first.completed_rps * ramp.step_seconds)
assert first.cpu_seconds_per_request is not None and math.isclose(
first.cpu_seconds_per_request, 10.0 / first_completed
)
assert first.redis_latency_mean_seconds is not None and math.isclose(first.redis_latency_mean_seconds, 0.01)
assert first.postgres_peak_connections == 18
assert second.redis_latency_mean_seconds is not None and math.isclose(second.redis_latency_mean_seconds, 0.4)
assert second.postgres_peak_connections == 25
assert first.time_to_first_token_p99 is None
names: Final = {name for name, _ in step_properties((first, second)) if name.startswith("perf_ramp_step_10_")}
assert names == {
"perf_ramp_step_10_gateway_added_p99",
"perf_ramp_step_10_completed_rps",
"perf_ramp_step_10_unexpected",
"perf_ramp_step_10_gateway_added_p50",
"perf_ramp_step_10_cpu_seconds_per_request",
"perf_ramp_step_10_redis_latency_mean_seconds",
"perf_ramp_step_10_postgres_peak_connections",
}

View file

@ -0,0 +1,173 @@
from __future__ import annotations
from typing import Final
import pytest
from setup.ui_journeys import (
UI_JOURNEY_LIMITS,
Sample,
assert_within_ceiling,
database_failures,
error_code,
journeys,
summarize,
ui_junit_properties,
)
def test_first_journey_is_separate_from_repeated_latency() -> None:
samples: Final = (Sample(seconds=80), *(Sample(seconds=i) for i in range(1, 101)))
result: Final = summarize(samples)
assert result.first_seconds == 80
assert result.repeated_samples == 100
assert result.repeated_p50_seconds == 50.5
assert result.repeated_p95_seconds == 95
assert result.repeated_p99_seconds == 99
assert result.repeated_max_seconds == 100
def test_failed_repeats_do_not_look_like_fast_successes() -> None:
result: Final = summarize((Sample(seconds=10), Sample(seconds=0.01, error="AssertionError: no rows")))
assert result.repeated_samples == 0
assert result.repeated_p50_seconds is None
assert result.repeated_p95_seconds is None
assert result.repeated_p99_seconds is None
assert result.repeated_max_seconds is None
def test_small_sample_does_not_report_tail_percentiles() -> None:
result: Final = summarize(tuple(Sample(seconds=1) for _ in range(6)))
assert result.repeated_samples == 5
assert result.repeated_p50_seconds == 1
assert result.repeated_p95_seconds is None
assert result.repeated_p99_seconds is None
def test_failed_first_journey_emits_error_code_without_latency() -> None:
sample: Final = Sample(seconds=120, error="timeout")
assert summarize((sample,)).first_seconds is None
assert dict(ui_junit_properties((sample,))) == {
"perf_ui_journeys": 1,
"perf_ui_failures": 1,
"perf_ui_samples": 0,
"perf_ui_repeated_samples": 0,
"perf_ui_error_timeout": 1,
}
def test_junit_counts_failures_but_only_measures_successful_latency() -> None:
samples: Final = (
Sample(seconds=2, rows=25),
Sample(seconds=3, rows=25),
Sample(seconds=120, error="timeout"),
Sample(seconds=1, error="AssertionError: Logs table rendered empty"),
Sample(seconds=4, rows=25),
)
assert dict(ui_junit_properties(samples)) == {
"perf_ui_journeys": 5,
"perf_ui_failures": 2,
"perf_ui_samples": 3,
"perf_ui_repeated_samples": 2,
"perf_ui_error_timeout": 1,
"perf_ui_error_ui": 1,
"perf_ui_seconds_p50": 3,
"perf_ui_seconds_p90": 4,
"perf_ui_seconds_max": 4,
"perf_ui_first_seconds": 2,
"perf_ui_repeated_seconds_p50": 3.5,
"perf_ui_repeated_seconds_max": 4,
}
def test_all_journey_percentiles_include_first_and_use_nearest_rank_p90() -> None:
samples: Final = tuple(Sample(seconds=seconds) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5))
metrics: Final = dict(ui_junit_properties(samples))
assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == {
"perf_ui_seconds_p50": 5.5,
"perf_ui_seconds_p90": 9,
"perf_ui_seconds_max": 10,
}
assert metrics["perf_ui_samples"] == 10
def test_journeys_have_unique_names_and_cover_every_dashboard_surface() -> None:
names: Final = tuple(journey.name for journey in journeys("key"))
assert len(names) == len(frozenset(names))
assert frozenset(journey.surface for journey in journeys("key")) == {"logs", "usage", "caching", "models", "keys"}
def _metrics(postgres: str, batch: str) -> str:
return (
"# TYPE litellm_postgres_failed_requests_total counter\n"
f'litellm_postgres_failed_requests_total{{call_type="query_raw"}} {postgres}\n'
"# TYPE litellm_batch_write_to_db_failed_requests_total counter\n"
f'litellm_batch_write_to_db_failed_requests_total{{call_type="batch"}} {batch}\n'
)
def test_postgres_failures_include_background_batch_writes() -> None:
assert database_failures(_metrics("0", "0")) == 0
assert database_failures(_metrics("2", "3")) == 5
def test_lazy_failure_counters_require_evidence_of_database_activity() -> None:
assert (
database_failures(
"# TYPE litellm_postgres_total_requests_total counter\nlitellm_postgres_total_requests_total 3\n"
)
== 0
)
@pytest.mark.parametrize(
"text", ("", "# TYPE unrelated counter\nunrelated_total 0\n", _metrics("NaN", "0"), _metrics("-1", "0"))
)
def test_missing_or_invalid_metrics_cannot_pass_as_zero(text: str) -> None:
with pytest.raises(AssertionError):
_ = database_failures(text)
def test_every_journey_has_a_latency_ceiling() -> None:
assert frozenset(journey.name for journey in journeys("key")) == frozenset(UI_JOURNEY_LIMITS)
def test_failed_sample_fails_the_journey_before_latency_is_judged() -> None:
with pytest.raises(AssertionError, match="1 of 2 logs_open_24h samples failed: timeout"):
assert_within_ceiling("logs_open_24h", (Sample(seconds=0.2, rows=25), Sample(seconds=120, error="timeout")))
def test_slow_journey_fails_its_ceiling() -> None:
fast: Final = tuple(Sample(seconds=1, rows=25) for _ in range(9))
assert_within_ceiling("logs_open_24h", fast)
with pytest.raises(AssertionError, match=r"logs_open_24h p90 25\.0s over 20s"):
assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25)))
with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"):
assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3)))
def test_failed_data_request_status_becomes_the_error_code() -> None:
gateway_error: Final = Sample(seconds=0.4, error="AssertionError: Locator expected to be enabled", status=500)
slow: Final = Sample(seconds=60.0, error="timeout", status=200, request="GET /spend/logs/ui")
hung: Final = Sample(seconds=700.0, error="wall_clock TimeoutError: logs_next_page exceeded its 660s wall clock")
assert (error_code(gateway_error), error_code(slow), error_code(hung)) == ("http_500", "timeout", "wall_clock")
metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung)))
assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1
assert "perf_ui_error_ui" not in metrics
def test_volume_properties_name_rows_and_bytes_per_queried_table() -> None:
from setup.ui_query_fixture import TableVolume, volume_properties
properties: Final = volume_properties(
(
TableVolume(table="LiteLLM_SpendLogs", rows=850_000_000, bytes=1_000_000_000_000),
TableVolume(table="LiteLLM_VerificationToken", rows=5_000, bytes=4_194_304),
)
)
assert properties == (
("perf_ui_db_rows_LiteLLM_SpendLogs", 850_000_000),
("perf_ui_db_bytes_LiteLLM_SpendLogs", 1_000_000_000_000),
("perf_ui_db_rows_LiteLLM_VerificationToken", 5_000),
("perf_ui_db_bytes_LiteLLM_VerificationToken", 4_194_304),
)

View file

@ -0,0 +1,28 @@
from __future__ import annotations
from pathlib import Path
from typing import Final
import pytest
from setup.ui_query_isolation import settled_counter, validate_target
@pytest.mark.parametrize("host", ("production.example.com", "db-perf-source.litellm.svc", "localhost"))
def test_query_cancellation_rejects_non_lane_databases(host: str, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("E2E_UI_PERF_DISPOSABLE", "1")
monkeypatch.setenv("E2E_RESET_SPEND_LOGS", "0")
with pytest.raises(AssertionError, match="restricted"):
validate_target(f"postgresql://user@{host}/litellm")
def test_query_cancellation_requires_explicit_disposable_opt_in(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("E2E_UI_PERF_DISPOSABLE", raising=False)
with pytest.raises(AssertionError, match="confirmation"):
validate_target("postgresql://user@db-perf-perf-test-1.litellm.svc/litellm")
def test_counter_stabilization_waits_for_delayed_failures(tmp_path: Path) -> None:
readings: Final = iter((0.0, 0.0, 2.0, 2.0, 2.0))
evidence: Final = tmp_path / "isolation.jsonl"
assert settled_counter(lambda: next(readings), evidence) == 2
assert len(evidence.read_text().splitlines()) == 5

Some files were not shown because too many files have changed in this diff Show more