mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test(e2e): sync main suite into rc 1.108.0
Copy the canonical E2E suite and harness from main at f91774e2f9. Preserve replacement integration coverage for cases moved out of E2E. Keep the older integration harness compatible by omitting its unsupported shares_worker_state marker. Application code is unchanged.
This commit is contained in:
parent
61fe5be5b8
commit
98deebe78c
107 changed files with 6874 additions and 3524 deletions
|
|
@ -172,12 +172,7 @@ def open_consent() -> None:
|
|||
raise RuntimeError("consent page timed out")
|
||||
|
||||
|
||||
@pytest.mark.mcp_oauth_live
|
||||
def test_oauth_dies_on_consent() -> None:
|
||||
open_consent()
|
||||
|
||||
|
||||
def test_plain_dies_on_consent() -> None:
|
||||
def test_dies_on_consent() -> None:
|
||||
open_consent()
|
||||
"""
|
||||
|
||||
|
|
@ -353,16 +348,13 @@ class TestStepsReachTheReport:
|
|||
|
||||
def test_a_failed_phase_s_own_report_carries_the_steps(tmp_path: Path) -> None:
|
||||
"""Plugins that read the failed setup or call report, not the teardown one
|
||||
junitxml writes from, see where the test died too, oauth-live or not."""
|
||||
junitxml writes from, see where the test died too."""
|
||||
write_suite(tmp_path, {"test_consent.py": FAILED_PHASE_SUITE, "report_spy.py": REPORT_SPY_PLUGIN})
|
||||
child: Final = run_child_pytest(tmp_path, "-p", "report_spy", env={"E2E_MCP_OAUTH_LIVE": "1"})
|
||||
child: Final = run_child_pytest(tmp_path, "-p", "report_spy")
|
||||
seen_path: Final = tmp_path / "failed-reports.jsonl"
|
||||
assert seen_path.exists(), f"no failed report reached the spy:\n{child.stdout}\n{child.stderr}"
|
||||
seen: Final = dict(map(FailedReport.validate_json, seen_path.read_text().splitlines()))
|
||||
assert seen == {
|
||||
"test_oauth_dies_on_consent": ("open the consent page",),
|
||||
"test_plain_dies_on_consent": ("open the consent page",),
|
||||
}, child.stdout
|
||||
assert seen == {"test_dies_on_consent": ("open the consent page",)}, child.stdout
|
||||
|
||||
|
||||
class TestDeclaredPropertiesReachTheReport:
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -90,18 +90,7 @@ The suites run against a live proxy, so bring one up first by running the litell
|
|||
|
||||
They also need a proxy whose bundled UI contains the change under test, so run the proxy from your branch (an editable install serves the UI your checkout builds)
|
||||
|
||||
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and point `PHOENIX_COLLECTOR_HTTP_ENDPOINT` at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy. The presidio guardrail tests need a running Presidio analyzer and anonymizer the proxy can reach, addressed by `PRESIDIO_ANALYZER_API_BASE` / `PRESIDIO_ANONYMIZER_API_BASE`
|
||||
|
||||
The Presidio spend-log audit test also requires prompt storage on the proxy, because its assertions inspect the detected entities and their scores. Add the following to the proxy config before starting it:
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
store_prompts_in_spend_logs: true
|
||||
```
|
||||
|
||||
Alternatively, start the proxy with `STORE_PROMPTS_IN_SPEND_LOGS=true litellm --config <your-e2e-config>.yml --port 4000`. Setting the variable only on the pytest process does not configure the proxy. With prompt storage disabled, the proxy correctly redacts `guardrail_response`, so that configuration cannot exercise this test's entity-detail assertions. Enable this only on a test stack using synthetic prompts
|
||||
|
||||
A couple of logging destinations are configured on the proxy rather than by the test. The Weave tests scope their callback to the key they create, but litellm builds the `weave_otel` logger from `WANDB_API_KEY` and `WANDB_PROJECT_ID` before it applies the per-key vars, so the proxy needs both in its own environment or the key-scoped callback never initializes and nothing ships
|
||||
Some suites need extra services the bare proxy does not start. The `logging/` OTEL trace-completeness tests read spans back from a jaeger query API at `http://localhost:16686` (override with `E2E_OTEL_QUERY_URL`); run a `jaegertracing/all-in-one` and start the proxy with `LITELLM_OTEL_V2=true`, the `otel` callback, and `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at its OTLP ingest. The `mcp/` suite needs the deterministic upstream MCP server in `mcp_tests/mcp_e2e_upstream_server.py` reachable by the proxy.
|
||||
|
||||
### Secret manager lanes
|
||||
|
||||
|
|
@ -261,55 +250,3 @@ The semantic header set is `content-type`, `accept`, `anthropic-version`, `anthr
|
|||
Excluded transport and telemetry headers are `host`, `content-length`, `connection`, `accept-encoding`, `user-agent`, `traceparent`, `tracestate`, `x-request-id`, `x-client-request-id` and `x-stainless-*`. Inbound transfer-encoding is unsupported; send JSON with content-length framing. The destination represents host identity and the relay carries original body bytes. Replay does not verify credentials, SDK timeout/retry behavior, transport performance, model availability or stateful remote IDs. Live relay uses original request bytes and header values, never the stored identity
|
||||
|
||||
Strict replay harness regression tests live in `tests/code_coverage_tests/test_provider_replay_harness.py`. The `python` job in `.github/workflows/test-linting.yml` runs them alongside the provider-edge and fixture tests at the root of `tests/e2e_harness/` with `--noconftest -o addopts= -o "pythonpath=tests/e2e tests/e2e_harness" -p no:rerunfailures`; they need only synthetic HTTP providers and temporary fixture storage
|
||||
|
||||
|
||||
## MCP OAuth happy path
|
||||
|
||||
`test_mcp_oauth_happy_path_e2e.py` runs one shared scenario with four variants:
|
||||
aggregate gateway SSO and explicitly configured per-server JWT, each directly
|
||||
against Linear and through the live provider edge. The edge forwards to real
|
||||
Linear without replay and compares the forwarded bearer to the encrypted
|
||||
canonical user/server credential. This observes the forwarding boundary, not
|
||||
Linear's internal logs. Direct variants independently exercise discovery
|
||||
|
||||
Use the existing database preparation, Prisma generation and Keycloak setup.
|
||||
Build and stage the dashboard from the tested checkout as in the UI runner.
|
||||
Provide `DATABASE_URL`, `LITELLM_MASTER_KEY`, `LITELLM_SALT_KEY`, `LITELLM_LICENSE`,
|
||||
and the `E2E_KEYCLOAK_*` settings. Capture a test-account Linear login using
|
||||
`mcp/linear_session_capture.py` and set `E2E_LINEAR_STORAGE_STATE` to that private
|
||||
file. The test workspace must contain a team. Do not publish browser state or
|
||||
raw test/proxy output
|
||||
|
||||
```bash
|
||||
E2E_MCP_OAUTH_LIVE=1 E2E_FIXTURE_MODE=live E2E_PROVIDER_CACHE=0 \
|
||||
uv run --no-sync pytest tests/e2e/mcp/test_mcp_oauth_happy_path_e2e.py \
|
||||
--rootdir=. --reruns 0
|
||||
```
|
||||
|
||||
The test starts and restarts its own source-built proxy on a free loopback port,
|
||||
retaining its database and SSO client but no Redis or process-local cache. It
|
||||
does not restart an existing proxy or clear shared databases. Gateway login,
|
||||
consent, immediate list/call and post-restart reconnect must all succeed. The
|
||||
aggregate client never injects a gateway header; the explicitly labeled JWT
|
||||
variant configures `x-litellm-api-key` for the first consent and reconnects with
|
||||
only its gateway JWT after restart
|
||||
|
||||
`.github/workflows/test-mcp-oauth-e2e.yml` automatically requests a run for
|
||||
same-repository pull requests changing MCP, gateway authentication/SSO, consent
|
||||
UI, dependencies or the relevant E2E harness/workflow paths. It retains manual
|
||||
`workflow_dispatch` for targeted verification. The four cases run in the
|
||||
protected `e2e-changed` environment after its normal deployment approval;
|
||||
reviewers should approve and inspect this separate OAuth check when it appears.
|
||||
Fork pull requests do not run this credentialed job; use a reviewed
|
||||
same-repository branch for their verification. The workflow's path-filtered
|
||||
check is not configured here as a globally required branch-protection check.
|
||||
Provision `E2E_LINEAR_STORAGE_STATE_B64` as a secret there and retain the existing E2E license/AWS role configuration. A missing or
|
||||
expired session fails the job; collection, deselection and skips are not passes.
|
||||
No LLM call is needed
|
||||
|
||||
Coverage remains limited to authorization-code OAuth over HTTP. M2M, OBO,
|
||||
PKCE passthrough, static/BYOK, ID-JAG, forwarding, SigV4 and stdio are outside this
|
||||
scenario; consult the registry and LIT-3559 for their existing coverage and gaps.
|
||||
LIT-4506 owns broader isolation/failure regressions. LIT-7737 retains ownership
|
||||
of dependency/Python compatibility and its matrix; this test reuses its delivered
|
||||
environment and does not change dependency constraints or compatibility gates
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from dataclasses import dataclass
|
|||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from e2e_config import settle_propagation
|
||||
from e2e_http import NoBody, Result, Success, get_external, is_ok
|
||||
from e2e_http import NoBody, Result, Success, is_ok
|
||||
from e2e_metadata import STEP_FRAMES, step
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
|
|
@ -36,11 +36,6 @@ class A2ASkill(BaseModel):
|
|||
examples: list[str] | None = None
|
||||
|
||||
|
||||
class A2AProvider(BaseModel):
|
||||
organization: str
|
||||
url: str
|
||||
|
||||
|
||||
class AgentCardParams(BaseModel):
|
||||
"""The upstream agent card an admin registers. `protocolVersion` is the field the
|
||||
proxy validates against SUPPORTED_A2A_PROTOCOL_VERSIONS on registration."""
|
||||
|
|
@ -57,29 +52,6 @@ class AgentCardParams(BaseModel):
|
|||
preferred_transport: str | None = Field(default=None, serialization_alias="preferredTransport")
|
||||
|
||||
|
||||
class UpstreamAgentCard(BaseModel):
|
||||
"""A real published agent card parsed from a public /.well-known endpoint. Keys on
|
||||
the A2A wire aliases so `model_validate_json` reads the served JSON and
|
||||
`model_dump(by_alias=True)` re-emits it unchanged for verbatim registration; it is
|
||||
only ever fetched-and-validated, never hand-constructed, so aliasing on the wire
|
||||
names does not affect any call site."""
|
||||
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
protocol_version: str = Field(alias="protocolVersion")
|
||||
name: str
|
||||
description: str
|
||||
version: str
|
||||
url: str
|
||||
provider: A2AProvider | None = None
|
||||
documentation_url: str | None = Field(default=None, alias="documentationUrl")
|
||||
capabilities: A2ACapabilities = A2ACapabilities()
|
||||
skills: list[A2ASkill]
|
||||
default_input_modes: list[str] = Field(default=["text"], alias="defaultInputModes")
|
||||
default_output_modes: list[str] = Field(default=["text"], alias="defaultOutputModes")
|
||||
preferred_transport: str | None = Field(default=None, alias="preferredTransport")
|
||||
|
||||
|
||||
class A2ABridgeParams(BaseModel):
|
||||
"""litellm_params that route the agent through the completion bridge: an A2A
|
||||
message/send is transformed into a litellm.acompletion against this provider."""
|
||||
|
|
@ -93,7 +65,7 @@ class A2ABridgeParams(BaseModel):
|
|||
|
||||
class AgentRegisterBody(BaseModel):
|
||||
agent_name: str
|
||||
agent_card_params: AgentCardParams | UpstreamAgentCard
|
||||
agent_card_params: AgentCardParams
|
||||
litellm_params: A2ABridgeParams | None = None
|
||||
|
||||
|
||||
|
|
@ -136,32 +108,9 @@ class A2ATextPart(BaseModel):
|
|||
text: str
|
||||
|
||||
|
||||
class A2ASearchPropertiesParams(BaseModel):
|
||||
"""The strict param schema of the published property agent's `search_properties`
|
||||
skill (unknown keys are rejected upstream), so a natural-language query like
|
||||
"properties for sale in SF under $2M" is expressed as typed fields."""
|
||||
|
||||
un_locode: str | None = None
|
||||
service_type: str | None = None
|
||||
property_type: str | None = None
|
||||
bedrooms_min: int | None = None
|
||||
asking_price_max: float | None = None
|
||||
limit: int | None = None
|
||||
|
||||
|
||||
class A2ASkillInvocation(BaseModel):
|
||||
skill: str
|
||||
params: A2ASearchPropertiesParams
|
||||
|
||||
|
||||
class A2ADataPart(BaseModel):
|
||||
kind: str = "data"
|
||||
data: A2ASkillInvocation
|
||||
|
||||
|
||||
class A2AOutboundMessage(BaseModel):
|
||||
role: str = "user"
|
||||
parts: list[A2ATextPart | A2ADataPart]
|
||||
parts: list[A2ATextPart]
|
||||
message_id: str = Field(serialization_alias="messageId")
|
||||
|
||||
|
||||
|
|
@ -189,57 +138,10 @@ class A2AResponseMessage(BaseModel):
|
|||
parts: list[A2AResponsePart] = []
|
||||
|
||||
|
||||
class A2ATaskStatus(BaseModel):
|
||||
state: str | None = None
|
||||
message: A2AResponseMessage | None = None
|
||||
|
||||
|
||||
class A2AListingLocation(BaseModel):
|
||||
"""Only the location fields a test reads back off a returned listing."""
|
||||
|
||||
un_locode: str | None = None
|
||||
|
||||
|
||||
class A2AListing(BaseModel):
|
||||
"""A single property card from the agent's `search_results` artifact; only the
|
||||
identity/location fields a test asserts on are modelled."""
|
||||
|
||||
raia_id: str
|
||||
property_type: str | None = None
|
||||
service_type: str | None = None
|
||||
location: A2AListingLocation = A2AListingLocation()
|
||||
|
||||
|
||||
class A2ASearchResults(BaseModel):
|
||||
"""The DataPart payload the property agent's `search_properties` skill returns:
|
||||
the run count plus the listing cards themselves. Proof the tool actually ran and
|
||||
matched, not just that the task completed with some text."""
|
||||
|
||||
total: int
|
||||
count: int
|
||||
listings: list[A2AListing] = []
|
||||
|
||||
|
||||
class A2AArtifactPart(BaseModel):
|
||||
kind: str | None = None
|
||||
data: A2ASearchResults | None = None
|
||||
|
||||
|
||||
class A2AArtifact(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
artifact_id: str | None = Field(default=None, alias="artifactId")
|
||||
name: str | None = None
|
||||
parts: list[A2AArtifactPart] = []
|
||||
|
||||
|
||||
class A2AResult(BaseModel):
|
||||
"""A message/send result. In 0.3 the message fields sit directly on the result
|
||||
(`kind`/`role`/`parts`); in 1.0 they are nested under `message`; a real agent that
|
||||
runs a task replies with a `task` whose agent text lives on `status.message` and
|
||||
whose tool output lives on `artifacts`. `text` reads the agent's reply from
|
||||
whichever shape the served version produced; `search_results` reads the tool's
|
||||
structured output when the agent ran a skill."""
|
||||
(`kind`/`role`/`parts`); in 1.0 they are nested under `message`. `text` reads the
|
||||
agent's reply from whichever shape the served version produced."""
|
||||
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
|
|
@ -248,8 +150,6 @@ class A2AResult(BaseModel):
|
|||
message_id: str | None = Field(default=None, alias="messageId")
|
||||
parts: list[A2AResponsePart] = []
|
||||
message: A2AResponseMessage | None = None
|
||||
status: A2ATaskStatus | None = None
|
||||
artifacts: list[A2AArtifact] = []
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
|
|
@ -257,8 +157,6 @@ class A2AResult(BaseModel):
|
|||
parts = self.message.parts
|
||||
elif self.parts:
|
||||
parts = self.parts
|
||||
elif self.status is not None and self.status.message is not None:
|
||||
parts = self.status.message.parts
|
||||
else:
|
||||
parts = []
|
||||
return "".join(part.text or "" for part in parts)
|
||||
|
|
@ -267,14 +165,6 @@ class A2AResult(BaseModel):
|
|||
def is_nested_v1_shape(self) -> bool:
|
||||
return self.message is not None
|
||||
|
||||
@property
|
||||
def search_results(self) -> A2ASearchResults | None:
|
||||
for artifact in self.artifacts:
|
||||
for part in artifact.parts:
|
||||
if part.data is not None:
|
||||
return part.data
|
||||
return None
|
||||
|
||||
|
||||
class A2AError(BaseModel):
|
||||
code: int
|
||||
|
|
@ -380,11 +270,3 @@ class A2AClient:
|
|||
|
||||
def build_a2a_client(proxy: ProxyClient) -> A2AClient:
|
||||
return A2AClient(proxy=proxy)
|
||||
|
||||
|
||||
@step("Fetch a published A2A agent card from its /.well-known endpoint")
|
||||
def fetch_agent_card(url: str, *, timeout: float = 20.0) -> Result[UpstreamAgentCard]:
|
||||
"""Fetch a live A2A agent card from its /.well-known endpoint and parse it into the
|
||||
registration model, so a test can register a real published card verbatim rather
|
||||
than a hand-rolled one."""
|
||||
return get_external(url, response_type=UpstreamAgentCard, timeout=timeout)
|
||||
|
|
|
|||
|
|
@ -17,18 +17,14 @@ import pytest
|
|||
from a2a_client import (
|
||||
A2ABridgeParams,
|
||||
A2AClient,
|
||||
A2ADataPart,
|
||||
A2AJsonRpcRequest,
|
||||
A2AMessageSendParams,
|
||||
A2AOutboundMessage,
|
||||
A2ASearchPropertiesParams,
|
||||
A2ASkill,
|
||||
A2ASkillInvocation,
|
||||
A2ATextPart,
|
||||
AgentCardParams,
|
||||
AgentRegisterBody,
|
||||
AgentResponse,
|
||||
fetch_agent_card,
|
||||
)
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import Result, UnknownApiError, unwrap
|
||||
|
|
@ -50,9 +46,6 @@ BRIDGE = A2ABridgeParams(
|
|||
model=BRIDGE_MODEL,
|
||||
)
|
||||
|
||||
MOVEHOME_AGENT_CARD_URL = "https://movehome.org/.well-known/agent.json"
|
||||
MOVEHOME_ORIGIN = "https://movehome.org"
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
|
|
@ -135,48 +128,6 @@ class TestA2AAgentLifecycle:
|
|||
assert result is not None
|
||||
assert result.text != ""
|
||||
|
||||
@pytest.mark.covers("other.a2a.message_send.real_world_agent_replies")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.AGENTS_API,
|
||||
route=Route.A2A,
|
||||
)
|
||||
)
|
||||
def test_real_world_agent_replies_to_property_query(self, client: A2AClient, resources: ResourceManager, scoped_key: str) -> None:
|
||||
upstream = unwrap(fetch_agent_card(MOVEHOME_AGENT_CARD_URL)).model_copy(update={"url": MOVEHOME_ORIGIN})
|
||||
assert upstream.protocol_version == "0.3.0"
|
||||
marker = unique_marker()
|
||||
body = AgentRegisterBody(agent_name=f"e2e-a2a-real-{marker}", agent_card_params=upstream)
|
||||
agent = unwrap(client.register_agent(body))
|
||||
resources.defer(lambda: client.delete_agent(agent.agent_id))
|
||||
assert agent.agent_card_params.protocol_version == "0.3"
|
||||
location = "GBLON"
|
||||
request = A2AJsonRpcRequest(
|
||||
id=f"e2e-{unique_marker()}",
|
||||
params=A2AMessageSendParams(
|
||||
message=A2AOutboundMessage(
|
||||
parts=[
|
||||
A2ADataPart(
|
||||
data=A2ASkillInvocation(
|
||||
skill="search_properties",
|
||||
params=A2ASearchPropertiesParams(un_locode=location, service_type="long_term", limit=3),
|
||||
)
|
||||
)
|
||||
],
|
||||
message_id=unique_marker(),
|
||||
)
|
||||
),
|
||||
)
|
||||
response = unwrap(client.send_message(agent.agent_id, scoped_key, request))
|
||||
assert response.error is None
|
||||
assert response.result is not None
|
||||
results = response.result.search_results
|
||||
assert results is not None, "agent returned no search_results artifact; skill did not run"
|
||||
assert results.total > 0
|
||||
assert results.listings, "search_properties matched nothing; agent returned no property cards"
|
||||
assert all(listing.raia_id for listing in results.listings)
|
||||
assert all(listing.location.un_locode == location for listing in results.listings)
|
||||
|
||||
@pytest.mark.covers("other.a2a.discovery.proxy_fronted_card")
|
||||
@meta(
|
||||
Subject(
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from __future__ import annotations
|
|||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from e2e_metadata import step
|
||||
from proxy_client import ProxyClient
|
||||
|
|
@ -77,6 +77,16 @@ class BatchErrorList(BaseModel):
|
|||
data: list[BatchErrorItem] = []
|
||||
|
||||
|
||||
class BatchUsage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
completion_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
input_tokens: int | None = None
|
||||
output_tokens: int | None = None
|
||||
|
||||
|
||||
class BatchObject(BaseModel):
|
||||
id: str
|
||||
object: str | None = None
|
||||
|
|
@ -90,6 +100,7 @@ class BatchObject(BaseModel):
|
|||
completion_window: str | None = None
|
||||
created_at: int | None = None
|
||||
model: str | None = None
|
||||
usage: BatchUsage | None = None
|
||||
|
||||
|
||||
class BatchList(BaseModel):
|
||||
|
|
|
|||
|
|
@ -1893,6 +1893,13 @@ class TestBatchTerminalState:
|
|||
f"listed-completed marker retrieved as {fetched.status!r}"
|
||||
)
|
||||
assert fetched.output_file_id, "completed batch has no output_file_id"
|
||||
usage: Final = fetched.usage
|
||||
assert usage is not None and usage.input_tokens and usage.output_tokens, (
|
||||
f"completed batch has no batch token counts: {usage!r}"
|
||||
)
|
||||
assert (usage.prompt_tokens, usage.completion_tokens) == (usage.input_tokens, usage.output_tokens), (
|
||||
f"chat token counts disagree with the batch token counts: {usage!r}"
|
||||
)
|
||||
|
||||
downloaded = client.proxy.transport.download(
|
||||
f"/v1/files/{fetched.output_file_id}/content",
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import functools
|
|||
import os
|
||||
from collections.abc import Generator, Iterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -29,7 +28,6 @@ from e2e_config import (
|
|||
FIXTURE_DIR,
|
||||
FIXTURE_MODE_RAW,
|
||||
MANAGED_FILES_OPT_IN_ENV,
|
||||
MCP_OAUTH_LIVE_OPT_IN_ENV,
|
||||
OTEL_TLS_OPT_IN_ENV,
|
||||
OTEL_V2_OPT_IN_ENV,
|
||||
OWNED_GATEWAY_OPT_IN_ENV,
|
||||
|
|
@ -72,7 +70,6 @@ OPT_IN_MARKERS: Final = MappingProxyType(
|
|||
"redis_chaos": PERF_TESTS_OPT_IN_ENV,
|
||||
"perf": PERF_TESTS_OPT_IN_ENV,
|
||||
"cli_determinism": CLI_DETERMINISM_OPT_IN_ENV,
|
||||
"mcp_oauth_live": MCP_OAUTH_LIVE_OPT_IN_ENV,
|
||||
"provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV,
|
||||
"owned_gateway": OWNED_GATEWAY_OPT_IN_ENV,
|
||||
"otel_v2": OTEL_V2_OPT_IN_ENV,
|
||||
|
|
@ -170,11 +167,6 @@ def pytest_configure(config: pytest.Config) -> None:
|
|||
"quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; "
|
||||
"every other test waits for it to finish",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless "
|
||||
"E2E_MCP_OAUTH_LIVE is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the "
|
||||
|
|
@ -331,8 +323,6 @@ def pytest_runtest_setup(item: pytest.Item) -> None:
|
|||
item.user_properties.extend(item.config.stash[_IDLE_RSS].junit_properties)
|
||||
if not _reaches_proxy(item):
|
||||
return
|
||||
if isinstance(item, pytest.Function) and "oauth_gateway" in item.fixturenames:
|
||||
return
|
||||
reason = _proxy_fail_reason()
|
||||
if reason is not None:
|
||||
pytest.fail(reason)
|
||||
|
|
@ -376,12 +366,6 @@ def pytest_runtest_makereport(
|
|||
report = yield
|
||||
if report.when in ("setup", "call"):
|
||||
attach_step_properties(item)
|
||||
if item.get_closest_marker("mcp_oauth_live") is not None and call.excinfo is not None:
|
||||
# Publish code locations only, never exception messages, source text or locals.
|
||||
item.user_properties.append(("oauth_failure_phase", report.when))
|
||||
item.user_properties.append(("oauth_exception_type", call.excinfo.type.__name__))
|
||||
for entry in call.excinfo.traceback:
|
||||
item.user_properties.append(("oauth_frame", f"{Path(entry.path).name}:{entry.lineno + 1}:{entry.name}"))
|
||||
if report.when == "call":
|
||||
item.stash[_CALL_PASSED] = report.passed
|
||||
report.user_properties = list(item.user_properties)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,6 @@
|
|||
# Guardrail enforcement (behavior features). Grounded in litellm/proxy/guardrails/guardrail_hooks/.
|
||||
# Rolls up into the "Logging & Guardrails" dashboard module together with logging.*
|
||||
- {id: guardrail.presidio.pre_call.masks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "PII masking pre-call; data-leak blast radius"}
|
||||
- {id: guardrail.presidio.post_call.masks, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Mask PII in model output"}
|
||||
- {id: guardrail.presidio.post_call.masks_generated_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, anthropic_messages_stream], source: "guardrail_hooks/presidio.py", rationale: "Mask model-generated credit-card output with the UI default scope"}
|
||||
- {id: guardrail.presidio.post_call.spend_log_stores_masked_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, messages, anthropic_messages_stream, responses], source: "guardrail_hooks/presidio.py", fail_before_fix: proven, rationale: "When an output guardrail masks the response, the spend log stores the masked text the caller received rather than the raw model output, on every endpoint and both stream modes (LIT-8325)"}
|
||||
- {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"}
|
||||
- {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"}
|
||||
- {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"}
|
||||
- {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"}
|
||||
- {id: guardrail.litellm_content_filter.pre_call.blocks_video, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [videos], source: "test_key_guardrail_video_e2e.py", fail_before_fix: proven, rationale: "A content-filter guardrail attached to a key (metadata.guardrails) blocks a banned prompt on POST /v1/videos before the provider is called; before the fix the route's call type was unknown to the unified guardrail hook and the prompt went to the provider unscanned (LIT-6685)"}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@
|
|||
- {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"}
|
||||
- {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"}
|
||||
- {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"}
|
||||
- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention sent through the proxy, litellm.completion or Router.completion reaches the body sent to OpenAI /v1/chat/completions, so a caller asking for 24h prompt caching gets it instead of a silent drop"}
|
||||
- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention on an openai/responses/ model (chat completions in, Responses out) reaches the body sent to OpenAI /v1/responses"}
|
||||
- {id: llm.chat_completions.openai.service_tier.stream.echoes_served_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: stream, assertions: [works], source: "litellm_core_utils/streaming_handler.py", fail_before_fix: proven, rationale: "Every relayed stream chunk carries the service_tier OpenAI stamped on it, so a streaming caller can see which tier served the request"}
|
||||
- {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"}
|
||||
- {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"}
|
||||
|
|
@ -43,6 +45,9 @@
|
|||
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
|
||||
- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
|
||||
- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}
|
||||
- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the AgentCore tool-use id shape"}
|
||||
- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the AgentCore tool-use id shape"}
|
||||
- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A short Gemini tool-call id still round-trips when the client drops provider fields"}
|
||||
- {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"}
|
||||
- {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"}
|
||||
|
||||
|
|
@ -68,13 +73,14 @@
|
|||
- {id: llm.messages.anthropic.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Messages API"}
|
||||
- {id: llm.messages.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching via Messages API"}
|
||||
- {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"}
|
||||
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven}
|
||||
- {id: llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works_for_kimi_k3_max_tokens_1], source: "llm_translation/test_bedrock_kimi_k3_messages_e2e.py", rationale: "Bedrock Converse accepts max_tokens=1 on the Anthropic Messages surface after LiteLLM applies the model minimum", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (#32831)", fail_before_fix: proven}
|
||||
- {id: llm.messages.bedrock_invoke.web_search_server_tool.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: web_search_server_tool, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py", rationale: "Bedrock hosts no web_search server tool, so this only works because interception rewrites it before the upstream call and the agentic loop feeds the results back in native shape; a regression that short-circuits or forwards it instead yields raw text or AWS's 400", fail_before_fix: unproven}
|
||||
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
|
||||
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
|
||||
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
|
||||
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
|
||||
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
|
||||
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must convert reminders to user turns in place (hoisting collapses the prompt cache) or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
|
||||
- {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
|
||||
- {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input and missing model are rejected"}
|
||||
- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
|
||||
|
|
@ -133,11 +139,6 @@
|
|||
- {id: llm.responses.ollama.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses streams text deltas, usage and a terminal event"}
|
||||
- {id: llm.responses.ollama.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses returns one addressable get_weather call"}
|
||||
- {id: llm.responses.ollama.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: ollama, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_ollama_e2e.py", rationale: "Ollama /api/generate (prompt-based tools) over /v1/responses tool result round trip reaches the model"}
|
||||
- {id: llm.chat_completions.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "service_tier balanced and auto map to Sail completion windows and bill the matching price columns; Sail serves flex only to background responses and Batch"}
|
||||
- {id: llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [rejects_unknown_tier], source: "llm_translation/test_sail_e2e.py", rationale: "A service_tier Sail has no completion window for is a 400 without drop_params"}
|
||||
- {id: llm.chat_completions.sail.service_tier.nonstream.drops_unknown_tier_and_bills_asap, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [drops_unknown_tier_and_bills_asap], source: "llm_translation/test_sail_e2e.py", rationale: "An unknown service_tier under drop_params is dropped and billed at asap in both the cost header and spend log"}
|
||||
- {id: llm.responses.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "A caller metadata.completion_window of balanced on /v1/responses bills Sail balanced rates"}
|
||||
- {id: llm.messages.sail.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: sail, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_sail_e2e.py", rationale: "Sail over /v1/messages"}
|
||||
- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"}
|
||||
- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"}
|
||||
- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"}
|
||||
|
|
@ -152,3 +153,4 @@
|
|||
- {id: llm.responses.anthropic.basic.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses streams Responses events"}
|
||||
- {id: llm.responses.anthropic.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /v1/responses: cost header and spend row agree"}
|
||||
- {id: llm.responses.anthropic.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic function_call_output round trip over /v1/responses"}
|
||||
- {id: llm.responses.anthropic.tool_search.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: tool_search, streaming: nonstream, assertions: [works], source: "llm_translation/test_responses_codex_tool_search_e2e.py", rationale: "Codex client tool_search over /v1/responses finds and loads a deferred tool on Claude"}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
- {id: llm.completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_completions_endpoint_e2e.py", rationale: "Legacy text /completions endpoint, second-highest production request volume"}
|
||||
- {id: llm.embeddings.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_embeddings_endpoint_e2e.py:23", rationale: "Core endpoint, live vector response"}
|
||||
- {id: llm.embeddings.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.3 / LIT-4778", rationale: "Missing model/input on /embeddings return client errors"}
|
||||
- {id: llm.embeddings.openai.supported_endpoints_gate.nonstream.works, module: llm, tier: P1, subject_endpoint: embeddings, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], fail_before_fix: proven, source: "test_embedding_only_supported_endpoints_e2e.py", rationale: "Explicit endpoint restrictions preserve real embeddings, refuse chat with a gateway 400, and route chat through a live fallback; zero-provider-call contracts live in integration/routing/test_non_chat_model_endpoint_gate.py"}
|
||||
- {id: llm.embeddings.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "SPEND_TRACKING_COVERAGE_MATRIX.md:34", rationale: "Cost tracking on embeddings"}
|
||||
- {id: llm.embeddings.openai.passthrough.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: embeddings, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "POST /openai_passthrough/v1/embeddings is costed; the route wrote no spend row at all, so budgets never saw traffic OpenAI was billing for (GitHub issue #36646)"}
|
||||
- {id: llm.embeddings.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: embeddings, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/azure/azure.py", rationale: "Azure embeddings via translation"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# MCP module. Grounded in litellm/proxy/_experimental/mcp_server/. See tests/e2e/AGENTS.md for the grammar.
|
||||
# MCP module. Grounded in litellm/proxy/mcp_server/. See tests/e2e/AGENTS.md for the grammar.
|
||||
- id: mcp.list_tools.api_key.succeeds
|
||||
module: mcp
|
||||
tier: P0
|
||||
|
|
@ -79,14 +79,6 @@
|
|||
assertions: [succeeds]
|
||||
source: "db.py user_oauth_credential lookup"
|
||||
rationale: OAuth2 token passthrough; per-user credential storage
|
||||
- id: mcp.call_tool.oauth.persists_across_processes
|
||||
module: mcp
|
||||
tier: P1
|
||||
operation: call_tool
|
||||
auth_family: oauth
|
||||
assertions: [persists_across_processes]
|
||||
source: "outbound_credentials/per_user_oauth_store.py V2PerUserTokenStore"
|
||||
rationale: Stored per-user token survives a verified restart of an owned gateway with no Redis cache
|
||||
- id: mcp.list_tools.none.succeeds
|
||||
module: mcp
|
||||
tier: P1
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@
|
|||
- {id: other.a2a.register.persists, module: other, tier: P1, area: a2a, assertions: [persists], source: "agent_endpoints/endpoints.py:325-443", rationale: "POST /v1/agents registers an agent card; GET /v1/agents/{id} reads it back"}
|
||||
- {id: other.a2a.register.unsupported_version_rejected, module: other, tier: P1, area: a2a, assertions: [unsupported_version_rejected], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a protocolVersion outside SUPPORTED_A2A_PROTOCOL_VERSIONS is refused with 400"}
|
||||
- {id: other.a2a.register.semver_version_accepted, module: other, tier: P1, area: a2a, assertions: [semver_version_accepted], source: "agent_endpoints/endpoints.py _validate_protocol_version", rationale: "A card pinning a patch-level semver like 0.3.0 (what the Google A2A SDK emits) registers, stores and serves the canonical 0.3 rather than 400ing; regression guard for the v1.92 report"}
|
||||
- {id: other.a2a.message_send.real_world_agent_replies, module: other, tier: P1, area: a2a, assertions: [real_world_agent_replies], source: "agent_endpoints/a2a_endpoints.py asend_message", rationale: "A real published a2a agent fetched live from a public /.well-known endpoint (pinning the full semver 0.3.0 the a2a-sdk emits) registers, serves the canonical 0.3, and a message/send skill invocation proxies to the live upstream and returns the agent's reply"}
|
||||
- {id: other.a2a.register.malformed_version_rejected, module: other, tier: P1, area: a2a, assertions: [malformed_version_rejected], source: "a2a/agent_card.py normalize_protocol_version", rationale: "A malformed protocolVersion like 0.3.garbage fails full-string semver validation and is refused with 400 instead of truncating to a supported family"}
|
||||
- {id: other.a2a.discovery.proxy_fronted_card, module: other, tier: P1, area: a2a, assertions: [proxy_fronted_card], source: "agent_endpoints/a2a_endpoints.py get_agent_card", rationale: "/.well-known/agent-card.json serves the proxy url + supportedInterfaces and the LiteLLM virtual-key bearer scheme, not the upstream"}
|
||||
- {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"}
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@
|
|||
- {id: quota_management.budget.spend_counter.reseed_matches_db, module: quota_management, tier: P2, behavior: budget, variant: spend_counter, assertions: [reseed_matches_db], exercised_on: [chat_completions], source: "proxy/spend_tracking/budget_reservation.py", rationale: "Concurrent cold-counter reseeds keep the enforcement counter equal to DB spend (#26829)"}
|
||||
- {id: quota_management.spend_tracking.chat_completions.logs_cost, module: quota_management, tier: P0, behavior: spend_tracking, variant: chat_completions, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A paid chat call writes a nonzero spend row"}
|
||||
- {id: quota_management.spend_tracking.stream.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Streaming responses aggregate token counts into a spend row"}
|
||||
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
|
||||
- {id: quota_management.spend_tracking.messages_bridge.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [logs_cost], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A streaming /v1/messages request served by an openai-provider model is bridged through the anthropic-messages -> Responses adapter and must aggregate the consumed SSE stream into one spend row with nonzero cost and token counts, attributed to custom_llm_provider openai under call_type anthropic_messages"}
|
||||
- {id: quota_management.spend_tracking.embeddings.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: embeddings, assertions: [logs_cost], exercised_on: [embeddings], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "Embedding calls write nonzero spend rows"}
|
||||
- {id: quota_management.spend_tracking.cache_hit.zero_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_hit, assertions: [zero_cost], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "A response-cache hit logs at zero cost with the cache-hit marker"}
|
||||
- {id: quota_management.spend_tracking.discount_config.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: discount_config, assertions: [logs_cost], exercised_on: [chat_completions], source: "proxy/management_endpoints/cost_tracking_settings.py + cost_calculator.py", rationale: "A persisted provider discount changes the real request's spend row"}
|
||||
|
|
@ -62,11 +62,11 @@
|
|||
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
|
||||
- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"}
|
||||
- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"}
|
||||
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
|
||||
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/messages/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
|
||||
- {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"}
|
||||
- {id: quota_management.spend_tracking.service_tier_stream.records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", fail_before_fix: proven, rationale: "A streamed call with no service_tier requested bills at the rates of the tier OpenAI stamps on its chunks and records that served tier on the row; the reassembled stream dropped the provider tier so the row recorded none and priced at the default rates"}
|
||||
- {id: quota_management.spend_tracking.service_tier_stream.responses_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [responses], source: "responses/streaming_iterator.py", rationale: "A streamed /v1/responses call bills at the tier carried on the response.completed event's inner response and records that served tier on the spend row"}
|
||||
- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/pass_through/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"}
|
||||
- {id: quota_management.spend_tracking.service_tier_stream.messages_records_served_tier, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier_stream, assertions: [records_served_tier], exercised_on: [messages], source: "llms/anthropic/messages/adapters/streaming_iterator.py", rationale: "A streamed /v1/messages call on an OpenAI-backed deployment bills at the tier OpenAI served; the Anthropic wire format has no tier field, so the spend row is the only record of it"}
|
||||
- {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"}
|
||||
- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"}
|
||||
- {id: quota_management.spend_tracking.websearch_interception.bills_under_request_session, module: quota_management, tier: P1, behavior: spend_tracking, variant: websearch_interception, assertions: [bills_under_request_session], exercised_on: [messages], source: "integrations/websearch_interception/handler.py", fail_before_fix: proven, rationale: "A web_search server tool the proxy intercepts into litellm.asearch writes its own asearch spend row, and that row carries the parent request's session_id so the session view counts the search and its cost next to the turn that triggered it (LIT-8063)"}
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ LlmCapability = Literal[
|
|||
"pdf_input",
|
||||
"logprobs",
|
||||
"prompt_cache_1h",
|
||||
"prompt_cache_24h",
|
||||
"prompt_cache_5m",
|
||||
"response_headers",
|
||||
"service_tier",
|
||||
|
|
|
|||
|
|
@ -111,10 +111,6 @@ CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haik
|
|||
CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5")
|
||||
S3_PARTITION_GRANULARITY = os.environ.get("E2E_S3_PARTITION_GRANULARITY", "day")
|
||||
|
||||
LINEAR_MCP_URL = os.environ.get("E2E_LINEAR_MCP_URL", "https://mcp.linear.app/mcp")
|
||||
LINEAR_STORAGE_STATE = os.environ.get("E2E_LINEAR_STORAGE_STATE", "")
|
||||
LINEAR_READONLY_TOOL: Final = "list_teams" # as listed by tools/list on mcp.linear.app when PR #33787 landed
|
||||
|
||||
# Jaeger query API of the compose stack's OTEL trace destination (the `jaeger`
|
||||
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
|
||||
# read exported spans back through it.
|
||||
|
|
@ -207,7 +203,6 @@ MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK"
|
|||
PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK"
|
||||
PERF_TESTS_OPT_IN_ENV: Final = "E2E_PERF_TESTS"
|
||||
CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM"
|
||||
MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
|
||||
PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE"
|
||||
OWNED_GATEWAY_OPT_IN_ENV: Final = "E2E_OWNED_GATEWAY"
|
||||
OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2"
|
||||
|
|
|
|||
|
|
@ -499,6 +499,30 @@ def get_external[R: BaseModel](
|
|||
return classify(resp, response_type)
|
||||
|
||||
|
||||
def get_text_external(
|
||||
url: str,
|
||||
*,
|
||||
headers: BaseModel | None = None,
|
||||
timeout: float = 30.0,
|
||||
ca_bundle: str | None = None,
|
||||
) -> str | NetworkError:
|
||||
"""GET an absolute URL outside the proxy whose body is plain text (e.g. a pod log), not JSON.
|
||||
A non-2xx status is reported as a NetworkError so the caller can fail loudly. JSON is accepted too so an API
|
||||
that answers an error as a JSON document (the Kubernetes Status on a pod log) returns that error instead of a 406."""
|
||||
try:
|
||||
resp = requests.get(
|
||||
url,
|
||||
headers={"Accept": "text/plain, application/json", **(_headers(headers) if headers is not None else {})},
|
||||
timeout=timeout,
|
||||
verify=ca_bundle if ca_bundle is not None else True,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return NetworkError(message=str(exc))
|
||||
if not (200 <= resp.status_code < 300):
|
||||
return NetworkError(message=f"{resp.status_code} from {url}: {resp.text[:300]}")
|
||||
return resp.text
|
||||
|
||||
|
||||
def post_external[R: BaseModel](
|
||||
url: str,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ litellm_settings:
|
|||
callbacks:
|
||||
- prometheus
|
||||
- otel
|
||||
service_callback:
|
||||
- prometheus_system
|
||||
require_auth_for_metrics_endpoint: false
|
||||
drop_params: true
|
||||
request_timeout: 120
|
||||
|
|
@ -44,6 +46,11 @@ model_list:
|
|||
model: openai/gpt-6-luna
|
||||
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
|
||||
api_key: perf-mock-not-checked
|
||||
- model_name: gpt-6-luna-responses
|
||||
litellm_params:
|
||||
model: openai/responses/gpt-6-luna
|
||||
api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL
|
||||
api_key: perf-mock-not-checked
|
||||
- model_name: claude-sonnet-5
|
||||
litellm_params:
|
||||
model: anthropic/claude-sonnet-5
|
||||
|
|
@ -56,6 +63,13 @@ model_list:
|
|||
aws_region_name: us-east-1
|
||||
aws_access_key_id: perf-mock-not-checked
|
||||
aws_secret_access_key: perf-mock-not-checked
|
||||
- model_name: claude-sonnet-5-bedrock-converse
|
||||
litellm_params:
|
||||
model: bedrock/converse/anthropic.claude-sonnet-5
|
||||
aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL
|
||||
aws_region_name: us-east-1
|
||||
aws_access_key_id: perf-mock-not-checked
|
||||
aws_secret_access_key: perf-mock-not-checked
|
||||
guardrails:
|
||||
- guardrail_name: perf-pre
|
||||
litellm_params:
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ litellm_settings:
|
|||
- host: 127.0.0.1
|
||||
port: 6379
|
||||
ssl: true
|
||||
callbacks: ["arize_phoenix", "datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
|
||||
callbacks: ["datadog", "smtp_email", "prometheus", "otel", "websearch_interception"]
|
||||
websearch_interception_params:
|
||||
enabled_providers: ["bedrock"]
|
||||
search_tool_name: e2e-search
|
||||
|
|
@ -97,9 +97,3 @@ files_settings:
|
|||
finetune_settings:
|
||||
- custom_llm_provider: openai
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
mcp_servers:
|
||||
devin:
|
||||
url: "https://mcp.devin.ai/mcp"
|
||||
auth_type: api_key
|
||||
auth_value: os.environ/DEVIN_API_KEY
|
||||
|
|
|
|||
|
|
@ -41,8 +41,6 @@ from pydantic import BaseModel, Field
|
|||
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
|
||||
|
||||
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
|
||||
PiiEntity = Literal["EMAIL_ADDRESS", "PHONE_NUMBER", "PERSON", "CREDIT_CARD", "US_SSN"]
|
||||
PiiAction = Literal["MASK", "BLOCK"]
|
||||
BlockedWordAction = Literal["BLOCK", "MASK"]
|
||||
|
||||
|
||||
|
|
@ -80,22 +78,6 @@ class BlockCodeExecutionParamsBody(GuardrailParamsBase):
|
|||
guardrail: Literal["block_code_execution"] = "block_code_execution"
|
||||
|
||||
|
||||
class PresidioParamsBody(GuardrailParamsBase):
|
||||
"""Presidio PII guardrail params. `presidio_filter_scope="input"` keeps the
|
||||
registration to a single callback on the configured mode; the default
|
||||
("both") also registers a second post_call output-masking callback, which a
|
||||
pre_call- or logging_only-scoped test must not drag in. `output_parse_pii`
|
||||
stays unset/False: True would unmask the response back to the caller."""
|
||||
|
||||
guardrail: Literal["presidio"] = "presidio"
|
||||
presidio_analyzer_api_base: str
|
||||
presidio_anonymizer_api_base: str
|
||||
presidio_filter_scope: Literal["input", "output", "both"] | None = None
|
||||
presidio_language: str | None = None
|
||||
output_parse_pii: bool | None = None
|
||||
pii_entities_config: dict[PiiEntity, PiiAction] | None = None
|
||||
|
||||
|
||||
class ToolPermissionRuleBody(BaseModel):
|
||||
"""One tool_permission rule: a decision for the tool named by `tool_name`."""
|
||||
|
||||
|
|
@ -130,7 +112,6 @@ GuardrailParamsBody = (
|
|||
| BedrockGuardrailParamsBody
|
||||
| OpenAIModerationParamsBody
|
||||
| BlockCodeExecutionParamsBody
|
||||
| PresidioParamsBody
|
||||
| ToolPermissionParamsBody
|
||||
| CustomCodeParamsBody
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,857 +0,0 @@
|
|||
"""Live e2e: the Presidio PII guardrail masks, per its configured hook point.
|
||||
|
||||
pre_call: the guardrail calls the Presidio analyzer/anonymizer on the request
|
||||
messages BEFORE the model runs, so the model only ever sees placeholders like
|
||||
<EMAIL_ADDRESS>. A prompt asking the model to repeat a fake email + phone back
|
||||
must come back with the placeholders echoed and the raw PII absent, on
|
||||
/chat/completions and on /v1/messages (Anthropic format).
|
||||
|
||||
post_call: the mirror hook. The request reaches the model unmasked and the
|
||||
MODEL OUTPUT is what gets anonymized, so the caller never receives raw PII the
|
||||
model repeated back. The two hooks are told apart behaviorally rather than by
|
||||
configuration: the post_call prompt asks for a value derived from the raw email
|
||||
(its local part, which is not itself an entity Presidio masks) alongside the
|
||||
address itself, so the answer proves the model saw the raw address while the
|
||||
address in the same response comes back as <EMAIL_ADDRESS>.
|
||||
|
||||
The analyzer/anonymizer endpoints come from PRESIDIO_ANALYZER_API_BASE /
|
||||
PRESIDIO_ANONYMIZER_API_BASE; missing env is a hard failure, never a skip.
|
||||
Each guardrail registers with an explicit presidio_filter_scope so only the
|
||||
configured hook's callback exists (the default "both" registers input masking
|
||||
AND a post_call output masker), and is deleted on teardown.
|
||||
|
||||
The spend-log audit test requires general_settings.store_prompts_in_spend_logs:
|
||||
true in the proxy config, or STORE_PROMPTS_IN_SPEND_LOGS=true in the proxy
|
||||
process environment before startup. Setting it only on pytest has no effect.
|
||||
Without this opt-in, redacting guardrail_response is expected proxy behavior;
|
||||
this suite deliberately requires the detected-entity details to remain visible.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import Result, StreamingResponse, Success
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from guardrails_client import GuardrailMode, GuardrailsClient, PiiAction, PiiEntity, PresidioParamsBody
|
||||
from lifecycle import ResourceManager
|
||||
from models import (
|
||||
AnthropicMessagesResponse,
|
||||
ChatResponse,
|
||||
GuardrailEntityMatch,
|
||||
GuardrailRunRecord,
|
||||
SpendLogRow,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
MODEL = "gemini-2.5-flash"
|
||||
|
||||
# A guardrail created via POST /guardrails reaches the worker that served the
|
||||
# create immediately, but every other worker only picks it up on its next
|
||||
# periodic DB sync (~30s), so the first requests can be served unguarded.
|
||||
GUARDRAIL_PROPAGATION_DEADLINE_SECONDS = 40.0
|
||||
GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS = 5.0
|
||||
|
||||
# Presidio's anonymizer replaces a detected entity with its unnumbered type
|
||||
# placeholder, e.g. <EMAIL_ADDRESS>. The pre_call assertions match on the bare
|
||||
# token because the model is echoing the masked prompt and may not preserve the
|
||||
# angle brackets; the logged payload keeps the placeholder verbatim.
|
||||
MASKED_EMAIL_TOKEN = "EMAIL_ADDRESS"
|
||||
MASKED_PHONE_TOKEN = "PHONE_NUMBER"
|
||||
MASKED_CREDIT_CARD_TOKEN = "CREDIT_CARD"
|
||||
|
||||
# Fictional NANP 555 number; a standard format Presidio's phone recognizer detects.
|
||||
FAKE_PHONE = "+1 415-555-0134"
|
||||
FAKE_VISA_TEST_CARD = "4111 1111 1111 1111"
|
||||
|
||||
_CARD_DIGIT_RUN: Final = re.compile(r"(?:\d[ -]?){13,19}")
|
||||
_CONTENT_KEYS: Final = frozenset({"content", "text"})
|
||||
|
||||
|
||||
def _presidio_bases() -> tuple[str, str]:
|
||||
analyzer = os.environ.get("PRESIDIO_ANALYZER_API_BASE", "").strip()
|
||||
anonymizer = os.environ.get("PRESIDIO_ANONYMIZER_API_BASE", "").strip()
|
||||
if not analyzer or not anonymizer:
|
||||
pytest.fail(
|
||||
"Presidio e2e requires PRESIDIO_ANALYZER_API_BASE and PRESIDIO_ANONYMIZER_API_BASE "
|
||||
"(the running Presidio analyzer/anonymizer services); missing env is a hard failure, not a skip"
|
||||
)
|
||||
return analyzer, anonymizer
|
||||
|
||||
|
||||
def _register_presidio(
|
||||
client: GuardrailsClient,
|
||||
resources: ResourceManager,
|
||||
*,
|
||||
name: str,
|
||||
mode: GuardrailMode | list[GuardrailMode] = "pre_call",
|
||||
filter_scope: Literal["input", "output", "both"] | None = "input",
|
||||
entities: dict[PiiEntity, PiiAction] | None = None,
|
||||
) -> None:
|
||||
analyzer, anonymizer = _presidio_bases()
|
||||
guardrail_id = client.register(
|
||||
name,
|
||||
PresidioParamsBody(
|
||||
mode=mode,
|
||||
default_on=False,
|
||||
presidio_analyzer_api_base=analyzer,
|
||||
presidio_anonymizer_api_base=anonymizer,
|
||||
presidio_filter_scope=filter_scope,
|
||||
pii_entities_config=entities,
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: client.delete_guardrail(guardrail_id))
|
||||
|
||||
|
||||
def _fake_email() -> str:
|
||||
return f"jane.doe.{unique_marker()}@example.com"
|
||||
|
||||
|
||||
def _pii_prompt(marker: str, email: str) -> str:
|
||||
return (
|
||||
f"{marker} Repeat this sentence back to me exactly, word for word: "
|
||||
f"My email address is {email} and my phone number is {FAKE_PHONE}."
|
||||
)
|
||||
|
||||
|
||||
def _first_content(response: ChatResponse) -> str:
|
||||
if not response.choices:
|
||||
return ""
|
||||
message = response.choices[0].message
|
||||
return (message.content if message else None) or ""
|
||||
|
||||
|
||||
class _StreamDelta(BaseModel):
|
||||
content: str | None = None
|
||||
|
||||
|
||||
class _StreamChoice(BaseModel):
|
||||
delta: _StreamDelta
|
||||
|
||||
|
||||
class _StreamChunk(BaseModel):
|
||||
choices: tuple[_StreamChoice, ...] = ()
|
||||
|
||||
|
||||
class _AnthropicStreamDelta(BaseModel):
|
||||
type: str | None = None
|
||||
text: str | None = None
|
||||
|
||||
|
||||
class _AnthropicStreamEvent(BaseModel):
|
||||
type: str
|
||||
delta: _AnthropicStreamDelta | None = None
|
||||
|
||||
|
||||
def _credit_card_prompt(marker: str) -> str:
|
||||
return (
|
||||
f"{marker} Reply with only the well known Visa sandbox test card number that starts with 4111, "
|
||||
"the 16 digits grouped in fours separated by spaces, and nothing else."
|
||||
)
|
||||
|
||||
|
||||
def _passes_luhn(digits: str) -> bool:
|
||||
checksum = sum(
|
||||
digit if position % 2 == 0 else (digit * 2 - 9 if digit * 2 > 9 else digit * 2)
|
||||
for position, digit in enumerate(int(char) for char in reversed(digits))
|
||||
)
|
||||
return checksum % 10 == 0
|
||||
|
||||
|
||||
def _contains_card_number(text: str) -> bool:
|
||||
"""Presidio's CREDIT_CARD recognizer only reports Luhn-valid digit runs, so a
|
||||
Luhn-invalid number the model hallucinates is not something masking can catch."""
|
||||
return any(
|
||||
13 <= len(digits) <= 19 and _passes_luhn(digits)
|
||||
for digits in (re.sub(r"[ -]", "", match.group()) for match in _CARD_DIGIT_RUN.finditer(text))
|
||||
)
|
||||
|
||||
|
||||
def _stream_content(result: StreamingResponse) -> str:
|
||||
return "".join(
|
||||
choice.delta.content
|
||||
for event in result.stream_events
|
||||
if event != "[DONE]"
|
||||
for choice in _StreamChunk.model_validate_json(event).choices[:1]
|
||||
if choice.delta.content
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_stream_content(result: StreamingResponse) -> str:
|
||||
return "".join(
|
||||
event.delta.text
|
||||
for payload in result.stream_events
|
||||
for event in [_AnthropicStreamEvent.model_validate_json(payload)]
|
||||
if event.type == "content_block_delta"
|
||||
and event.delta is not None
|
||||
and event.delta.type == "text_delta"
|
||||
and event.delta.text
|
||||
)
|
||||
|
||||
|
||||
def _messages_text(response: AnthropicMessagesResponse) -> str:
|
||||
"""The text of a /v1/messages answer, whichever shape the proxy produced
|
||||
(Anthropic-native content blocks or OpenAI-normalized choices)."""
|
||||
parts: list[str] = []
|
||||
for block in response.content or []:
|
||||
if block.text:
|
||||
parts.append(block.text)
|
||||
for choice in response.choices or []:
|
||||
if choice.message and choice.message.content:
|
||||
parts.append(choice.message.content)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _assert_eventually_masked[R: BaseModel](
|
||||
fetch: Callable[[], Result[R]], extract: Callable[[R], str], *, email: str
|
||||
) -> None:
|
||||
"""Retry the call until the response comes back masked, to the propagation
|
||||
deadline. An unmasked early response is in-flight guardrail propagation, not
|
||||
a failure, and neither is a transient non-Success (a replica that has not
|
||||
reloaded the guardrail answers 404, the live model can rate-limit) - only a
|
||||
response that still carries the raw PII at the deadline is."""
|
||||
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
|
||||
last: str = "<no successful response yet>"
|
||||
while True:
|
||||
result = fetch()
|
||||
match result:
|
||||
case Success(data=data):
|
||||
content = extract(data)
|
||||
last = content
|
||||
masked = MASKED_EMAIL_TOKEN in content and MASKED_PHONE_TOKEN in content and email not in content
|
||||
if masked:
|
||||
assert FAKE_PHONE not in content, (
|
||||
f"the raw phone number must be masked before the model sees it, but the "
|
||||
f"response echoed it: {content[:300]!r}"
|
||||
)
|
||||
return
|
||||
case _:
|
||||
last = f"<non-Success result: {result}>"
|
||||
if time.monotonic() >= deadline:
|
||||
pytest.fail(
|
||||
f"presidio pre_call guardrail never masked the PII within "
|
||||
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
|
||||
)
|
||||
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
class TestPresidioPreCallMasking:
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.pre_call.masks",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_pre_call_masks_pii_on_chat_completions(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
name = f"e2e-presidio-pre-chat-{unique_marker()}"
|
||||
_register_presidio(client, resources, name=name)
|
||||
|
||||
email = _fake_email()
|
||||
prompt = _pii_prompt(unique_marker(), email)
|
||||
|
||||
_assert_eventually_masked(
|
||||
lambda: client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128),
|
||||
_first_content,
|
||||
email=email,
|
||||
)
|
||||
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.pre_call.masks",
|
||||
exercised_on=["messages"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_pre_call_masks_pii_on_messages(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
name = f"e2e-presidio-pre-msg-{unique_marker()}"
|
||||
_register_presidio(client, resources, name=name)
|
||||
|
||||
email = _fake_email()
|
||||
prompt = _pii_prompt(unique_marker(), email)
|
||||
|
||||
_assert_eventually_masked(
|
||||
lambda: client.messages(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128),
|
||||
_messages_text,
|
||||
email=email,
|
||||
)
|
||||
|
||||
|
||||
#: Room for the model's reasoning tokens plus the three-line answer; a lower cap
|
||||
#: truncates the response before the address it is supposed to mask.
|
||||
_POST_CALL_MAX_TOKENS = 512
|
||||
|
||||
#: The post_call scenario masks these two entities and nothing else. Left
|
||||
#: unscoped, Presidio's broader recognizers claim the local part too (a random
|
||||
#: marker reads as an NRP), which would erase the very token that tells output
|
||||
#: masking apart from input masking.
|
||||
_POST_CALL_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"}
|
||||
|
||||
|
||||
def _post_call_prompt(marker: str, local_part: str) -> str:
|
||||
"""Ask for the local part and the full address in one answer. Presidio masks
|
||||
an EMAIL_ADDRESS entity and a bare local part is not one, so the two land
|
||||
differently in the same response and pin the hook point behaviorally."""
|
||||
return (
|
||||
f"{marker} My email address is {local_part}@example.com and my phone number is {FAKE_PHONE}. "
|
||||
"Reply with exactly three lines and nothing else. "
|
||||
"Line 1: the part of the email address before the @ sign. "
|
||||
"Line 2: the full email address. "
|
||||
"Line 3: the phone number."
|
||||
)
|
||||
|
||||
|
||||
class TestPresidioPostCallMasking:
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.post_call.masks",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_post_call_masks_pii_in_model_output(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
"""A guardrail scoped to the output must anonymize the PII the model
|
||||
repeats back, so a caller (or a downstream log of the response) never
|
||||
receives it, while the request itself reaches the model untouched.
|
||||
|
||||
Both facts are asserted from one response: the local part comes back raw,
|
||||
which is only possible if the model saw the real address, and the address
|
||||
itself comes back as <EMAIL_ADDRESS> in the same answer.
|
||||
"""
|
||||
name = f"e2e-presidio-post-chat-{unique_marker()}"
|
||||
_register_presidio(
|
||||
client,
|
||||
resources,
|
||||
name=name,
|
||||
mode="post_call",
|
||||
filter_scope="output",
|
||||
entities=_POST_CALL_ENTITIES,
|
||||
)
|
||||
|
||||
local_part = f"e2euser{unique_marker()}"
|
||||
email = f"{local_part}@example.com"
|
||||
prompt = _post_call_prompt(unique_marker(), local_part)
|
||||
|
||||
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
|
||||
last = "<no successful response yet>"
|
||||
while True:
|
||||
result = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=_POST_CALL_MAX_TOKENS)
|
||||
match result:
|
||||
case Success(data=data):
|
||||
last = _first_content(data)
|
||||
if MASKED_EMAIL_TOKEN in last and email not in last:
|
||||
assert local_part in last, (
|
||||
"the model must have seen the RAW address (it is asked for the local "
|
||||
"part, which Presidio does not mask); the local part is missing, so "
|
||||
f"this response cannot tell post_call masking from pre_call: {last[:300]!r}"
|
||||
)
|
||||
assert MASKED_PHONE_TOKEN in last and FAKE_PHONE not in last, (
|
||||
f"the phone number in the model's answer must be masked too, got: {last[:300]!r}"
|
||||
)
|
||||
return
|
||||
case _:
|
||||
last = f"<non-Success result: {result}>"
|
||||
if time.monotonic() >= deadline:
|
||||
pytest.fail(
|
||||
f"presidio post_call guardrail never masked the model's output within "
|
||||
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
|
||||
)
|
||||
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
def _assert_eventually_masks_generated_card(fetch: Callable[[], str | None]) -> None:
|
||||
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
|
||||
last: str = "<no successful response yet>"
|
||||
while True:
|
||||
content = fetch()
|
||||
if content is not None:
|
||||
last = content
|
||||
if _contains_card_number(content):
|
||||
pytest.fail(
|
||||
"the post_call output masking let a card number through: "
|
||||
f"{content[:300]!r}"
|
||||
)
|
||||
if MASKED_CREDIT_CARD_TOKEN in content:
|
||||
return
|
||||
if time.monotonic() >= deadline:
|
||||
pytest.fail(
|
||||
"presidio post_call output masking never masked the generated card within "
|
||||
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
|
||||
)
|
||||
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
class TestPresidioCreditCardOutputMasking:
|
||||
"""Proves the UI-default Presidio scope masks model-generated card output."""
|
||||
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.post_call.masks_generated_output",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_ui_default_scope_masks_a_card_number_the_model_generates_on_chat_completions(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
name = f"e2e-presidio-card-chat-{unique_marker()}"
|
||||
_register_presidio(
|
||||
client,
|
||||
resources,
|
||||
name=name,
|
||||
mode=["pre_call", "post_call"],
|
||||
filter_scope=None,
|
||||
entities={"CREDIT_CARD": "MASK"},
|
||||
)
|
||||
prompt: Final = _credit_card_prompt(unique_marker())
|
||||
|
||||
def fetch() -> str | None:
|
||||
result: Final = client.chat(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=512)
|
||||
match result:
|
||||
case Success(data=data):
|
||||
return _first_content(data)
|
||||
case _:
|
||||
return None
|
||||
|
||||
_assert_eventually_masks_generated_card(fetch)
|
||||
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.post_call.masks_generated_output",
|
||||
exercised_on=["chat_completions_stream"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_chat_completions(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
name = f"e2e-presidio-card-stream-{unique_marker()}"
|
||||
_register_presidio(
|
||||
client,
|
||||
resources,
|
||||
name=name,
|
||||
mode=["pre_call", "post_call"],
|
||||
filter_scope=None,
|
||||
entities={"CREDIT_CARD": "MASK"},
|
||||
)
|
||||
prompt: Final = _credit_card_prompt(unique_marker())
|
||||
|
||||
def fetch() -> str | None:
|
||||
result: Final = client.chat_stream_raw(
|
||||
scoped_key,
|
||||
MODEL,
|
||||
prompt,
|
||||
guardrails=[name],
|
||||
max_tokens=512,
|
||||
)
|
||||
if not result.ok or result.stream_error:
|
||||
return None
|
||||
return _stream_content(result)
|
||||
|
||||
_assert_eventually_masks_generated_card(fetch)
|
||||
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.post_call.masks_generated_output",
|
||||
exercised_on=["anthropic_messages_stream"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_ui_default_scope_masks_a_card_number_the_model_generates_on_streaming_anthropic_messages(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
name = f"e2e-presidio-card-messages-stream-{unique_marker()}"
|
||||
_register_presidio(
|
||||
client,
|
||||
resources,
|
||||
name=name,
|
||||
mode=["pre_call", "post_call"],
|
||||
filter_scope=None,
|
||||
entities={"CREDIT_CARD": "MASK"},
|
||||
)
|
||||
prompt: Final = _credit_card_prompt(unique_marker())
|
||||
|
||||
def fetch() -> str | None:
|
||||
result: Final = client.messages_stream_raw(
|
||||
scoped_key,
|
||||
MODEL,
|
||||
prompt,
|
||||
guardrails=[name],
|
||||
max_tokens=512,
|
||||
)
|
||||
if not result.ok or result.stream_error:
|
||||
return None
|
||||
return _anthropic_stream_content(result)
|
||||
|
||||
_assert_eventually_masks_generated_card(fetch)
|
||||
|
||||
|
||||
def _wire_text(outcome: StreamingResponse) -> str:
|
||||
return "\n".join(outcome.stream_events) if outcome.is_streaming else outcome.body
|
||||
|
||||
|
||||
def _poll_until_generated_card_masked(fetch: Callable[[], StreamingResponse]) -> StreamingResponse:
|
||||
"""The raw HTTP outcome once the output masker is in effect on the serving
|
||||
worker: whichever wire shape the endpoint speaks, a masked body carries the
|
||||
CREDIT_CARD placeholder and no Luhn-valid card run. A raw card is a worker
|
||||
that has not loaded the guardrail yet, so it is polled through like any
|
||||
other unmasked answer."""
|
||||
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
|
||||
last: str = "<no successful response yet>"
|
||||
while True:
|
||||
outcome = fetch()
|
||||
if outcome.ok and not outcome.stream_error:
|
||||
wire = _wire_text(outcome)
|
||||
last = wire
|
||||
if MASKED_CREDIT_CARD_TOKEN in wire and not _contains_card_number(wire):
|
||||
return outcome
|
||||
if time.monotonic() >= deadline:
|
||||
pytest.fail(
|
||||
"presidio post_call output masking never masked the generated card within "
|
||||
f"{GUARDRAIL_PROPAGATION_DEADLINE_SECONDS}s; last observation: {last[:300]!r}"
|
||||
)
|
||||
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
def _spend_log_response_text(client: GuardrailsClient, key: str, call_id: str) -> str:
|
||||
rows = client.proxy.poll_logs_for_key(
|
||||
key,
|
||||
predicate=lambda logged: any(row.litellm_call_id == call_id for row in logged),
|
||||
)
|
||||
row = next((row for row in rows if row.litellm_call_id == call_id), None)
|
||||
assert row is not None, f"no spend log row ever appeared for x-litellm-call-id {call_id}"
|
||||
return "\n".join(_stored_content(row.response))
|
||||
|
||||
|
||||
def _stored_content(node: JsonValue, key: str | None = None) -> Iterator[str]:
|
||||
match node:
|
||||
case str() if key in _CONTENT_KEYS:
|
||||
yield node
|
||||
case dict():
|
||||
for child_key, child in node.items():
|
||||
yield from _stored_content(child, child_key)
|
||||
case list():
|
||||
for item in node:
|
||||
yield from _stored_content(item, key)
|
||||
case _:
|
||||
return
|
||||
|
||||
|
||||
class TestPresidioSpendLogStoresMaskedOutput:
|
||||
"""The spend log stores the response the caller received, on every endpoint
|
||||
and both stream modes, when an output-only post_call Presidio guardrail masks
|
||||
a card number the model generated."""
|
||||
|
||||
_CELL: Final = "guardrail.presidio.post_call.spend_log_stores_masked_output"
|
||||
|
||||
def _assert_spend_log_is_masked(
|
||||
self,
|
||||
client: GuardrailsClient,
|
||||
resources: ResourceManager,
|
||||
key: str,
|
||||
*,
|
||||
name: str,
|
||||
fetch: Callable[[str, str], StreamingResponse],
|
||||
) -> None:
|
||||
_register_presidio(
|
||||
client,
|
||||
resources,
|
||||
name=name,
|
||||
mode="post_call",
|
||||
filter_scope="output",
|
||||
entities={"CREDIT_CARD": "MASK"},
|
||||
)
|
||||
prompt: Final = _credit_card_prompt(unique_marker())
|
||||
outcome = _poll_until_generated_card_masked(lambda: fetch(prompt, name))
|
||||
assert outcome.call_id, f"the served response must carry x-litellm-call-id: {dict(outcome.headers)}"
|
||||
|
||||
logged = _spend_log_response_text(client, key, outcome.call_id)
|
||||
assert not _contains_card_number(logged), (
|
||||
"the caller got the masked response but the spend log stored the raw model output: "
|
||||
f"{logged[:400]!r}"
|
||||
)
|
||||
assert MASKED_CREDIT_CARD_TOKEN in logged, (
|
||||
f"the spend log response carries neither the card nor the placeholder: {logged[:400]!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers(_CELL, exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_log_stores_masked_output_on_chat_completions(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
self._assert_spend_log_is_masked(
|
||||
client,
|
||||
resources,
|
||||
scoped_key,
|
||||
name=f"e2e-presidio-log-card-chat-{unique_marker()}",
|
||||
fetch=lambda prompt, guardrail: client.chat_raw(
|
||||
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.covers(_CELL, exercised_on=["chat_completions_stream"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_log_stores_masked_output_on_streaming_chat_completions(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
self._assert_spend_log_is_masked(
|
||||
client,
|
||||
resources,
|
||||
scoped_key,
|
||||
name=f"e2e-presidio-log-card-chat-stream-{unique_marker()}",
|
||||
fetch=lambda prompt, guardrail: client.chat_stream_raw(
|
||||
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.covers(_CELL, exercised_on=["messages"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_log_stores_masked_output_on_anthropic_messages(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
self._assert_spend_log_is_masked(
|
||||
client,
|
||||
resources,
|
||||
scoped_key,
|
||||
name=f"e2e-presidio-log-card-messages-{unique_marker()}",
|
||||
fetch=lambda prompt, guardrail: client.messages_raw(
|
||||
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.covers(_CELL, exercised_on=["anthropic_messages_stream"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.STREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_log_stores_masked_output_on_streaming_anthropic_messages(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
self._assert_spend_log_is_masked(
|
||||
client,
|
||||
resources,
|
||||
scoped_key,
|
||||
name=f"e2e-presidio-log-card-messages-stream-{unique_marker()}",
|
||||
fetch=lambda prompt, guardrail: client.messages_stream_raw(
|
||||
scoped_key, MODEL, prompt, guardrails=[guardrail], max_tokens=512
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.covers(_CELL, exercised_on=["responses"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_spend_log_stores_masked_output_on_responses(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
self._assert_spend_log_is_masked(
|
||||
client,
|
||||
resources,
|
||||
scoped_key,
|
||||
name=f"e2e-presidio-log-card-responses-{unique_marker()}",
|
||||
fetch=lambda prompt, guardrail: client.responses(scoped_key, MODEL, prompt, guardrails=[guardrail]),
|
||||
)
|
||||
|
||||
|
||||
_LOGGED_ENTITIES: dict[PiiEntity, PiiAction] = {"EMAIL_ADDRESS": "MASK", "PHONE_NUMBER": "MASK"}
|
||||
|
||||
_ENTITY_LIST_ADAPTER: Final = TypeAdapter(list[GuardrailEntityMatch])
|
||||
|
||||
|
||||
def _applied_guardrails(outcome: StreamingResponse) -> str:
|
||||
return outcome.headers.get("x-litellm-applied-guardrails", "")
|
||||
|
||||
|
||||
def _guardrail_records(row: SpendLogRow) -> tuple[GuardrailRunRecord, ...]:
|
||||
metadata = row.metadata
|
||||
if metadata is None:
|
||||
return ()
|
||||
return tuple(metadata.guardrail_information or ())
|
||||
|
||||
|
||||
def _poll_until_guardrail_applied(
|
||||
client: GuardrailsClient, key: str, guardrail_name: str, prompt: str
|
||||
) -> StreamingResponse:
|
||||
deadline = time.monotonic() + GUARDRAIL_PROPAGATION_DEADLINE_SECONDS
|
||||
last = client.chat_raw(key, MODEL, prompt, guardrails=[guardrail_name], max_tokens=128)
|
||||
while time.monotonic() < deadline:
|
||||
if last.ok and guardrail_name in _applied_guardrails(last):
|
||||
return last
|
||||
time.sleep(GUARDRAIL_PROPAGATION_POLL_INTERVAL_SECONDS)
|
||||
last = client.chat_raw(key, MODEL, prompt, guardrails=[guardrail_name], max_tokens=128)
|
||||
return last
|
||||
|
||||
|
||||
class TestPresidioSpendLogRecord:
|
||||
@pytest.mark.covers(
|
||||
"guardrail.presidio.pre_call.logs_masked_entities",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.GUARDRAILS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_masking_run_is_recorded_on_the_spend_log(
|
||||
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
name = f"e2e-presidio-log-{unique_marker()}"
|
||||
_register_presidio(client, resources, name=name, entities=_LOGGED_ENTITIES)
|
||||
|
||||
email = _fake_email()
|
||||
prompt = _pii_prompt(unique_marker(), email)
|
||||
|
||||
outcome = _poll_until_guardrail_applied(client, scoped_key, name, prompt)
|
||||
assert outcome.ok, f"the guarded call must be served, got {outcome.status_code}: {outcome.body[:400]}"
|
||||
assert name in _applied_guardrails(outcome), (
|
||||
"the response must carry x-litellm-applied-guardrails naming the guardrail; without it "
|
||||
f"the 200 only proves the guardrail never attached. Got {_applied_guardrails(outcome)!r}"
|
||||
)
|
||||
|
||||
request_id = ChatResponse.model_validate_json(outcome.body).id
|
||||
assert request_id, f"the served response must carry an id to look the spend log up by: {outcome.body[:400]}"
|
||||
|
||||
rows = client.proxy.poll_logs_for_request_id(
|
||||
request_id,
|
||||
predicate=lambda logged: bool(_guardrail_records(logged[0])),
|
||||
)
|
||||
assert rows, f"no spend log row ever appeared for request {request_id}"
|
||||
|
||||
records = _guardrail_records(rows[0])
|
||||
assert records, (
|
||||
f"the spend log for {request_id} carries no guardrail_information, so the dashboard's "
|
||||
"guardrail panel would render nothing for a request the guardrail demonstrably ran on"
|
||||
)
|
||||
|
||||
record = next(
|
||||
(entry for entry in records if entry.guardrail_name == name and entry.guardrail_mode == "pre_call"),
|
||||
None,
|
||||
)
|
||||
assert record is not None, (
|
||||
"guardrail_information carries no pre_call record for this guardrail, only "
|
||||
f"{[(entry.guardrail_name, entry.guardrail_mode) for entry in records]}"
|
||||
)
|
||||
assert record.guardrail_status == "success", (
|
||||
f"the recorded status must be success for a run that masked and served, got {record.guardrail_status!r}"
|
||||
)
|
||||
assert record.guardrail_provider == "presidio", (
|
||||
f"the record must attribute the run to presidio so the dashboard picks the right "
|
||||
f"renderer, got {record.guardrail_provider!r}"
|
||||
)
|
||||
|
||||
counts = record.masked_entity_count or {}
|
||||
assert _LOGGED_ENTITIES.keys() <= counts.keys(), (
|
||||
f"every entity the guardrail was configured to mask must appear in masked_entity_count, got {counts}"
|
||||
)
|
||||
assert all(counts[entity] >= 1 for entity in _LOGGED_ENTITIES), (
|
||||
f"each masked entity must be counted at least once, got {counts}"
|
||||
)
|
||||
|
||||
assert not isinstance(record.guardrail_response, str), (
|
||||
"This audit test requires general_settings.store_prompts_in_spend_logs: true in the proxy config "
|
||||
"or STORE_PROMPTS_IN_SPEND_LOGS=true in the proxy process environment before startup "
|
||||
"(not just the pytest environment). Default prompt redaction is valid proxy behavior, "
|
||||
f"but prevents entity-detail assertions; got guardrail_response={record.guardrail_response!r}"
|
||||
)
|
||||
entities = _ENTITY_LIST_ADAPTER.validate_python(record.guardrail_response)
|
||||
assert entities, (
|
||||
"guardrail_response must carry the detected entities; the dashboard's Detected Entities "
|
||||
"list and its per-entity scores are rendered from exactly this array"
|
||||
)
|
||||
assert {entity.entity_type for entity in entities} >= _LOGGED_ENTITIES.keys(), (
|
||||
f"the detected entities must cover what was masked, got "
|
||||
f"{sorted(entity.entity_type for entity in entities)}"
|
||||
)
|
||||
for entity in entities:
|
||||
assert 0.0 < entity.score <= 1.0, f"{entity.entity_type} carries an out-of-range score: {entity.score}"
|
||||
assert 0 <= entity.start < entity.end, (
|
||||
f"{entity.entity_type} carries a degenerate span: {entity.start}-{entity.end}"
|
||||
)
|
||||
|
|
@ -47,6 +47,7 @@ class RealtimeProvider:
|
|||
id: str
|
||||
alias: str
|
||||
litellm_params: LiteLLMParamsBody
|
||||
modalities: tuple[str, ...] = ("text",)
|
||||
|
||||
|
||||
PROVIDERS = (
|
||||
|
|
@ -75,6 +76,7 @@ PROVIDERS = (
|
|||
model="gemini/gemini-3.1-flash-live-preview",
|
||||
api_key="os.environ/GEMINI_API_KEY",
|
||||
),
|
||||
modalities=("audio",),
|
||||
),
|
||||
RealtimeProvider(
|
||||
"vertex_ai",
|
||||
|
|
@ -84,6 +86,7 @@ PROVIDERS = (
|
|||
vertex_location="us-central1",
|
||||
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
|
||||
),
|
||||
modalities=("audio",),
|
||||
),
|
||||
RealtimeProvider(
|
||||
"bedrock",
|
||||
|
|
|
|||
|
|
@ -124,7 +124,8 @@ def test_text_conversation(
|
|||
session.send(
|
||||
SessionUpdate(
|
||||
session=SessionConfig(
|
||||
instructions="You are a terse assistant. Reply in one short sentence."
|
||||
modalities=list(provider.modalities),
|
||||
instructions="You are a terse assistant. Reply in one short sentence.",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
@ -159,6 +160,7 @@ def test_tool_call_round_trip(
|
|||
session.send(
|
||||
SessionUpdate(
|
||||
session=SessionConfig(
|
||||
modalities=list(provider.modalities),
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="auto",
|
||||
instructions=(
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
|
|||
AZURE_OPENAI_API_VERSION: Final = "v1"
|
||||
|
||||
|
||||
def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody:
|
||||
def azure_openai_params(
|
||||
api_version: str = AZURE_OPENAI_API_VERSION, responses_api_version: str | None = None
|
||||
) -> LiteLLMParamsBody:
|
||||
return LiteLLMParamsBody(
|
||||
model=AZURE_OPENAI_BACKEND,
|
||||
api_base="os.environ/AZURE_API_BASE",
|
||||
api_key="os.environ/AZURE_API_KEY",
|
||||
api_version=api_version,
|
||||
responses_api_version=responses_api_version,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from anthropic.types import Message, MessageParam
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
from proxy_client import ProxyClient
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BEDROCK_MODEL: Final = "bedrock/us.moonshotai.kimi-k3"
|
||||
MODEL_NAME: Final = "claude-kimi-k3"
|
||||
MODEL_SWITCH_BETA: Final = (
|
||||
"claude-code-20250219,context-1m-2025-08-07,interleaved-thinking-2025-05-14,"
|
||||
"redact-thinking-2026-02-12,context-management-2025-06-27,"
|
||||
"prompt-caching-scope-2026-01-05,mid-conversation-system-2026-04-07"
|
||||
)
|
||||
|
||||
|
||||
def _register_model(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
|
||||
model_id: Final = proxy.create_model(
|
||||
MODEL_NAME,
|
||||
LiteLLMParamsBody(
|
||||
model=BEDROCK_MODEL,
|
||||
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
||||
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
aws_region_name="os.environ/AWS_REGION",
|
||||
drop_params=True,
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return MODEL_NAME, resources.key(models=[MODEL_NAME])
|
||||
|
||||
|
||||
class TestBedrockKimiK3Messages:
|
||||
@pytest.mark.covers("llm.messages.bedrock_converse.basic.nonstream.works_for_kimi_k3_max_tokens_1")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.BEDROCK,),
|
||||
models=(BEDROCK_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_model_switch_probe_max_tokens_1_returns_message(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model, key = _register_model(proxy, resources)
|
||||
client: Final = sdk.anthropic(key)
|
||||
message_param: Final[MessageParam] = {
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "Hi", "cache_control": {"type": "ephemeral"}}],
|
||||
}
|
||||
message: Final = client.messages.create(
|
||||
model=model,
|
||||
max_tokens=1,
|
||||
stream=False,
|
||||
system="You are Claude Code, an assistant for software development.",
|
||||
messages=[message_param],
|
||||
extra_headers={"anthropic-beta": MODEL_SWITCH_BETA},
|
||||
extra_query={"beta": "true"},
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
|
||||
assert isinstance(message, Message)
|
||||
assert message.role == "assistant"
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
"""Live provider checks for embeddings, the chat refusal, and chat fallback from an explicitly restricted model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, ModelInfoBody, ModelMode, ModelNewBody
|
||||
from openai import BadRequestError, OpenAI
|
||||
from proxy_client import ProxyClient
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
EMBEDDING_BACKEND: Final = "openai/text-embedding-3-small"
|
||||
CHAT_FALLBACK_BACKEND: Final = "openai/gpt-5.4-nano"
|
||||
EMBEDDINGS_ONLY: Final = ("/v1/embeddings",)
|
||||
PROMPT: Final = "Say this is a test!"
|
||||
|
||||
|
||||
def _subject(route: Route) -> Subject:
|
||||
return Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=route,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(EMBEDDING_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
|
||||
|
||||
def _register(
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
prefix: str,
|
||||
supported_endpoints: tuple[str, ...] | None,
|
||||
backend: str = EMBEDDING_BACKEND,
|
||||
mode: ModelMode = "embedding",
|
||||
) -> str:
|
||||
model: Final = f"{prefix}-{unique_marker()}"
|
||||
model_id: Final = proxy.register_model(
|
||||
ModelNewBody(
|
||||
model_name=model,
|
||||
litellm_params=LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY"),
|
||||
model_info=ModelInfoBody(
|
||||
enforce_supported_endpoints=supported_endpoints is not None,
|
||||
mode=mode,
|
||||
supported_endpoints=None if supported_endpoints is None else list(supported_endpoints),
|
||||
),
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model
|
||||
|
||||
|
||||
def _openai_at(sdk: SdkClients, api_prefix: str, key: str) -> OpenAI:
|
||||
return dataclasses.replace(sdk, base_url=f"{sdk.base_url}{api_prefix}").openai(key)
|
||||
|
||||
|
||||
class TestEmbeddingOnlySupportedEndpoints:
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.EMBEDDINGS))
|
||||
@pytest.mark.parametrize(
|
||||
("prefix", "supported_endpoints"),
|
||||
(
|
||||
pytest.param("e2e-embed-only", EMBEDDINGS_ONLY, id="embed-only"),
|
||||
pytest.param("e2e-embed-control", None, id="control"),
|
||||
),
|
||||
)
|
||||
def test_embeddings_return_a_vector(
|
||||
self,
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
sdk: SdkClients,
|
||||
prefix: str,
|
||||
supported_endpoints: tuple[str, ...] | None,
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, prefix, supported_endpoints)
|
||||
embeddings: Final = sdk.openai(resources.key()).embeddings.create(
|
||||
model=model, input=PROMPT, extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
assert embeddings.data, f"/embeddings returned no data for {model}: {embeddings!r}"
|
||||
assert any(component != 0.0 for component in embeddings.data[0].embedding), (
|
||||
f"/embeddings returned an empty or all-zero vector for {model}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(dataclasses.replace(_subject(Route.CHAT_COMPLETIONS), models=(EMBEDDING_BACKEND, CHAT_FALLBACK_BACKEND)))
|
||||
def test_chat_completions_with_a_fallback_are_answered_by_the_fallback(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
|
||||
fallback: Final = _register(proxy, resources, "e2e-chat-fallback", None, CHAT_FALLBACK_BACKEND, "chat")
|
||||
completion: Final = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": PROMPT}],
|
||||
extra_body={**NO_PROXY_CACHE, "fallbacks": [fallback]},
|
||||
)
|
||||
assert completion.choices and completion.choices[0].message.content, (
|
||||
f"the fallback {fallback} should answer a chat call to {model}, got {completion!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.openai.supported_endpoints_gate.nonstream.works")
|
||||
@meta(_subject(Route.CHAT_COMPLETIONS))
|
||||
def test_chat_completions_without_a_fallback_are_refused_by_the_gateway(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, "e2e-embed-only", EMBEDDINGS_ONLY)
|
||||
with pytest.raises(BadRequestError) as refused:
|
||||
_ = _openai_at(sdk, "/v1", resources.key()).chat.completions.create(
|
||||
model=model, messages=[{"role": "user", "content": PROMPT}], extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
assert (
|
||||
f"/v1/chat/completions is not supported by model={model}. Supported endpoints: /v1/embeddings"
|
||||
in refused.value.message
|
||||
), f"chat on {model} should be refused by the gateway before any provider call, got {refused.value!r}"
|
||||
187
tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py
Normal file
187
tests/e2e/llm_translation/test_gemini_tool_call_id_length_e2e.py
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
from models import LiteLLMParamsBody
|
||||
from openai.types.chat import (
|
||||
ChatCompletion,
|
||||
ChatCompletionAssistantMessageParam,
|
||||
ChatCompletionChunk,
|
||||
ChatCompletionMessageFunctionToolCallParam,
|
||||
ChatCompletionToolMessageParam,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionUserMessageParam,
|
||||
)
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
|
||||
from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall
|
||||
from passthrough_client import PassthroughClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND_MODEL: Final = "gemini/gemini-3.8-flash"
|
||||
GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY"
|
||||
# bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20
|
||||
AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
|
||||
WEATHER_TOOL: Final[ChatCompletionToolParam] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _subject(mode: Mode) -> Subject:
|
||||
return Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.GEMINI,),
|
||||
models=(BACKEND_MODEL,),
|
||||
mode=mode,
|
||||
capabilities=(Capability.FUNCTION_CALLING, Capability.TOOL_CHOICE),
|
||||
)
|
||||
|
||||
|
||||
def _register_model(client: PassthroughClient, resources: ResourceManager) -> str:
|
||||
model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}"
|
||||
model_id: Final = client.proxy.create_model(
|
||||
model_name,
|
||||
LiteLLMParamsBody(
|
||||
model=BACKEND_MODEL,
|
||||
api_key=GEMINI_API_KEY,
|
||||
thought_signature_in_tool_call_id=False,
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
return model_name
|
||||
|
||||
|
||||
def _user_message() -> ChatCompletionUserMessageParam:
|
||||
return {"role": "user", "content": f"Use get_weather for San Francisco and return its result. {unique_marker()}"}
|
||||
|
||||
|
||||
def _tool_calls(completion: ChatCompletion) -> tuple[ChatCompletionMessageFunctionToolCall, ...]:
|
||||
assert completion.choices, f"Gemini returned no choices: {completion.model_dump_json()[:400]}"
|
||||
calls: Final = tuple(
|
||||
call
|
||||
for call in completion.choices[0].message.tool_calls or ()
|
||||
if isinstance(call, ChatCompletionMessageFunctionToolCall)
|
||||
)
|
||||
assert calls, f"Gemini returned no tool calls for tool_choice='required': {completion.model_dump_json()[:400]}"
|
||||
return calls
|
||||
|
||||
|
||||
def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None:
|
||||
rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id))
|
||||
assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}"
|
||||
|
||||
|
||||
def _streamed_tool_calls(chunks: tuple[ChatCompletionChunk, ...]) -> Iterator[ChoiceDeltaToolCall]:
|
||||
for chunk in chunks:
|
||||
for choice in chunk.choices:
|
||||
yield from choice.delta.tool_calls or ()
|
||||
|
||||
|
||||
def _short_id_tool_call(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionMessageFunctionToolCallParam:
|
||||
return {
|
||||
"id": call.id,
|
||||
"type": "function",
|
||||
"function": {"name": call.function.name, "arguments": call.function.arguments},
|
||||
}
|
||||
|
||||
|
||||
def _tool_result(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionToolMessageParam:
|
||||
return {"role": "tool", "tool_call_id": call.id, "content": '{"temperature": 21, "condition": "sunny"}'}
|
||||
|
||||
|
||||
class TestGeminiToolCallIdLength:
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.gemini.tool_use.nonstream.works",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(_subject(Mode.NONSTREAM))
|
||||
def test_non_streaming_tool_call_id_fits_agentcore(
|
||||
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register_model(client, resources)
|
||||
completion: Final = sdk.openai(resources.key()).chat.completions.create(
|
||||
model=model,
|
||||
messages=[_user_message()],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="required",
|
||||
max_tokens=2048,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
|
||||
_assert_agentcore_accepts_ids(tuple(call.id for call in _tool_calls(completion)))
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.gemini.tool_use.stream.works",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(_subject(Mode.STREAM))
|
||||
def test_streaming_tool_call_id_fits_agentcore(
|
||||
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register_model(client, resources)
|
||||
chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple(
|
||||
sdk.openai(resources.key()).chat.completions.create(
|
||||
model=model,
|
||||
messages=[_user_message()],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="required",
|
||||
max_tokens=2048,
|
||||
stream=True,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
)
|
||||
ids: Final = tuple(call.id for call in _streamed_tool_calls(chunks) if call.id)
|
||||
assert ids, f"Gemini stream returned no tool call ids in {len(chunks)} chunks"
|
||||
|
||||
_assert_agentcore_accepts_ids(ids)
|
||||
|
||||
@pytest.mark.covers(
|
||||
"llm.chat_completions.gemini.multi_turn.nonstream.works",
|
||||
exercised_on=["chat_completions"],
|
||||
)
|
||||
@meta(_subject(Mode.NONSTREAM))
|
||||
def test_tool_call_round_trip_succeeds(
|
||||
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register_model(client, resources)
|
||||
openai_client: Final = sdk.openai(resources.key())
|
||||
question: Final = _user_message()
|
||||
first: Final = openai_client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[question],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="required",
|
||||
max_tokens=2048,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
calls: Final = _tool_calls(first)
|
||||
assistant: Final[ChatCompletionAssistantMessageParam] = {
|
||||
"role": "assistant",
|
||||
"tool_calls": [_short_id_tool_call(call) for call in calls],
|
||||
}
|
||||
|
||||
second: Final = openai_client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[question, assistant, *(_tool_result(call) for call in calls)],
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="none",
|
||||
max_tokens=2048,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
|
||||
assert second.choices, f"Gemini turn 2 returned no choices: {second.model_dump_json()[:400]}"
|
||||
answer: Final = second.choices[0].message.content
|
||||
assert answer and answer.strip(), f"Gemini turn 2 returned an empty answer: {second.model_dump_json()[:400]}"
|
||||
|
|
@ -15,48 +15,85 @@ pytestmark = pytest.mark.e2e
|
|||
|
||||
_BETA_CONFIG: Final = files("litellm").joinpath("anthropic_beta_headers_config.json")
|
||||
_ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5"
|
||||
_VERTEX_MODEL: Final = "vertex_ai/claude-sonnet-5-5"
|
||||
_VERTEX_LOCATION: Final = "global"
|
||||
_AZURE_AI_MODEL: Final = "azure_ai/claude-fable-5-1"
|
||||
_BEDROCK_MODEL_ID: Final = "us.anthropic.claude-fable-5-1"
|
||||
_BETA_CONFIG_FILE: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_BETA_MAPPING: Final = TypeAdapter(dict[str, str | None])
|
||||
_CODE_EXECUTION: Final[ToolUnionParam] = {"type": "code_execution_20250825", "name": "code_execution"}
|
||||
# A live Vertex probe on 2026-10-31: each Claude 4.5 and 4.6 model takes one of these and Claude 5.5 takes neither
|
||||
_VERTEX_PER_MODEL_BETAS: Final = frozenset({"computer-use-2025-01-24", "computer-use-2025-11-24"})
|
||||
_NO_PER_MODEL_BETAS: Final[frozenset[str]] = frozenset()
|
||||
|
||||
|
||||
def _mapped_beta_headers(provider: str) -> str:
|
||||
mapping: Final = _BETA_MAPPING.validate_python(_BETA_CONFIG_FILE.validate_json(_BETA_CONFIG.read_text())[provider])
|
||||
return ",".join(name for name, value in mapping.items() if value is not None)
|
||||
def _mapped_beta_headers(provider: str, per_model: frozenset[str]) -> str:
|
||||
config: Final = _BETA_CONFIG_FILE.validate_json(_BETA_CONFIG.read_text())
|
||||
mapping: Final = _BETA_MAPPING.validate_python(config[provider])
|
||||
header_only: Final = _BETA_MAPPING.validate_python(config.get(f"{provider}_header_only", {}))
|
||||
forwarded: Final = {name for name, value in {**mapping, **header_only}.items() if value is not None}
|
||||
assert per_model <= forwarded, per_model - forwarded
|
||||
return ",".join(sorted(forwarded - per_model))
|
||||
|
||||
|
||||
def _deployment(backend: str) -> LiteLLMParamsBody:
|
||||
if backend.startswith("anthropic/"):
|
||||
return LiteLLMParamsBody(model=backend, api_key="os.environ/ANTHROPIC_API_KEY")
|
||||
return LiteLLMParamsBody(
|
||||
model=backend,
|
||||
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
||||
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
match backend.split("/", 1)[0]:
|
||||
case "anthropic":
|
||||
return LiteLLMParamsBody(model=backend, api_key="os.environ/ANTHROPIC_API_KEY")
|
||||
case "vertex_ai":
|
||||
return LiteLLMParamsBody(
|
||||
model=backend,
|
||||
vertex_project="os.environ/VERTEXAI_PROJECT",
|
||||
vertex_location=_VERTEX_LOCATION,
|
||||
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
|
||||
)
|
||||
case "azure_ai":
|
||||
return LiteLLMParamsBody(
|
||||
model=backend,
|
||||
api_base="os.environ/AZURE_AI_API_BASE",
|
||||
api_key="os.environ/AZURE_AI_API_KEY",
|
||||
)
|
||||
case _:
|
||||
return LiteLLMParamsBody(
|
||||
model=backend,
|
||||
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
||||
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
|
||||
class TestAnthropicBetaHeaders:
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "beta_provider", "tools"),
|
||||
("backend", "beta_provider", "tools", "per_model"),
|
||||
[
|
||||
pytest.param(_ANTHROPIC_MODEL, "anthropic", [_CODE_EXECUTION], id="anthropic"),
|
||||
pytest.param(f"bedrock/invoke/{_BEDROCK_MODEL_ID}", "bedrock", [], id="bedrock-invoke"),
|
||||
pytest.param(f"bedrock/converse/{_BEDROCK_MODEL_ID}", "bedrock_converse", [], id="bedrock-converse"),
|
||||
pytest.param(_ANTHROPIC_MODEL, "anthropic", [_CODE_EXECUTION], _NO_PER_MODEL_BETAS, id="anthropic"),
|
||||
pytest.param(_VERTEX_MODEL, "vertex_ai", [], _VERTEX_PER_MODEL_BETAS, id="vertex-ai"),
|
||||
pytest.param(_AZURE_AI_MODEL, "azure_ai", [], _NO_PER_MODEL_BETAS, id="azure-ai"),
|
||||
pytest.param(
|
||||
f"bedrock/invoke/{_BEDROCK_MODEL_ID}", "bedrock", [], _NO_PER_MODEL_BETAS, id="bedrock-invoke"
|
||||
),
|
||||
pytest.param(
|
||||
f"bedrock/converse/{_BEDROCK_MODEL_ID}",
|
||||
"bedrock_converse",
|
||||
[],
|
||||
_NO_PER_MODEL_BETAS,
|
||||
id="bedrock-converse",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.covers(
|
||||
"llm.messages.anthropic.basic.nonstream.works",
|
||||
"llm.messages.bedrock_invoke.basic.nonstream.works",
|
||||
"llm.messages.bedrock_converse.basic.nonstream.works",
|
||||
"llm.messages.vertex.basic.nonstream.works",
|
||||
"llm.messages.azure_foundry.basic.nonstream.works",
|
||||
)
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.ANTHROPIC, Provider.BEDROCK),
|
||||
models=(_ANTHROPIC_MODEL, _BEDROCK_MODEL_ID),
|
||||
providers=(Provider.ANTHROPIC, Provider.BEDROCK, Provider.VERTEX_AI, Provider.AZURE_AI),
|
||||
models=(_ANTHROPIC_MODEL, _BEDROCK_MODEL_ID, _VERTEX_MODEL, _AZURE_AI_MODEL),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
|
|
@ -68,6 +105,7 @@ class TestAnthropicBetaHeaders:
|
|||
backend: str,
|
||||
beta_provider: str,
|
||||
tools: list[ToolUnionParam],
|
||||
per_model: frozenset[str],
|
||||
) -> None:
|
||||
model: Final = f"e2e-messages-beta-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(model, _deployment(backend))
|
||||
|
|
@ -78,7 +116,7 @@ class TestAnthropicBetaHeaders:
|
|||
max_tokens=64,
|
||||
messages=[{"role": "user", "content": "Say 'hello' and nothing else"}],
|
||||
tools=tools,
|
||||
extra_headers={"anthropic-beta": _mapped_beta_headers(beta_provider)},
|
||||
extra_headers={"anthropic-beta": _mapped_beta_headers(beta_provider, per_model)},
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Live e2e for model-specific request features: service_tier.
|
||||
"""Live e2e for model-specific request features: service_tier and prompt_cache_retention.
|
||||
|
||||
Each case asserts the feature took effect, not just a 200.
|
||||
|
||||
|
|
@ -11,26 +11,38 @@ avoided here because it is capacity-constrained and returns a transient 429 when
|
|||
flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so
|
||||
that cell is OpenAI-only by design.
|
||||
|
||||
Prompt caching lives in test_cache_control.py.
|
||||
prompt_cache_retention is judged on the body OpenAI receives, captured by a live
|
||||
provider edge, not on the reply: OpenAI echoes "24h" on gpt-5.5 even when the
|
||||
request never carried the param, so an echo would pass with the param dropped.
|
||||
|
||||
Prompt caching hits and misses live in test_cache_control.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker
|
||||
from e2e_http import unwrap
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, LiteLLMParamsBody
|
||||
from passthrough_client import PassthroughClient
|
||||
from provider_edge import LiveEdge, start_provider_edge
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
SERVICE_TIER = "priority"
|
||||
OPENAI_BACKEND: Final = "openai/gpt-5.5"
|
||||
OPENAI_RESPONSES_BRIDGE_BACKEND: Final = "openai/responses/gpt-5.5"
|
||||
PROMPT_CACHE_RETENTION: Final = "24h"
|
||||
|
||||
|
||||
class TestServiceTier:
|
||||
|
|
@ -74,3 +86,153 @@ class TestServiceTier:
|
|||
f"service_tier not honored: sent {SERVICE_TIER!r}, response reported "
|
||||
f"{response.service_tier!r} ({response})"
|
||||
)
|
||||
|
||||
|
||||
class _OutboundRetention(BaseModel, frozen=True):
|
||||
prompt_cache_retention: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OpenAIBodyCapture:
|
||||
"""The raw bodies sent to one OpenAI path, seen by a live edge in front of
|
||||
api.openai.com. Pass ``observe`` as the edge's ``observe_request``."""
|
||||
|
||||
path: str
|
||||
_bodies: queue.SimpleQueue[bytes] = field(default_factory=queue.SimpleQueue)
|
||||
|
||||
def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
|
||||
if body is not None and url.split("?")[0].endswith(self.path):
|
||||
self._bodies.put(body)
|
||||
|
||||
def retentions(self) -> tuple[str | None, ...]:
|
||||
bodies: Final = tuple(self._bodies.get_nowait() for _ in range(self._bodies.qsize()))
|
||||
return tuple(_OutboundRetention.model_validate_json(body).prompt_cache_retention for body in bodies)
|
||||
|
||||
|
||||
def _openai_edge_api_base(resources: ResourceManager, capture: _OpenAIBodyCapture, *, reachable_from_proxy: bool) -> str:
|
||||
"""The proxy may run in a container, so its edge binds where the proxy can
|
||||
reach it; an in-process SDK call reaches the edge on loopback."""
|
||||
edge: Final = (
|
||||
start_provider_edge(
|
||||
LiveEdge(observe_request=capture.observe),
|
||||
bind_host=PROVIDER_EDGE_BIND_HOST,
|
||||
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
|
||||
)
|
||||
if reachable_from_proxy
|
||||
else start_provider_edge(LiveEdge(observe_request=capture.observe))
|
||||
)
|
||||
resources.defer(edge.shutdown)
|
||||
return f"{edge.edge.api_base('openai')}/v1"
|
||||
|
||||
|
||||
def _retention_backends() -> tuple[object, ...]:
|
||||
"""gpt-5.5 straight to /v1/chat/completions, and the openai/responses/ bridge
|
||||
(chat completions in, Responses out) to /v1/responses."""
|
||||
return (
|
||||
pytest.param(
|
||||
OPENAI_BACKEND,
|
||||
"/chat/completions",
|
||||
marks=(
|
||||
pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention"),
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(OPENAI_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
),
|
||||
),
|
||||
id="chat_completions",
|
||||
),
|
||||
pytest.param(
|
||||
OPENAI_RESPONSES_BRIDGE_BACKEND,
|
||||
"/responses",
|
||||
marks=(
|
||||
pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention"),
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(OPENAI_RESPONSES_BRIDGE_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
),
|
||||
),
|
||||
id="responses_bridge",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TestPromptCacheRetention:
|
||||
@pytest.mark.provider_edge_host
|
||||
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
|
||||
def test_proxy_chat_completions_forwards_prompt_cache_retention_to_openai(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, backend: str, openai_path: str
|
||||
) -> None:
|
||||
capture: Final = _OpenAIBodyCapture(openai_path)
|
||||
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=True)
|
||||
model: Final = f"e2e-cache-retention-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model, LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY", api_base=api_base)
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
|
||||
unwrap(
|
||||
proxy.chat(
|
||||
resources.key(),
|
||||
ChatBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content="reply with one word")],
|
||||
prompt_cache_retention=PROMPT_CACHE_RETENTION,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
|
||||
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
|
||||
def test_sdk_completion_forwards_prompt_cache_retention_to_openai(
|
||||
self, resources: ResourceManager, backend: str, openai_path: str
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
capture: Final = _OpenAIBodyCapture(openai_path)
|
||||
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
|
||||
|
||||
litellm.completion(
|
||||
model=backend,
|
||||
messages=[{"role": "user", "content": "reply with one word"}],
|
||||
api_base=api_base,
|
||||
prompt_cache_retention=PROMPT_CACHE_RETENTION,
|
||||
)
|
||||
|
||||
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
|
||||
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
|
||||
def test_router_completion_forwards_prompt_cache_retention_to_openai(
|
||||
self, resources: ResourceManager, backend: str, openai_path: str
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
capture: Final = _OpenAIBodyCapture(openai_path)
|
||||
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[{"model_name": "cache-retention", "litellm_params": {"model": backend, "api_base": api_base}}]
|
||||
)
|
||||
|
||||
router.completion( # pyright: ignore[reportUnknownMemberType] # Router.completion forwards an untyped **kwargs to litellm.completion
|
||||
model="cache-retention",
|
||||
messages=[{"role": "user", "content": "reply with one word"}],
|
||||
prompt_cache_retention=PROMPT_CACHE_RETENTION,
|
||||
)
|
||||
|
||||
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
|
||||
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,205 @@
|
|||
"""Live e2e: Codex's client tool_search over /v1/responses on Claude.
|
||||
|
||||
Codex holds MCP tools back behind its client-executed `tool_search` tool when the model catalog says
|
||||
`supports_search_tool`. Over the completion bridge the proxy offers that search to Claude as a plain function,
|
||||
hands Claude's call back as a `tool_search_call` Codex runs itself, and loads the tools Codex returns in
|
||||
`tool_search_output` so Claude can call them on the next turn
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
from openai.types.responses import (
|
||||
FunctionToolParam,
|
||||
Response,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseInputParam,
|
||||
ResponseToolSearchCall,
|
||||
ToolParam,
|
||||
ToolSearchToolParam,
|
||||
)
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-5-5"
|
||||
DEFERRED_TOOL_COUNT: Final = 50
|
||||
CALENDAR_TOOL_NAME: Final = "mcp__codex_apps__calendar_create_event"
|
||||
|
||||
CODEX_TOOL_SEARCH: Final[ToolSearchToolParam] = {
|
||||
"type": "tool_search",
|
||||
"execution": "client",
|
||||
"description": (
|
||||
"# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next "
|
||||
"model call. Some of the tools may not have been provided to you upfront, and you should use this tool "
|
||||
"(`tool_search`) to search for the required tools."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Search query for deferred tools."},
|
||||
"limit": {"type": "number", "description": "Maximum number of tools to return. Defaults to 8."},
|
||||
},
|
||||
"required": ["query"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
}
|
||||
|
||||
SHELL_TOOL: Final[FunctionToolParam] = {
|
||||
"type": "function",
|
||||
"name": "shell",
|
||||
"description": "Runs a shell command.",
|
||||
"parameters": {"type": "object", "properties": {"cmd": {"type": "string"}}, "required": ["cmd"]},
|
||||
"strict": False,
|
||||
}
|
||||
|
||||
CALENDAR_TOOL: Final[FunctionToolParam] = {
|
||||
"type": "function",
|
||||
"name": CALENDAR_TOOL_NAME,
|
||||
"description": "Create a calendar event.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"title": {"type": "string"}},
|
||||
"required": ["title"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": False,
|
||||
"defer_loading": True,
|
||||
}
|
||||
|
||||
PROMPT: Final = "Create a calendar event titled Standup. Find the calendar tool with tool_search first, then call it."
|
||||
|
||||
|
||||
class _CalendarArguments(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
title: str
|
||||
|
||||
|
||||
def _filler_tool(index: int) -> FunctionToolParam:
|
||||
return {
|
||||
"type": "function",
|
||||
"name": f"mcp__codex_apps__records_{index:02d}",
|
||||
"description": f"List, inspect, update and archive records of family {index} in the connected workspace.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string", "description": "Title of the record to act on"},
|
||||
"notes": {"type": "string", "description": "Free-form notes and context for the operation"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}},
|
||||
"priority": {"type": "integer", "minimum": 0, "maximum": 5},
|
||||
},
|
||||
"required": ["title"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": False,
|
||||
"defer_loading": True,
|
||||
}
|
||||
|
||||
|
||||
DEFERRED_TOOLS: Final[tuple[FunctionToolParam, ...]] = (
|
||||
*(_filler_tool(index) for index in range(DEFERRED_TOOL_COUNT - 1)),
|
||||
CALENDAR_TOOL,
|
||||
)
|
||||
|
||||
|
||||
def _eager(tool: FunctionToolParam) -> ToolParam:
|
||||
return FunctionToolParam(
|
||||
type="function",
|
||||
name=tool["name"],
|
||||
description=tool.get("description"),
|
||||
parameters=tool.get("parameters"),
|
||||
strict=False,
|
||||
)
|
||||
|
||||
|
||||
def _input_tokens(response: Response) -> int:
|
||||
assert response.usage is not None, f"response carried no usage: {response.id}"
|
||||
return response.usage.input_tokens
|
||||
|
||||
|
||||
def _search_call(response: Response) -> ResponseToolSearchCall:
|
||||
calls: Final = tuple(item for item in response.output if isinstance(item, ResponseToolSearchCall))
|
||||
assert len(calls) == 1, f"expected one tool_search_call, got {response.output!r}"
|
||||
return calls[0]
|
||||
|
||||
|
||||
class TestResponsesCodexToolSearchOverAnthropic:
|
||||
@pytest.mark.covers("llm.responses.anthropic.tool_search.nonstream.works")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(ANTHROPIC_BACKEND,),
|
||||
capabilities=(Capability.TOOL_SEARCH, Capability.FUNCTION_CALLING),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_claude_finds_a_deferred_tool_through_codex_tool_search_and_calls_it(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = f"e2e-codex-tool-search-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model, LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY")
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
client: Final = sdk.openai(resources.key())
|
||||
searchable_tools: Final[list[ToolParam]] = [SHELL_TOOL, CODEX_TOOL_SEARCH]
|
||||
|
||||
eager: Final = client.responses.create(
|
||||
model=model,
|
||||
input=PROMPT,
|
||||
tools=[SHELL_TOOL, *(_eager(tool) for tool in DEFERRED_TOOLS)],
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
first: Final = client.responses.create(
|
||||
model=model, input=PROMPT, tools=searchable_tools, extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
|
||||
search: Final = _search_call(first)
|
||||
assert search.execution == "client", f"tool_search ran on the wrong side: {search!r}"
|
||||
assert search.call_id, f"tool_search_call had no call_id: {search!r}"
|
||||
assert not any(isinstance(item, ResponseFunctionToolCall) for item in first.output), first.output
|
||||
assert _input_tokens(first) * 4 < _input_tokens(eager), (
|
||||
f"deferred first turn used {_input_tokens(first)} input tokens against {_input_tokens(eager)} eager"
|
||||
)
|
||||
|
||||
follow_up: Final[ResponseInputParam] = [
|
||||
{"role": "user", "content": PROMPT},
|
||||
{
|
||||
"type": "tool_search_call",
|
||||
"call_id": search.call_id,
|
||||
"execution": "client",
|
||||
"arguments": search.arguments,
|
||||
},
|
||||
{
|
||||
"type": "tool_search_output",
|
||||
"call_id": search.call_id,
|
||||
"status": "completed",
|
||||
"execution": "client",
|
||||
"tools": [CALENDAR_TOOL],
|
||||
},
|
||||
]
|
||||
second: Final = client.responses.create(
|
||||
model=model, input=follow_up, tools=searchable_tools, extra_body=NO_PROXY_CACHE
|
||||
)
|
||||
|
||||
calendar_call: Final = next(
|
||||
(
|
||||
item
|
||||
for item in second.output
|
||||
if isinstance(item, ResponseFunctionToolCall) and item.name == CALENDAR_TOOL_NAME
|
||||
),
|
||||
None,
|
||||
)
|
||||
assert calendar_call is not None, f"Claude did not call the loaded tool: {second.output!r}"
|
||||
assert "standup" in _CalendarArguments.model_validate_json(calendar_call.arguments).title.lower()
|
||||
|
|
@ -40,7 +40,7 @@ from provider_edge import LiveEdge, start_provider_edge
|
|||
from provider_edge_bedrock import ConverseBodyCapture, bedrock_signer
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params
|
||||
from responses_helpers import AZURE_OPENAI_API_VERSION, AZURE_OPENAI_BACKEND, azure_openai_params
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
|
@ -183,6 +183,48 @@ class TestResponses:
|
|||
)
|
||||
assert response.output_text.strip(), f"/responses returned no output text: {response.output!r}"
|
||||
|
||||
@pytest.mark.covers("llm.responses.openai.basic.nonstream.works")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(OPENAI_RESPONSES_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_responses_forwards_moderation(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(proxy, resources, _openai_responses_params())
|
||||
client: Final = sdk.openai(resources.key())
|
||||
|
||||
response: Final = client.responses.create(
|
||||
model=model,
|
||||
input="hi",
|
||||
extra_body={**NO_PROXY_CACHE, "moderation": {"model": "omni-moderation-latest"}},
|
||||
)
|
||||
response_body: Final = TypeAdapter(dict[str, object]).validate_python(cast(object, response.model_dump()))
|
||||
moderation_value: Final = response_body.get("moderation")
|
||||
assert moderation_value is not None, f"response had no moderation results: {response_body!r}"
|
||||
moderation: Final = TypeAdapter(dict[str, object]).validate_python(moderation_value)
|
||||
moderation_input: Final = TypeAdapter(dict[str, object]).validate_python(moderation["input"])
|
||||
moderation_output: Final = TypeAdapter(dict[str, object]).validate_python(moderation["output"])
|
||||
assert moderation_input.get("type") == "moderation_result", (
|
||||
f"unexpected input moderation: {moderation_input!r}"
|
||||
)
|
||||
assert (
|
||||
moderation_output.get("type") == "moderation_result"
|
||||
), f"unexpected output moderation: {moderation_output!r}"
|
||||
|
||||
with pytest.raises(openai.BadRequestError) as exc_info:
|
||||
client.responses.create(
|
||||
model=model,
|
||||
input="hi",
|
||||
extra_body={**NO_PROXY_CACHE, "moderation": {"model": "does-not-exist"}},
|
||||
)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
||||
@pytest.mark.covers("llm.responses.openai.basic.stream.works")
|
||||
@meta(
|
||||
Subject(
|
||||
|
|
@ -748,6 +790,70 @@ class TestResponses:
|
|||
f"/responses over azure openai preview returned no output text: {response.output!r}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.AZURE,),
|
||||
models=(AZURE_OPENAI_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_responses_azure_openai_dated_ga_chat_api_version_serves_responses(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model: Final = _register(
|
||||
proxy,
|
||||
resources,
|
||||
azure_openai_params(api_version="2024-10-21"),
|
||||
prefix="e2e-responses-azure-dated-chat",
|
||||
)
|
||||
client: Final = sdk.openai(resources.key())
|
||||
|
||||
response: Final = client.responses.create(
|
||||
model=model,
|
||||
input="reply with one word",
|
||||
instructions=INSTRUCTIONS,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert response.output_text.strip(), (
|
||||
f"/responses over an azure deployment pinned to a dated GA chat api_version returned no output text: "
|
||||
f"{response.output!r}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.AZURE,),
|
||||
models=(AZURE_OPENAI_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_responses_azure_openai_responses_api_version_overrides_a_pre_responses_api_version(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
# Azure rejects /openai/responses below api-version 2025-03-01-preview (probed live 2026-10-15),
|
||||
# so this call only succeeds if the proxy sends responses_api_version instead of api_version.
|
||||
model: Final = _register(
|
||||
proxy,
|
||||
resources,
|
||||
azure_openai_params(api_version="2024-10-01-preview", responses_api_version=AZURE_OPENAI_API_VERSION),
|
||||
prefix="e2e-responses-azure-override",
|
||||
)
|
||||
client: Final = sdk.openai(resources.key())
|
||||
|
||||
response: Final = client.responses.create(
|
||||
model=model,
|
||||
input="reply with one word",
|
||||
instructions=INSTRUCTIONS,
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert response.output_text.strip(), (
|
||||
f"/responses over an azure deployment with responses_api_version set returned no output text: "
|
||||
f"{response.output!r}"
|
||||
)
|
||||
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
|
|
|
|||
|
|
@ -1,256 +0,0 @@
|
|||
"""Live e2e: Sail through the gateway, where LiteLLM turns ``service_tier`` into Sail's
|
||||
``metadata.completion_window`` and bills the price columns of the window it sent.
|
||||
|
||||
The deployment carries its own base, balanced and flex rates, each distinct, so a bill at
|
||||
the wrong tier cannot pass. They are registered on the deployment instead of read from the
|
||||
proxy's cost map, because a stack that loads the published map has no ``sail/`` rows until
|
||||
this provider ships. Requires SAIL_API_KEY on the proxy; no skip gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS, unique_marker
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, SpendLogRow
|
||||
from openai import OpenAI
|
||||
from proxy_client import ProxyClient
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND: Final = "sail/zai-org/GLM-5.3"
|
||||
PricedTier = Literal["base", "balanced", "flex"]
|
||||
PRICED_TIERS: Final[tuple[PricedTier, ...]] = ("base", "balanced", "flex")
|
||||
PROMPT: Final = "Reply with one word."
|
||||
MAX_TOKENS: Final = 512
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rates:
|
||||
input: float
|
||||
output: float
|
||||
cache_read: float
|
||||
|
||||
|
||||
RATES: Final[Mapping[PricedTier, _Rates]] = {
|
||||
"base": _Rates(input=3e-06, output=9e-06, cache_read=1e-06),
|
||||
"balanced": _Rates(input=2e-06, output=6e-06, cache_read=7e-07),
|
||||
"flex": _Rates(input=1e-06, output=3e-06, cache_read=4e-07),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Tokens:
|
||||
prompt: int
|
||||
cached: int
|
||||
completion: int
|
||||
|
||||
|
||||
def _approx_equal(actual: float, expected: float) -> bool:
|
||||
return abs(actual - expected) <= max(1e-12, abs(expected) * 1e-2)
|
||||
|
||||
|
||||
def _cost(rates: _Rates, tokens: _Tokens) -> float:
|
||||
return (
|
||||
(tokens.prompt - tokens.cached) * rates.input
|
||||
+ tokens.cached * rates.cache_read
|
||||
+ tokens.completion * rates.output
|
||||
)
|
||||
|
||||
|
||||
def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
|
||||
model: Final = f"e2e-sail-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(
|
||||
model=BACKEND,
|
||||
api_key="os.environ/SAIL_API_KEY",
|
||||
input_cost_per_token=RATES["base"].input,
|
||||
output_cost_per_token=RATES["base"].output,
|
||||
cache_read_input_token_cost=RATES["base"].cache_read,
|
||||
input_cost_per_token_balanced=RATES["balanced"].input,
|
||||
output_cost_per_token_balanced=RATES["balanced"].output,
|
||||
cache_read_input_token_cost_balanced=RATES["balanced"].cache_read,
|
||||
input_cost_per_token_flex=RATES["flex"].input,
|
||||
output_cost_per_token_flex=RATES["flex"].output,
|
||||
cache_read_input_token_cost_flex=RATES["flex"].cache_read,
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model, resources.key()
|
||||
|
||||
|
||||
def _openai(sdk: SdkClients, key: str) -> OpenAI:
|
||||
return sdk.openai(key).with_options(timeout=SLOW_PROVIDER_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
def _assert_billed_at(tier: PricedTier, tokens: _Tokens, header_cost: str | None) -> float:
|
||||
assert tokens.prompt > 0 and tokens.completion > 0, f"Sail reported no usage, so no cost is real: {tokens}"
|
||||
assert header_cost is not None, "x-litellm-response-cost header missing"
|
||||
costs: Final = {priced: _cost(rates, tokens) for priced, rates in RATES.items()}
|
||||
assert not any(_approx_equal(costs[other], costs[tier]) for other in PRICED_TIERS if other != tier), (
|
||||
f"{BACKEND} tier rates too close together to tell {tier} apart at {tokens}: {costs}"
|
||||
)
|
||||
assert _approx_equal(float(header_cost), costs[tier]), (
|
||||
f"header cost {header_cost} is not the {tier} price at {tokens}: expected {costs[tier]}, all tiers {costs}"
|
||||
)
|
||||
return float(header_cost)
|
||||
|
||||
|
||||
def _assert_spend_row_matches(proxy: ProxyClient, key: str, header_cost: float) -> None:
|
||||
def priced(rows: list[SpendLogRow]) -> bool:
|
||||
return any((row.spend or 0) > 0 for row in rows)
|
||||
|
||||
rows: Final = [row for row in proxy.poll_logs_for_key(key, predicate=priced) if (row.spend or 0) > 0]
|
||||
assert rows, f"no priced spend row landed for key {key}"
|
||||
assert rows[0].custom_llm_provider == "sail", f"spend row misattributed: {rows[0]}"
|
||||
assert rows[0].spend is not None and _approx_equal(rows[0].spend, header_cost), (
|
||||
f"logged spend {rows[0].spend} disagrees with the x-litellm-response-cost header {header_cost}"
|
||||
)
|
||||
|
||||
|
||||
class TestSailChatCompletions:
|
||||
@pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.cost_logged")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.SAIL,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "billed_tier"), [("balanced", "balanced"), ("auto", "base")]
|
||||
)
|
||||
def test_service_tier_bills_the_matching_completion_window(
|
||||
self,
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
sdk: SdkClients,
|
||||
service_tier: str,
|
||||
billed_tier: PricedTier,
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
raw: Final = _openai(sdk, key).chat.completions.with_raw_response.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": f"{PROMPT} {unique_marker()}"}],
|
||||
max_completion_tokens=MAX_TOKENS,
|
||||
extra_body={**NO_PROXY_CACHE, "service_tier": service_tier},
|
||||
)
|
||||
usage: Final = raw.parse().usage
|
||||
assert usage is not None, "chat response carries no usage"
|
||||
details: Final = usage.prompt_tokens_details
|
||||
tokens: Final = _Tokens(
|
||||
prompt=usage.prompt_tokens,
|
||||
cached=(details.cached_tokens or 0) if details else 0,
|
||||
completion=usage.completion_tokens,
|
||||
)
|
||||
|
||||
header_cost: Final = _assert_billed_at(
|
||||
billed_tier, tokens, response_header(raw.headers, "x-litellm-response-cost")
|
||||
)
|
||||
_assert_spend_row_matches(proxy, key, header_cost)
|
||||
|
||||
@pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.drops_unknown_tier_and_bills_asap")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.SAIL,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
@pytest.mark.parametrize("service_tier", ["bogus", 5])
|
||||
def test_unknown_service_tier_is_dropped_and_billed_asap(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, service_tier: str | int
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
raw: Final = _openai(sdk, key).chat.completions.with_raw_response.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": f"{PROMPT} {unique_marker()}"}],
|
||||
max_completion_tokens=MAX_TOKENS,
|
||||
extra_body={**NO_PROXY_CACHE, "service_tier": service_tier, "drop_params": True},
|
||||
)
|
||||
usage: Final = raw.parse().usage
|
||||
assert usage is not None, "chat response carries no usage"
|
||||
details: Final = usage.prompt_tokens_details
|
||||
tokens: Final = _Tokens(
|
||||
prompt=usage.prompt_tokens,
|
||||
cached=(details.cached_tokens or 0) if details else 0,
|
||||
completion=usage.completion_tokens,
|
||||
)
|
||||
header_cost: Final = _assert_billed_at("base", tokens, response_header(raw.headers, "x-litellm-response-cost"))
|
||||
_assert_spend_row_matches(proxy, key, header_cost)
|
||||
|
||||
|
||||
class TestSailResponses:
|
||||
@pytest.mark.covers("llm.responses.sail.service_tier.nonstream.cost_logged")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.SPEND_BUDGETS,
|
||||
route=Route.RESPONSES,
|
||||
providers=(Provider.SAIL,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_caller_completion_window_bills_its_rates(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
raw: Final = _openai(sdk, key).responses.with_raw_response.create(
|
||||
model=model,
|
||||
input=f"{PROMPT} {unique_marker()}",
|
||||
max_output_tokens=MAX_TOKENS,
|
||||
metadata={"completion_window": "balanced"},
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
usage: Final = raw.parse().usage
|
||||
assert usage is not None, "responses answer carries no usage"
|
||||
tokens: Final = _Tokens(
|
||||
prompt=usage.input_tokens,
|
||||
cached=usage.input_tokens_details.cached_tokens,
|
||||
completion=usage.output_tokens,
|
||||
)
|
||||
|
||||
header_cost: Final = _assert_billed_at(
|
||||
"balanced", tokens, response_header(raw.headers, "x-litellm-response-cost")
|
||||
)
|
||||
_assert_spend_row_matches(proxy, key, header_cost)
|
||||
|
||||
|
||||
class TestSailMessages:
|
||||
@pytest.mark.covers("llm.messages.sail.basic.nonstream.works")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.MESSAGES,
|
||||
providers=(Provider.SAIL,),
|
||||
models=(BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_plain_call_returns_a_message(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
message: Final = sdk.anthropic(key).messages.create(
|
||||
model=model,
|
||||
max_tokens=MAX_TOKENS,
|
||||
messages=[{"role": "user", "content": PROMPT}],
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert message.role == "assistant" and message.content, f"/v1/messages returned no content: {message}"
|
||||
assert message.usage.output_tokens > 0, f"/v1/messages reported no output usage: {message.usage}"
|
||||
|
|
@ -213,45 +213,6 @@ class LangfuseCreds:
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WeaveCreds:
|
||||
"""Weights & Biases Weave credentials for a key-scoped ``weave_otel`` callback.
|
||||
|
||||
The proxy still needs WANDB_API_KEY / WANDB_PROJECT_ID in its own environment:
|
||||
the weave_otel logger is constructed from those before the per-key vars are
|
||||
applied, so a key-scoped callback on a proxy without them never initializes.
|
||||
The per-key vars are what direct THIS key's spans at this project.
|
||||
"""
|
||||
|
||||
api_key: str
|
||||
project_id: str
|
||||
|
||||
def key_logging_metadata(self) -> KeyMetadata:
|
||||
return KeyMetadata(
|
||||
logging=[
|
||||
KeyLoggingCallback(
|
||||
callback_name="weave_otel",
|
||||
callback_type="success_and_failure",
|
||||
callback_vars=KeyLoggingCallbackVars(
|
||||
wandb_api_key=self.api_key,
|
||||
weave_project_id=self.project_id,
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def load_weave_creds() -> WeaveCreds:
|
||||
api_key = os.getenv("WANDB_API_KEY")
|
||||
project_id = (os.getenv("WEAVE_PROJECT_ID") or os.getenv("WANDB_PROJECT_ID") or "").strip()
|
||||
if not (api_key and project_id):
|
||||
pytest.fail(
|
||||
"Weave e2e requires WANDB_API_KEY and WEAVE_PROJECT_ID (or WANDB_PROJECT_ID, "
|
||||
"format <entity>/<project>); missing credentials is a hard failure, not a skip"
|
||||
)
|
||||
return WeaveCreds(api_key=api_key, project_id=project_id)
|
||||
|
||||
|
||||
def load_langfuse_creds() -> LangfuseCreds:
|
||||
public_key = os.getenv("LANGFUSE_PUBLIC_KEY")
|
||||
secret_key = os.getenv("LANGFUSE_SECRET_KEY")
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ def _assert_otel_destination_configured(client: LoggingClient) -> None:
|
|||
details = _ReadinessDetails.model_validate_json(readiness_details_body(client))
|
||||
assert OTEL_V2_LOGGER_NAME in details.success_callbacks, (
|
||||
f"the proxy must report the {OTEL_V2_LOGGER_NAME} callback active "
|
||||
f"(LITELLM_OTEL_V2 + arize_phoenix preset in the compose config); got: {details.success_callbacks}"
|
||||
f"(LITELLM_OTEL_V2=true + the otel callback in the stack config); got: {details.success_callbacks}"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,212 +0,0 @@
|
|||
"""Live e2e: key-scoped Weave (Weights & Biases) delivery, success and failure.
|
||||
|
||||
Covers the two `logging.niche_integrations.*.logs_spend` cells with a real member
|
||||
of that cohort. A key carrying a `weave_otel` callback in its logging metadata
|
||||
must deliver its calls to the real Weave project, and each call must arrive
|
||||
exactly once, carrying the same cost the response header reported:
|
||||
|
||||
- success: one `litellm_request` call, OTEL status OK, `llm.response.cost` equal
|
||||
to `x-litellm-response-cost`, and non-zero tokens
|
||||
- failure: a provider-rejected call arrives too, as one call with OTEL status
|
||||
ERROR naming the provider exception, and with no cost - a failed call that
|
||||
silently never reaches the destination is an invisible outage, and a billed
|
||||
one is worse
|
||||
|
||||
Both halves assert the recorded state (the key's callback registration answers
|
||||
success and the destination holds the call) and the enforced behavior (the
|
||||
delivered payload's status and cost). Delivery is read back through Weave's own
|
||||
query API; nothing is mocked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
|
||||
from e2e_http import StreamingResponse
|
||||
from e2e_metadata import Domain, Mode, Provider, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from logging_client import (
|
||||
INVALID_UPSTREAM_API_KEY,
|
||||
LoggingClient,
|
||||
WeaveCreds,
|
||||
costs_agree,
|
||||
first_ok,
|
||||
load_weave_creds,
|
||||
)
|
||||
from models import LiteLLMParamsBody
|
||||
from weave_reader import WeaveCall, WeaveReader, build_weave_reader
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
UNREACHABLE_ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def weave_creds() -> WeaveCreds:
|
||||
return load_weave_creds()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def weave_reader() -> WeaveReader:
|
||||
return build_weave_reader()
|
||||
|
||||
|
||||
#: How far before the request the Weave read-back window opens, to absorb clock
|
||||
#: skew between this host and Weave. Without it a host running slightly fast
|
||||
#: would filter out its own call.
|
||||
_WINDOW_SKEW_SECONDS = 120.0
|
||||
|
||||
|
||||
def _window_start() -> float:
|
||||
return time.time() - _WINDOW_SKEW_SECONDS
|
||||
|
||||
|
||||
def _exactly_one(calls: tuple[WeaveCall, ...], *, marker: str, what: str) -> WeaveCall:
|
||||
assert calls, f"no Weave call for the {what} (marker {marker}) reached the project within the deadline"
|
||||
assert len(calls) == 1, (
|
||||
f"expected exactly ONE Weave call for the {what} (marker {marker}), got {len(calls)}: "
|
||||
f"{[call.id for call in calls]} - more than one call for one request is the "
|
||||
"duplicate-delivery bug"
|
||||
)
|
||||
return calls[0]
|
||||
|
||||
|
||||
WEAVE_STAGE_RED_REASON = (
|
||||
"stage red: product gap, key-scoped weave_otel spans are not delivered when the OTEL v2 callback is active"
|
||||
)
|
||||
|
||||
|
||||
class TestWeaveLogDelivery:
|
||||
@pytest.mark.skip(reason=WEAVE_STAGE_RED_REASON)
|
||||
@pytest.mark.covers("logging.niche_integrations.success.logs_spend", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.OBSERVABILITY,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_chat_completions_delivers_one_call_with_spend(
|
||||
self,
|
||||
client: LoggingClient,
|
||||
weave_creds: WeaveCreds,
|
||||
weave_reader: WeaveReader,
|
||||
resources: ResourceManager,
|
||||
) -> None:
|
||||
alias = f"weave-key-{unique_marker()}"
|
||||
key = client.key_with_alias(
|
||||
alias,
|
||||
models=[CHEAP_ANTHROPIC_MODEL],
|
||||
metadata=weave_creds.key_logging_metadata(),
|
||||
)
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
marker = unique_marker()
|
||||
since = _window_start()
|
||||
outcome = first_ok(
|
||||
client,
|
||||
lambda: client.chat_raw(key, CHEAP_ANTHROPIC_MODEL, f"reply with one word {marker}", max_tokens=64),
|
||||
)
|
||||
assert outcome.response_cost is not None and outcome.response_cost > 0, (
|
||||
f"the response must report x-litellm-response-cost, got {outcome.response_cost!r}"
|
||||
)
|
||||
|
||||
call = _exactly_one(
|
||||
weave_reader.poll_calls_matching(marker, since=since), marker=marker, what="successful call"
|
||||
)
|
||||
|
||||
assert call.status_code == "OK", f"a successful call must land at OK span status, got {call.status_code!r}"
|
||||
cost = call.response_cost
|
||||
assert cost is not None and costs_agree(outcome.response_cost, cost), (
|
||||
f"the Weave call's llm.response.cost {cost!r} must agree with the header cost "
|
||||
f"{outcome.response_cost} - a delivered span with the wrong cost is a silent "
|
||||
"billing-attribution bug"
|
||||
)
|
||||
assert call.total_tokens is not None and call.total_tokens > 0, (
|
||||
f"the delivered call must carry token usage, got {call.total_tokens!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.skip(reason=WEAVE_STAGE_RED_REASON)
|
||||
@pytest.mark.covers("logging.niche_integrations.failure.logs_spend", exercised_on=["chat_completions"])
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.OBSERVABILITY,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(UNREACHABLE_ANTHROPIC_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_failed_chat_completions_delivers_one_error_call(
|
||||
self,
|
||||
client: LoggingClient,
|
||||
weave_creds: WeaveCreds,
|
||||
weave_reader: WeaveReader,
|
||||
resources: ResourceManager,
|
||||
) -> None:
|
||||
"""A deployment with an invalid upstream key passes proxy auth and fails
|
||||
at the provider, so exactly one provider failure exists for it. Proxy-side
|
||||
401s during key propagation never reach the provider and ship no payload,
|
||||
which is what the retry loop below relies on."""
|
||||
model_name = f"weave-err-{unique_marker()}"
|
||||
model_id = client.create_model(
|
||||
model_name,
|
||||
LiteLLMParamsBody(model=UNREACHABLE_ANTHROPIC_BACKEND, api_key=INVALID_UPSTREAM_API_KEY),
|
||||
)
|
||||
resources.defer(lambda: client.delete_model(model_id))
|
||||
key = client.key_with_alias(
|
||||
f"weave-err-key-{unique_marker()}",
|
||||
models=[model_name],
|
||||
metadata=weave_creds.key_logging_metadata(),
|
||||
)
|
||||
resources.defer(lambda: client.delete_key(key))
|
||||
|
||||
marker = unique_marker()
|
||||
since = _window_start()
|
||||
outcome = _provoke_provider_failure(client, key, model_name, marker)
|
||||
|
||||
call = _exactly_one(weave_reader.poll_calls_matching(marker, since=since), marker=marker, what="failed call")
|
||||
|
||||
assert call.status_code == "ERROR", (
|
||||
f"a failed call must land at ERROR span status, got {call.status_code!r} - "
|
||||
"Weave's own summary.weave.status reads success either way, which is exactly "
|
||||
"why the span status is what this asserts on"
|
||||
)
|
||||
error = call.error
|
||||
assert error is not None and error.message is not None and "AnthropicException" in error.message, (
|
||||
f"the delivered call must carry the provider error, got {error!r}"
|
||||
)
|
||||
assert not call.response_cost, f"a failed call must not be billed, got llm.response.cost={call.response_cost!r}"
|
||||
assert outcome.status_code == 401, (
|
||||
f"an upstream auth failure must map to 401, got {outcome.status_code}: {outcome.body[:200]}"
|
||||
)
|
||||
|
||||
|
||||
def _provoke_provider_failure(client: LoggingClient, key: str, model_name: str, marker: str) -> StreamingResponse:
|
||||
"""Send until the provider (not the proxy) is the one rejecting the call.
|
||||
|
||||
A network failure between the test and the proxy is NOT retried: the request
|
||||
may have been served, and a retry would double-log the failure payload and
|
||||
falsely trip the exactly-one assertion.
|
||||
"""
|
||||
deadline = time.monotonic() + client.proxy.poll_timeout
|
||||
while True:
|
||||
outcome = client.chat_raw(key, model_name, f"trigger an upstream auth failure {marker}", max_tokens=16)
|
||||
assert not outcome.ok, "the call must fail; the deployment's upstream key is invalid"
|
||||
assert outcome.status_code != -1, (
|
||||
"network failure between the test and the proxy while provoking the provider failure; "
|
||||
"retrying now could double-log the failure payload and falsely trip the exactly-one "
|
||||
f"assertion - fix the rig connectivity first: {outcome.body[:200]}"
|
||||
)
|
||||
if "AnthropicException" in outcome.body or time.monotonic() >= deadline:
|
||||
break
|
||||
time.sleep(client.proxy.poll_interval)
|
||||
assert "AnthropicException" in outcome.body, (
|
||||
"never saw the upstream provider failure before the deadline; the key may still be "
|
||||
f"propagating - last outcome {outcome.status_code}: {outcome.body[:200]}"
|
||||
)
|
||||
return outcome
|
||||
|
|
@ -1,285 +0,0 @@
|
|||
"""Read-back for the Weave (Weights & Biases) logging tests against the real
|
||||
Weave project.
|
||||
|
||||
The proxy ships OTEL spans to https://trace.wandb.ai/otel/v1/traces with the
|
||||
``weave_otel`` callback, and the tests read the ingested calls back through
|
||||
Weave's own query API (``POST /calls/stream_query``), which answers JSON Lines:
|
||||
one JSON object per call, so the body is parsed line by line rather than as one
|
||||
document.
|
||||
|
||||
The project is shared with other traffic, so the read never relies on the target
|
||||
being among the newest N calls: the query is scoped server-side to the
|
||||
``litellm_request`` op and to calls that started after the test's own request,
|
||||
and pages with ``offset`` until the window is exhausted.
|
||||
|
||||
Weave's own ``summary.weave.status`` is a rollup that reads "success" even for a
|
||||
span the exporter marked failed, so status comes from the OTEL span itself
|
||||
(``attributes.otel_span.status.code``), and the shipped cost from
|
||||
``attributes.otel_span.attributes.llm.response.cost`` - the StandardLogging
|
||||
``response_cost``, which is what makes this a spend assertion rather than a
|
||||
delivery ping.
|
||||
|
||||
Missing configuration is a hard failure, never a skip.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import count, takewhile
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT
|
||||
from e2e_http import URL, AuthHeaders, send
|
||||
from e2e_metadata import step
|
||||
|
||||
_WEAVE_TRACE_API: Final = "https://trace.wandb.ai"
|
||||
|
||||
#: The op every litellm LLM call lands under. The proxy also exports a root
|
||||
#: server span ("Received Proxy Server Request") and management spans; only the
|
||||
#: LLM call carries the usage and cost this suite asserts on.
|
||||
LITELLM_REQUEST_OP: Final = "litellm_request"
|
||||
|
||||
#: How long to keep re-reading after the first matching call before trusting the
|
||||
#: exactly-one assertion. The OTEL batch exporter flushes on its own schedule, so
|
||||
#: a duplicate export can surface well after the first one, and a duplicate IS
|
||||
#: the bug being guarded against.
|
||||
WEAVE_SETTLE_SECONDS: Final = 45.0
|
||||
|
||||
#: Rows per page. The query is already scoped to this run's time window, so this
|
||||
#: only bounds one round trip, not what the read can see.
|
||||
_PAGE_SIZE: Final = 500
|
||||
|
||||
|
||||
class _WeaveSortBy(BaseModel):
|
||||
field: str
|
||||
direction: str
|
||||
|
||||
|
||||
class _WeaveOpFilter(BaseModel):
|
||||
op_names: list[str]
|
||||
|
||||
|
||||
class _WeaveGetField(BaseModel):
|
||||
get_field: str = Field(serialization_alias="$getField")
|
||||
|
||||
|
||||
class _WeaveLiteral(BaseModel):
|
||||
literal: float = Field(serialization_alias="$literal")
|
||||
|
||||
|
||||
class _WeaveGreaterThan(BaseModel):
|
||||
gt: tuple[_WeaveGetField, _WeaveLiteral] = Field(serialization_alias="$gt")
|
||||
|
||||
|
||||
class _WeaveQuery(BaseModel):
|
||||
expr: _WeaveGreaterThan = Field(serialization_alias="$expr")
|
||||
|
||||
|
||||
class _WeaveQueryBody(BaseModel):
|
||||
project_id: str
|
||||
filter: _WeaveOpFilter
|
||||
query: _WeaveQuery
|
||||
limit: int = _PAGE_SIZE
|
||||
offset: int = 0
|
||||
sort_by: list[_WeaveSortBy] = [_WeaveSortBy(field="started_at", direction="asc")]
|
||||
|
||||
|
||||
class _OtelStatus(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
code: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
class _OtelError(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
code: str | None = None
|
||||
type: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
class _LlmResponse(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
cost: float | None = None
|
||||
|
||||
|
||||
class _LlmAttributes(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
response: _LlmResponse | None = None
|
||||
|
||||
|
||||
class _OtelSpanAttributes(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
llm: _LlmAttributes | None = None
|
||||
error: _OtelError | None = None
|
||||
|
||||
|
||||
class _OtelSpan(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
name: str | None = None
|
||||
status: _OtelStatus | None = None
|
||||
attributes: _OtelSpanAttributes | None = None
|
||||
|
||||
|
||||
class _CallAttributes(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
otel_span: _OtelSpan | None = None
|
||||
|
||||
|
||||
class _Usage(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
total_tokens: int | None = None
|
||||
|
||||
|
||||
class _WeaveSummary(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
usage: dict[str, _Usage] = {}
|
||||
|
||||
|
||||
class WeaveCall(BaseModel):
|
||||
"""One ingested Weave call, reduced to what the scenarios assert on."""
|
||||
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
id: str
|
||||
op_name: str
|
||||
started_at: str | None = None
|
||||
inputs: dict[str, object] = {}
|
||||
attributes: _CallAttributes | None = None
|
||||
summary: _WeaveSummary | None = Field(default=None)
|
||||
|
||||
@property
|
||||
def op(self) -> str:
|
||||
"""The bare op name out of ``weave:///<entity>/<project>/op/<op>:<digest>``."""
|
||||
return self.op_name.split("/op/")[-1].split(":")[0]
|
||||
|
||||
@property
|
||||
def status_code(self) -> str | None:
|
||||
"""The OTEL span status, not Weave's own rollup (which reads "success"
|
||||
even for a span the exporter marked ERROR)."""
|
||||
span = self.attributes.otel_span if self.attributes else None
|
||||
return span.status.code if span and span.status else None
|
||||
|
||||
@property
|
||||
def error(self) -> _OtelError | None:
|
||||
span = self.attributes.otel_span if self.attributes else None
|
||||
return span.attributes.error if span and span.attributes else None
|
||||
|
||||
@property
|
||||
def response_cost(self) -> float | None:
|
||||
span = self.attributes.otel_span if self.attributes else None
|
||||
llm = span.attributes.llm if span and span.attributes else None
|
||||
return llm.response.cost if llm and llm.response else None
|
||||
|
||||
@property
|
||||
def total_tokens(self) -> int | None:
|
||||
"""Weave keys usage by model, so the total is summed across whatever
|
||||
models the call reported."""
|
||||
if not self.summary or not self.summary.usage:
|
||||
return None
|
||||
totals = [usage.total_tokens for usage in self.summary.usage.values() if usage.total_tokens is not None]
|
||||
return sum(totals) if totals else None
|
||||
|
||||
def mentions(self, needle: str) -> bool:
|
||||
return needle in json.dumps(self.inputs, default=str)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WeaveReader:
|
||||
project_id: str
|
||||
api_key: str = field(repr=False)
|
||||
|
||||
@property
|
||||
def _headers(self) -> AuthHeaders:
|
||||
"""Weave authenticates with HTTP Basic as the fixed user ``api``."""
|
||||
token = base64.b64encode(f"api:{self.api_key}".encode()).decode()
|
||||
return AuthHeaders(authorization=f"Basic {token}")
|
||||
|
||||
def _query_body(self, *, since: float, offset: int, op: str) -> _WeaveQueryBody:
|
||||
return _WeaveQueryBody(
|
||||
project_id=self.project_id,
|
||||
filter=_WeaveOpFilter(op_names=[f"weave:///{self.project_id}/op/{op}:*"]),
|
||||
query=_WeaveQuery(
|
||||
expr=_WeaveGreaterThan(gt=(_WeaveGetField(get_field="started_at"), _WeaveLiteral(literal=since)))
|
||||
),
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
def _page(self, *, since: float, offset: int, op: str) -> tuple[WeaveCall, ...]:
|
||||
outcome = send(
|
||||
URL(f"{_WEAVE_TRACE_API}/calls/stream_query"),
|
||||
headers=self._headers,
|
||||
json=self._query_body(since=since, offset=offset, op=op),
|
||||
)
|
||||
if not outcome.ok:
|
||||
pytest.fail(
|
||||
f"Weave calls query for project {self.project_id!r} failed "
|
||||
f"({outcome.status_code}): {outcome.body[:300]}"
|
||||
)
|
||||
return tuple(WeaveCall.model_validate_json(line) for line in outcome.body.splitlines() if line.strip())
|
||||
|
||||
@step("Read the Weave {op} calls carrying the marker {marker}")
|
||||
def calls_matching(self, marker: str, *, since: float, op: str = LITELLM_REQUEST_OP) -> tuple[WeaveCall, ...]:
|
||||
"""Every call under ``op`` started after ``since`` whose inputs carry
|
||||
``marker``, paging until the window is exhausted.
|
||||
|
||||
More than one is the duplicate-delivery bug, so this never collapses to a
|
||||
single call.
|
||||
"""
|
||||
pages = tuple(
|
||||
takewhile(
|
||||
bool,
|
||||
(self._page(since=since, offset=offset, op=op) for offset in count(0, _PAGE_SIZE)),
|
||||
)
|
||||
)
|
||||
return tuple(call for page in pages for call in page if call.mentions(marker))
|
||||
|
||||
@step("Wait for Weave to ingest a {op} call carrying the marker {marker}, then watch for duplicates")
|
||||
def poll_calls_matching(self, marker: str, *, since: float, op: str = LITELLM_REQUEST_OP) -> tuple[WeaveCall, ...]:
|
||||
"""Poll until the call is readable, then keep re-reading for
|
||||
WEAVE_SETTLE_SECONDS so a duplicate exported by a later batch flush
|
||||
cannot hide from the exactly-one assertion. A duplicate ends the settle
|
||||
early, because more waiting cannot clear it."""
|
||||
deadline = time.monotonic() + POLL_TIMEOUT
|
||||
while time.monotonic() < deadline:
|
||||
calls = self.calls_matching(marker, since=since, op=op)
|
||||
if calls:
|
||||
return self._settled(marker, since=since, op=op, first=calls)
|
||||
time.sleep(POLL_INTERVAL)
|
||||
return ()
|
||||
|
||||
def _settled(self, marker: str, *, since: float, op: str, first: tuple[WeaveCall, ...]) -> tuple[WeaveCall, ...]:
|
||||
"""A transiently empty re-read never downgrades what was already seen."""
|
||||
settle_deadline = time.monotonic() + WEAVE_SETTLE_SECONDS
|
||||
latest = first # rebind-ok: one settle window, re-read per poll interval
|
||||
while time.monotonic() < settle_deadline and len(latest) <= 1:
|
||||
time.sleep(POLL_INTERVAL)
|
||||
latest = self.calls_matching(marker, since=since, op=op) or latest
|
||||
return latest
|
||||
|
||||
|
||||
def build_weave_reader() -> WeaveReader:
|
||||
project_id = (os.environ.get("WEAVE_PROJECT_ID") or os.environ.get("WANDB_PROJECT_ID") or "").strip()
|
||||
api_key = os.environ.get("WANDB_API_KEY", "").strip()
|
||||
if not project_id or not api_key:
|
||||
pytest.fail(
|
||||
"Weave e2e requires WANDB_API_KEY and WEAVE_PROJECT_ID (or WANDB_PROJECT_ID, "
|
||||
"format <entity>/<project>): the test reads the proxy's weave_otel delivery "
|
||||
"back from the real Weave project; missing credentials is a hard failure, not a skip"
|
||||
)
|
||||
return WeaveReader(project_id=project_id, api_key=api_key)
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
"""One-time helper to capture a logged-in Linear browser session for the
|
||||
real-Linear MCP e2e test.
|
||||
|
||||
The real-Linear test drives the genuine gateway-managed authorization_code
|
||||
dance against ``mcp.linear.app``. The only step that cannot be scripted is
|
||||
Linear's login (magic link / SSO), so a human authenticates once here and the
|
||||
resulting session (cookies + local storage) is persisted to disk. The e2e test
|
||||
then loads that session in a headless Playwright context and clicks Approve on
|
||||
Linear's consent screen every run, with no human and no login automation.
|
||||
|
||||
Run it with the e2e venv, log into Linear in the window that opens, then return
|
||||
to the terminal and press Enter:
|
||||
|
||||
LITELLM=~/litellm-mcpe2e
|
||||
"$LITELLM"/.venv/bin/python "$LITELLM"/tests/e2e/mcp/linear_session_capture.py
|
||||
|
||||
The session is written to ``E2E_LINEAR_STORAGE_STATE`` (default
|
||||
``~/.litellm-e2e/linear_storage_state.json``), outside the repo. It is a
|
||||
secret: never commit it. Re-run this whenever Linear expires the session.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
DEFAULT_STATE_PATH = Path.home() / ".litellm-e2e" / "linear_storage_state.json"
|
||||
|
||||
|
||||
def capture(state_path: Path) -> None:
|
||||
"""Open a headed browser at Linear, wait for the human to log in, then save
|
||||
the authenticated session to ``state_path``."""
|
||||
state_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=False)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
page.goto("https://linear.app/login", wait_until="domcontentloaded")
|
||||
print("\n" + "=" * 72)
|
||||
print("Log into Linear in the browser window that just opened.")
|
||||
print("If Linear emails you a magic link, paste the link into THIS window's")
|
||||
print("address bar (opening it in your default browser won't capture the")
|
||||
print("session). Google SSO works too as long as you complete it here.")
|
||||
print("When your Linear workspace has loaded, come back and press Enter.")
|
||||
print("=" * 72)
|
||||
input("Press Enter once you are logged in... ")
|
||||
page.goto("https://mcp.linear.app/", wait_until="domcontentloaded")
|
||||
context.storage_state(path=str(state_path))
|
||||
browser.close()
|
||||
print(f"\nSaved Linear session to {state_path}")
|
||||
print("Point the e2e test at it with:")
|
||||
print(f' export E2E_LINEAR_STORAGE_STATE="{state_path}"')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
capture(Path(os.environ.get("E2E_LINEAR_STORAGE_STATE", str(DEFAULT_STATE_PATH))))
|
||||
|
|
@ -1,442 +0,0 @@
|
|||
"""Client for the mcp chat-completion OAuth e2e suite.
|
||||
|
||||
Registers a gateway-managed OAuth (authorization_code) MCP server, seeds the
|
||||
per-user upstream token by driving the interactive authorize dance with the
|
||||
official mcp SDK's OAuthClientProvider (the browser leg is a headless Chromium
|
||||
primed with a human's saved Linear session), then exercises the server through
|
||||
/chat/completions, where the gateway lists and executes its tools with the
|
||||
stored per-user token.
|
||||
|
||||
Management routes (/v1/mcp/server CRUD, /chat/completions) go through the
|
||||
shared ProxyClient transport. The MCP protocol used to seed the token goes through
|
||||
the mcp SDK, the same library production MCP hosts run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
import httpx
|
||||
import httpx2
|
||||
import pytest
|
||||
from e2e_config import PROXY_BASE_URL, REQUEST_TIMEOUT
|
||||
from e2e_http import AuthHeaders, NoBody, unwrap
|
||||
from e2e_metadata import step
|
||||
from idp import Identity
|
||||
from mcp import ClientSession
|
||||
from mcp.client.auth import OAuthClientProvider
|
||||
from mcp.client.streamable_http import streamable_http_client
|
||||
from mcp.shared.auth import AuthorizationCodeResult, OAuthClientInformationFull, OAuthClientMetadata, OAuthToken
|
||||
from mcp.types import TextContent
|
||||
from models import (
|
||||
ChatBody,
|
||||
ChatResponse,
|
||||
McpOauthUserCredentialStatus,
|
||||
McpServerCreateBody,
|
||||
McpServerInfo,
|
||||
McpServerUserCredentialListResponse,
|
||||
McpServerUserCredentialRow,
|
||||
)
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from playwright.async_api import Route
|
||||
|
||||
# Where the "browser" lands at the end of the authorize dance. Nothing listens
|
||||
# here: the route interceptor short-circuits the final redirect and reads the
|
||||
# code/state off its query string, exactly like a desktop MCP host intercepting
|
||||
# its loopback redirect.
|
||||
OAUTH_CLIENT_REDIRECT_URI = "http://127.0.0.1:53682/e2e/callback"
|
||||
BROWSER_CONSENT_TIMEOUT = 60.0
|
||||
|
||||
|
||||
def _mcp_url(alias: str, base_url: str = PROXY_BASE_URL) -> str:
|
||||
return f"{base_url.rstrip('/')}/{alias}/mcp"
|
||||
|
||||
|
||||
class InMemoryTokenStorage:
|
||||
"""The mcp SDK's TokenStorage protocol, in memory for one dance: the
|
||||
DCR-registered client and the gateway tokens minted for it."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._tokens: OAuthToken | None = None
|
||||
self._client_info: OAuthClientInformationFull | None = None
|
||||
|
||||
async def get_tokens(self) -> OAuthToken | None:
|
||||
return self._tokens
|
||||
|
||||
async def set_tokens(self, tokens: OAuthToken) -> None:
|
||||
self._tokens = tokens
|
||||
|
||||
async def get_client_info(self) -> OAuthClientInformationFull | None:
|
||||
return self._client_info
|
||||
|
||||
async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:
|
||||
self._client_info = client_info
|
||||
|
||||
|
||||
async def _browser_follow_authorize(
|
||||
start_url: str,
|
||||
storage_state_path: str,
|
||||
identity: Identity | None = None,
|
||||
server_alias: str | None = None,
|
||||
allow_upstream_consent: bool = True,
|
||||
) -> tuple[str, str | None]:
|
||||
"""Play the browser's role for a real upstream whose authorize endpoint
|
||||
serves an interactive consent page (Linear). A headless Chromium primed
|
||||
with a human's saved Linear session opens the gateway authorize URL and
|
||||
clicks through Linear's consent screens (the mcp.linear.app Approve form,
|
||||
then the linear.app workspace-selection page), riding the rest of the chain
|
||||
(Linear -> gateway callback -> host redirect_uri). The final hop is
|
||||
intercepted and short-circuited, since nothing listens there, and its
|
||||
code/state are read off the query string."""
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
captured: dict[str, str] = {} # mutable-ok: hand-off from the request listener
|
||||
trail: list[str] = [] # mutable-ok: navigation diagnostics for a failed dance
|
||||
|
||||
def _note_request(request: object) -> None:
|
||||
url = getattr(request, "url", "")
|
||||
host = httpx.URL(url).host
|
||||
if not allow_upstream_consent and (host == "linear.app" or host.endswith(".linear.app")):
|
||||
captured["upstream_consent"] = "seen"
|
||||
if url.startswith(OAUTH_CLIENT_REDIRECT_URI) and "url" not in captured:
|
||||
captured["url"] = url
|
||||
|
||||
async def _swallow_redirect(route: Route) -> None:
|
||||
await route.fulfill(status=200, content_type="text/plain", body="ok")
|
||||
|
||||
async with async_playwright() as playwright:
|
||||
browser = await playwright.chromium.launch(headless=True)
|
||||
context = await browser.new_context(storage_state=storage_state_path)
|
||||
await context.route(re.compile(re.escape(OAUTH_CLIENT_REDIRECT_URI) + r".*"), _swallow_redirect)
|
||||
page = await context.new_page()
|
||||
context.on("request", _note_request)
|
||||
page.on("framenavigated", lambda frame: trail.append(frame.url.split("?", 1)[0]))
|
||||
await page.goto(start_url, wait_until="domcontentloaded")
|
||||
deadline = time.monotonic() + BROWSER_CONSENT_TIMEOUT
|
||||
while "url" not in captured and time.monotonic() < deadline:
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=8000)
|
||||
except Exception: # noqa: BLE001 - a busy consent page never idles; fall through and try to advance it
|
||||
pass
|
||||
if "upstream_consent" in captured or "url" in captured:
|
||||
break
|
||||
if await page.locator("#username").count() and identity is not None:
|
||||
await page.locator("#username").fill(identity.username)
|
||||
await page.locator("#password").fill(identity.password)
|
||||
await page.locator("#kc-login").click()
|
||||
continue
|
||||
if "/ui/connect" in page.url and server_alias is not None:
|
||||
card = page.locator("div.cursor-pointer").filter(has=page.get_by_text(server_alias, exact=True))
|
||||
if await card.count() != 1:
|
||||
await asyncio.sleep(0.5)
|
||||
continue
|
||||
connect = card.get_by_text("Connect", exact=True)
|
||||
if await connect.count():
|
||||
await connect.click()
|
||||
continue
|
||||
if not await card.locator("svg.text-success").count():
|
||||
await asyncio.sleep(0.5)
|
||||
continue
|
||||
finish = page.get_by_role("button", name="Finish connecting", exact=True)
|
||||
if await finish.count() and await finish.is_enabled():
|
||||
await finish.click()
|
||||
continue
|
||||
control = page.locator(
|
||||
'button[name="action"][value="approve"], button:has-text("Authorize"), '
|
||||
'button:has-text("Allow"), button:has-text("@"), a:has-text("@")'
|
||||
).first
|
||||
try:
|
||||
await control.click(timeout=5000)
|
||||
except Exception: # noqa: BLE001 - nothing to advance yet; loop and re-check
|
||||
await asyncio.sleep(0.5)
|
||||
final_url = page.url
|
||||
await browser.close()
|
||||
|
||||
# A redirect chain can finish inside goto/networkidle before the loop checks the page.
|
||||
assert "upstream_consent" not in captured, "cold reconnect required upstream consent"
|
||||
landing = captured.get("url")
|
||||
assert landing is not None, (
|
||||
f"consent flow never reached {OAUTH_CLIENT_REDIRECT_URI}; "
|
||||
f"final={final_url.split('?', 1)[0]!r}; trail={trail[-6:]}"
|
||||
)
|
||||
params = dict(parse_qsl(httpx.URL(landing).query.decode()))
|
||||
assert "code" in params, "client redirect_uri carried no authorization code"
|
||||
return params["code"], params.get("state")
|
||||
|
||||
|
||||
def _oauth_provider(
|
||||
url: str,
|
||||
storage: InMemoryTokenStorage,
|
||||
storage_state_path: str | None,
|
||||
identity: Identity | None = None,
|
||||
server_alias: str | None = None,
|
||||
allow_upstream_consent: bool = True,
|
||||
) -> OAuthClientProvider:
|
||||
"""The SDK's real OAuth machinery (RFC 9728/8414 discovery, RFC 7591 DCR,
|
||||
PKCE, token exchange) with the browser leg driven by Playwright against the
|
||||
upstream's consent screen."""
|
||||
code_holder: dict[str, str | None] = {} # mutable-ok: hand-off between the two SDK callbacks
|
||||
|
||||
async def _reject_redirect(_: str) -> None:
|
||||
raise AssertionError("gateway demanded a fresh upstream consent; stored per-user token was not reused")
|
||||
|
||||
async def _follow_redirect(authorize_url: str) -> None:
|
||||
assert storage_state_path is not None
|
||||
code, state = await _browser_follow_authorize(
|
||||
authorize_url, storage_state_path, identity, server_alias, allow_upstream_consent
|
||||
)
|
||||
code_holder["code"] = code
|
||||
code_holder["state"] = state
|
||||
|
||||
redirect_handler: Final = _reject_redirect if storage_state_path is None else _follow_redirect
|
||||
|
||||
async def callback_handler() -> AuthorizationCodeResult:
|
||||
code = code_holder.get("code")
|
||||
assert code is not None, "callback_handler ran before the authorize redirect completed"
|
||||
return AuthorizationCodeResult(code=code, state=code_holder.get("state"))
|
||||
|
||||
return OAuthClientProvider(
|
||||
server_url=url,
|
||||
client_metadata=OAuthClientMetadata.model_validate(
|
||||
{
|
||||
"redirect_uris": [OAUTH_CLIENT_REDIRECT_URI],
|
||||
"token_endpoint_auth_method": "none",
|
||||
"grant_types": ["authorization_code", "refresh_token"],
|
||||
"response_types": ["code"],
|
||||
"client_name": "e2e-mcp-host",
|
||||
}
|
||||
),
|
||||
storage=storage,
|
||||
redirect_handler=redirect_handler,
|
||||
callback_handler=callback_handler,
|
||||
)
|
||||
|
||||
|
||||
class _HeaderInjectingTransport(httpx2.AsyncBaseTransport):
|
||||
"""Adds the caller's LiteLLM key header to every outgoing SDK request
|
||||
(discovery, DCR, token exchange), so the gateway resolves which user to
|
||||
store the upstream token for from the key on the token exchange, exactly
|
||||
like a production MCP host configured with a LiteLLM key header."""
|
||||
|
||||
def __init__(self, inner: httpx2.AsyncBaseTransport, headers: dict[str, str], gateway_url: str) -> None:
|
||||
self._inner = inner
|
||||
self._headers = headers
|
||||
self._gateway_url = httpx2.URL(gateway_url)
|
||||
|
||||
@staticmethod
|
||||
def _port(url: httpx2.URL) -> int | None:
|
||||
if url.port is not None:
|
||||
return url.port
|
||||
return {"http": 80, "https": 443}.get(url.scheme)
|
||||
|
||||
async def handle_async_request(self, request: httpx2.Request) -> httpx2.Response:
|
||||
same_origin: Final = (
|
||||
request.url.scheme == self._gateway_url.scheme
|
||||
and request.url.host == self._gateway_url.host
|
||||
and self._port(request.url) == self._port(self._gateway_url)
|
||||
)
|
||||
if same_origin:
|
||||
for name, value in self._headers.items():
|
||||
if name not in request.headers:
|
||||
request.headers[name] = value
|
||||
else:
|
||||
for name, value in self._headers.items():
|
||||
if request.headers.get(name) == value:
|
||||
del request.headers[name]
|
||||
return await self._inner.handle_async_request(request)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._inner.aclose()
|
||||
|
||||
|
||||
def _oauth_http_client(
|
||||
headers: dict[str, str], auth: OAuthClientProvider, gateway_url: str = PROXY_BASE_URL
|
||||
) -> httpx2.AsyncClient:
|
||||
return httpx2.AsyncClient(
|
||||
auth=auth,
|
||||
timeout=httpx2.Timeout(REQUEST_TIMEOUT),
|
||||
follow_redirects=True,
|
||||
transport=_HeaderInjectingTransport(httpx2.AsyncHTTPTransport(), headers, gateway_url),
|
||||
)
|
||||
|
||||
|
||||
async def _seed_via_dance(
|
||||
url: str, headers: dict[str, str], storage: InMemoryTokenStorage, storage_state_path: str
|
||||
) -> tuple[str, ...]:
|
||||
async with _oauth_http_client(headers, _oauth_provider(url, storage, storage_state_path)) as http_client:
|
||||
async with streamable_http_client(url, http_client=http_client) as (read, write):
|
||||
async with ClientSession(read, write) as session:
|
||||
await session.initialize()
|
||||
listed = await session.list_tools()
|
||||
return tuple(sorted(tool.name for tool in listed.tools))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OauthToolRun:
|
||||
tools: tuple[str, ...]
|
||||
is_error: bool
|
||||
text: str
|
||||
|
||||
|
||||
async def _list_and_call(
|
||||
url: str,
|
||||
headers: dict[str, str],
|
||||
storage: InMemoryTokenStorage,
|
||||
storage_state_path: str | None,
|
||||
tool: str,
|
||||
arguments: dict[str, str],
|
||||
gateway_url: str = PROXY_BASE_URL,
|
||||
identity: Identity | None = None,
|
||||
server_alias: str | None = None,
|
||||
allow_upstream_consent: bool = True,
|
||||
) -> OauthToolRun:
|
||||
async with _oauth_http_client(
|
||||
headers,
|
||||
_oauth_provider(url, storage, storage_state_path, identity, server_alias, allow_upstream_consent),
|
||||
gateway_url,
|
||||
) as http_client:
|
||||
async with streamable_http_client(url, http_client=http_client) as (read, write):
|
||||
async with ClientSession(read, write) as session:
|
||||
await session.initialize()
|
||||
listed: Final = await session.list_tools()
|
||||
result: Final = await session.call_tool(tool, arguments)
|
||||
text: Final = "".join(content.text for content in result.content if isinstance(content, TextContent))
|
||||
return OauthToolRun(
|
||||
tools=tuple(sorted(tool_item.name for tool_item in listed.tools)),
|
||||
is_error=result.is_error,
|
||||
text=text,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChatMcpClient:
|
||||
proxy: ProxyClient
|
||||
|
||||
@step("Register the MCP server with the alias {body.alias}")
|
||||
def create_server(self, body: McpServerCreateBody) -> McpServerInfo:
|
||||
return unwrap(
|
||||
self.proxy.transport.post(
|
||||
"/v1/mcp/server",
|
||||
headers=self.proxy.transport.master,
|
||||
json=body,
|
||||
response_type=McpServerInfo,
|
||||
)
|
||||
)
|
||||
|
||||
@step("Read the MCP server back from /v1/mcp/server")
|
||||
def server_info(self, server_id: str) -> McpServerInfo:
|
||||
return unwrap(
|
||||
self.proxy.transport.get(
|
||||
f"/v1/mcp/server/{server_id}",
|
||||
headers=self.proxy.transport.master,
|
||||
params=NoBody(),
|
||||
response_type=McpServerInfo,
|
||||
)
|
||||
)
|
||||
|
||||
@step("Delete the MCP server")
|
||||
def delete_server(self, server_id: str) -> None:
|
||||
_ = self.proxy.transport.delete(
|
||||
f"/v1/mcp/server/{server_id}",
|
||||
headers=self.proxy.transport.master,
|
||||
json=NoBody(),
|
||||
response_type=NoBody,
|
||||
)
|
||||
|
||||
@step("Sign the key's user in to the MCP server {alias} through the OAuth consent flow")
|
||||
def seed_user_token(self, alias: str, key: str, storage_state_path: str) -> tuple[str, ...]:
|
||||
"""Drive the interactive authorize dance for `key`'s user so the gateway
|
||||
stores their upstream token, retried to the shared deadline since the
|
||||
just-created server and key propagate asynchronously. The LiteLLM key
|
||||
rides x-litellm-api-key so the gateway binds the token to that user.
|
||||
Returns the upstream tool names the dance listed, proof the token works."""
|
||||
headers = {"x-litellm-api-key": f"Bearer {key}"}
|
||||
storage = InMemoryTokenStorage()
|
||||
deadline = time.monotonic() + self.proxy.poll_timeout
|
||||
last_error: Exception | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
return asyncio.run(_seed_via_dance(_mcp_url(alias), headers, storage, storage_state_path))
|
||||
except Exception as exc: # noqa: BLE001 - retried to the deadline; the last error surfaces below
|
||||
last_error = exc
|
||||
time.sleep(self.proxy.poll_interval)
|
||||
pytest.fail(
|
||||
f"authorize dance for {alias!r} never completed within {self.proxy.poll_timeout}s; "
|
||||
f"last error: {last_error!r}"
|
||||
)
|
||||
|
||||
@step("List the tools on the MCP server {alias} and call {tool} over the MCP protocol")
|
||||
def list_and_call(
|
||||
self,
|
||||
alias: str,
|
||||
headers: dict[str, str],
|
||||
storage: InMemoryTokenStorage,
|
||||
storage_state_path: str | None,
|
||||
tool: str,
|
||||
arguments: dict[str, str],
|
||||
base_url: str = PROXY_BASE_URL,
|
||||
identity: Identity | None = None,
|
||||
allow_upstream_consent: bool = True,
|
||||
) -> OauthToolRun:
|
||||
return asyncio.run(
|
||||
_list_and_call(
|
||||
f"{base_url.rstrip('/')}/mcp" if identity is not None else _mcp_url(alias, base_url),
|
||||
headers,
|
||||
storage,
|
||||
storage_state_path,
|
||||
tool,
|
||||
arguments,
|
||||
base_url,
|
||||
identity,
|
||||
alias,
|
||||
allow_upstream_consent,
|
||||
)
|
||||
)
|
||||
|
||||
@step("List the users with a stored OAuth token for the MCP server")
|
||||
def server_user_credentials(self, server_id: str) -> tuple[McpServerUserCredentialRow, ...]:
|
||||
return unwrap(
|
||||
self.proxy.transport.get(
|
||||
f"/v1/mcp/server/{server_id}/user-credentials",
|
||||
headers=self.proxy.transport.master,
|
||||
params=NoBody(),
|
||||
response_type=McpServerUserCredentialListResponse,
|
||||
)
|
||||
).root
|
||||
|
||||
@step("Revoke the user's stored OAuth token for the MCP server")
|
||||
def revoke_user_token(self, server_id: str, headers: AuthHeaders) -> None:
|
||||
_ = unwrap(
|
||||
self.proxy.transport.delete(
|
||||
f"/v1/mcp/server/{server_id}/oauth-user-credential",
|
||||
headers=headers,
|
||||
json=NoBody(),
|
||||
response_type=McpOauthUserCredentialStatus,
|
||||
)
|
||||
)
|
||||
|
||||
@step("Send a /chat/completions request to {body.model} with an MCP server attached as a tool")
|
||||
def chat_with_mcp(self, headers: AuthHeaders, body: ChatBody) -> ChatResponse:
|
||||
"""POST /chat/completions carrying the LiteLLM key in `headers` (either
|
||||
ingress form) with an MCP server attached in `body.tools`. The gateway
|
||||
resolves the user from the key and lists/executes the server's tools
|
||||
with that user's stored upstream token."""
|
||||
return unwrap(
|
||||
self.proxy.transport.post(
|
||||
"/chat/completions",
|
||||
headers=headers,
|
||||
json=body,
|
||||
response_type=ChatResponse,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def build_chat_client(proxy: ProxyClient) -> ChatMcpClient:
|
||||
return ChatMcpClient(proxy=proxy)
|
||||
|
|
@ -1,197 +0,0 @@
|
|||
"""An owned, source-built OAuth gateway with cold restarts and credential observations.
|
||||
|
||||
Only this child process is restarted. Its database and SSO client survive while
|
||||
its process-local caches do not; Redis is deliberately absent from its config.
|
||||
The optional live edge measures headers without recording credentials or bodies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import psycopg
|
||||
from e2e_config import INHERITED_ENV_PREFIXES, available_port
|
||||
from e2e_http import NoBody
|
||||
from e2e_metadata import step
|
||||
from idp import Keycloak, stop_process_group
|
||||
from proxy_client import ProxyClient, build_proxy_client
|
||||
from psycopg.rows import class_row
|
||||
from pydantic import BaseModel, SecretStr, ValidationError
|
||||
|
||||
|
||||
class StoredOAuth(BaseModel):
|
||||
type: str
|
||||
access_token: SecretStr
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CredentialRow:
|
||||
credential_b64: str = field(repr=False)
|
||||
|
||||
|
||||
@step("Read the user's stored OAuth credential for the MCP server from the database and decrypt it")
|
||||
def stored_oauth(user_id: str, server_id: str) -> StoredOAuth:
|
||||
"""Read the encrypted credential because management APIs omit the plaintext token."""
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
|
||||
with psycopg.Connection[CredentialRow].connect(
|
||||
os.environ["DATABASE_URL"], row_factory=class_row(CredentialRow)
|
||||
) as conn:
|
||||
row: Final = conn.execute(
|
||||
'SELECT credential_b64 FROM "LiteLLM_MCPUserCredentials" WHERE user_id = %s AND server_id = %s',
|
||||
(user_id, server_id),
|
||||
).fetchone()
|
||||
assert row is not None, "canonical user/server has no persisted credential"
|
||||
plaintext: Final = decrypt_value_helper(
|
||||
row.credential_b64, "e2e_mcp_oauth", exception_type="debug", return_original_value=False
|
||||
)
|
||||
assert plaintext is not None, "persisted credential must decrypt with the gateway salt"
|
||||
assert plaintext != row.credential_b64, "persisted credential must be encrypted"
|
||||
try:
|
||||
credential: Final = StoredOAuth.model_validate_json(plaintext)
|
||||
except ValidationError:
|
||||
raise AssertionError("decrypted credential is not an OAuth payload") from None
|
||||
assert credential.type == "oauth2"
|
||||
assert bool(credential.access_token.get_secret_value()), "stored upstream token is empty"
|
||||
return credential
|
||||
|
||||
|
||||
class RpcMethod(BaseModel):
|
||||
method: str = ""
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class OAuthObservation:
|
||||
gateway_token: str = field(default="", repr=False)
|
||||
_seen: tuple[tuple[str, str, bool], ...] = field(default=(), init=False, repr=False)
|
||||
_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
|
||||
|
||||
def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
|
||||
if body is None or not url.endswith("/mcp"):
|
||||
return
|
||||
try:
|
||||
operation: Final = RpcMethod.model_validate_json(body).method
|
||||
except ValidationError:
|
||||
return
|
||||
if operation not in ("tools/list", "tools/call"):
|
||||
return
|
||||
received: Final = headers.get("authorization", "")
|
||||
gateway_leaked: Final = any(
|
||||
value in (self.gateway_token, f"Bearer {self.gateway_token}") for value in headers.values()
|
||||
)
|
||||
with self._lock:
|
||||
self._seen = (*self._seen, (operation, received, gateway_leaked))
|
||||
|
||||
def assert_forwarded(self, expected: StoredOAuth) -> None:
|
||||
with self._lock:
|
||||
snapshot: Final = self._seen
|
||||
self._seen = ()
|
||||
assert {item[0] for item in snapshot} == {"tools/list", "tools/call"}, "missing upstream observations"
|
||||
expected_header: Final = f"Bearer {expected.access_token.get_secret_value()}"
|
||||
assert all(item[1] == expected_header for item in snapshot), "upstream bearer did not match the stored token"
|
||||
assert all(not item[2] for item in snapshot), "gateway bearer leaked to the upstream"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class OAuthGateway:
|
||||
base_url: str
|
||||
proxy: ProxyClient
|
||||
_environment: Mapping[str, str] = field(repr=False)
|
||||
_command: tuple[str, ...] = field(repr=False)
|
||||
_log_path: Path
|
||||
_child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False)
|
||||
|
||||
@step("Start the separate LiteLLM proxy and wait for /health/liveliness")
|
||||
def start(self) -> None:
|
||||
with self._log_path.open("ab") as log:
|
||||
self._child = subprocess.Popen(
|
||||
self._command,
|
||||
env=self._environment,
|
||||
stdout=log,
|
||||
stderr=log,
|
||||
start_new_session=True,
|
||||
)
|
||||
deadline: Final = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
assert self._child.poll() is None, "owned OAuth gateway exited; inspect its private log"
|
||||
result = self.proxy.transport.probe("/health/liveliness", params=NoBody())
|
||||
if result.status_code == 200:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("owned OAuth gateway did not become ready")
|
||||
|
||||
@step("Stop the separate LiteLLM proxy")
|
||||
def stop(self) -> None:
|
||||
if self._child is not None:
|
||||
stop_process_group(self._child)
|
||||
assert self._child.poll() is not None, "old gateway process is still alive"
|
||||
|
||||
@step("Restart the separate LiteLLM proxy process so its in-memory caches start empty")
|
||||
def restart(self) -> None:
|
||||
assert self._child is not None
|
||||
previous: Final = self._child.pid
|
||||
self.stop()
|
||||
self.start()
|
||||
assert self._child.pid != previous, "gateway restart did not create a new process"
|
||||
|
||||
|
||||
@step("Start a separate LiteLLM proxy from source with JWT auth against Keycloak")
|
||||
def owned_gateway(idp: Keycloak, directory: Path, cleanup: ExitStack) -> OAuthGateway:
|
||||
for name in ("DATABASE_URL", "LITELLM_LICENSE", "LITELLM_SALT_KEY", "LITELLM_MASTER_KEY"):
|
||||
assert os.environ.get(name), f"{name} is required for the owned OAuth gateway"
|
||||
port: Final = available_port()
|
||||
base_url: Final = f"http://127.0.0.1:{port}"
|
||||
|
||||
def defer(callback: Callable[[], object]) -> None:
|
||||
cleanup.callback(callback)
|
||||
|
||||
browser: Final = idp.browser_client(callback_url=f"{base_url}/sso/callback", defer=defer)
|
||||
config: Final = directory / "oauth-gateway.yaml"
|
||||
config.write_text(
|
||||
"model_list: []\n"
|
||||
"general_settings:\n"
|
||||
" master_key: os.environ/LITELLM_MASTER_KEY\n"
|
||||
" database_url: os.environ/DATABASE_URL\n"
|
||||
" enable_jwt_auth: true\n"
|
||||
" litellm_jwtauth:\n"
|
||||
" user_id_jwt_field: sub\n"
|
||||
" user_email_jwt_field: email\n"
|
||||
" team_ids_jwt_field: groups\n"
|
||||
" user_id_upsert: true\n"
|
||||
)
|
||||
environment: Final = {
|
||||
**{key: value for key, value in os.environ.items() if not key.startswith(INHERITED_ENV_PREFIXES)},
|
||||
**browser.environment(idp.discovery()),
|
||||
"PROXY_BASE_URL": base_url,
|
||||
"JWT_PUBLIC_KEY_URL": idp.jwks_url,
|
||||
"JWT_ISSUER": idp.issuer,
|
||||
"JWT_AUDIENCE": "litellm-e2e",
|
||||
"DISABLE_SCHEMA_UPDATE": "true",
|
||||
"STORE_MODEL_IN_DB": "True",
|
||||
"PYTHONPATH": str(Path(__file__).resolve().parents[3]),
|
||||
}
|
||||
gateway: Final = OAuthGateway(
|
||||
base_url=base_url,
|
||||
proxy=build_proxy_client(
|
||||
base_url=base_url,
|
||||
control_plane_base_url=base_url,
|
||||
replica_urls=(base_url,),
|
||||
control_replica_urls=(base_url,),
|
||||
master_key=os.environ["LITELLM_MASTER_KEY"],
|
||||
),
|
||||
_environment=environment,
|
||||
_command=(sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config), "--port", str(port)),
|
||||
_log_path=directory / "oauth-gateway.log",
|
||||
)
|
||||
cleanup.callback(gateway.stop)
|
||||
gateway.start()
|
||||
return gateway
|
||||
|
|
@ -1,218 +0,0 @@
|
|||
"""On-demand e2e: a chat completion drives a gateway-managed OAuth MCP server.
|
||||
|
||||
The real end-user flow for MCP over an OAuth server: a user registers a Linear
|
||||
authorization_code server, authorizes it once so the gateway stores their
|
||||
upstream token, then sends a normal /chat/completions request with the Linear
|
||||
MCP attached. The gateway resolves the user from the LiteLLM key, lists Linear's
|
||||
tools with the stored per-user token, lets the model call one, executes it
|
||||
upstream with that token, and returns the answer. This is proven against the
|
||||
real Linear MCP server (mcp.linear.app) and a real Anthropic model, once per
|
||||
documented ingress header (x-litellm-api-key and Authorization).
|
||||
|
||||
The authorize dance is seeded through the mcp SDK's OAuthClientProvider; the one
|
||||
step Linear cannot auto-approve is the human consent, so it is captured once out
|
||||
of band (mcp/linear_session_capture.py) into a saved browser session and a
|
||||
headless Chromium clicks Approve every run. The test therefore skips unless
|
||||
E2E_LINEAR_STORAGE_STATE points at that session, so it never runs on the per-PR
|
||||
CI path; it is a nightly/on-demand real-server smoke test.
|
||||
|
||||
Fail-before-fix: without the stored per-user token the gateway lists no Linear
|
||||
tools, so mcp_list_tools comes back empty, nothing is called, and the
|
||||
assertions fail; a served, called, non-empty Linear tool proves the gateway
|
||||
pulled and used the user's token.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import CHEAP_ANTHROPIC_MODEL, LINEAR_MCP_URL, LINEAR_STORAGE_STATE, unique_marker
|
||||
from e2e_http import AuthHeaders
|
||||
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, KeyGenerateBody, McpChatTool, McpServerCreateBody, ObjectPermission
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytest.importorskip("mcp", reason="mcp SDK not installed; run `uv sync --inexact --group e2e-dev`")
|
||||
pytest.importorskip(
|
||||
"playwright.async_api",
|
||||
reason="playwright not installed; run `uv pip install playwright` and `playwright install chromium`",
|
||||
)
|
||||
|
||||
from oauth_chat_client import ChatMcpClient, build_chat_client # noqa: E402 # imports follow the importorskip guards
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.e2e,
|
||||
pytest.mark.skipif(
|
||||
not LINEAR_STORAGE_STATE or not os.path.exists(LINEAR_STORAGE_STATE),
|
||||
reason="set E2E_LINEAR_STORAGE_STATE to a Linear session captured via mcp/linear_session_capture.py",
|
||||
),
|
||||
]
|
||||
|
||||
# Pinned from a live dance during verification (never guessed); the gateway
|
||||
# prefixes every upstream tool name with the server alias. list_teams is a
|
||||
# read-only Linear tool that takes no arguments and returns the caller's teams.
|
||||
LINEAR_READONLY_TOOL = "list_teams"
|
||||
LINEAR_PROMPT = "Use the list_teams tool to list my Linear teams, then reply with the name of one of them."
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def chat_client(proxy: ProxyClient) -> ChatMcpClient:
|
||||
return build_chat_client(proxy)
|
||||
|
||||
|
||||
class TestMcpChatCompletionOauth:
|
||||
"""A scoped internal-user key on a real Linear authorization_code server,
|
||||
used through /chat/completions once per ingress header: the gateway pulls
|
||||
the user's stored upstream token, lists and executes Linear's tools during
|
||||
the completion, and returns the answer."""
|
||||
|
||||
@pytest.mark.covers("mcp.list_tools.oauth.succeeds")
|
||||
@pytest.mark.covers("mcp.call_tool.oauth.succeeds")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.MCP,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
capabilities=(Capability.FUNCTION_CALLING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_chat_completion_uses_linear_with_x_litellm_api_key_header(
|
||||
self, chat_client: ChatMcpClient, resources: ResourceManager
|
||||
) -> None:
|
||||
marker = unique_marker()
|
||||
alias = f"e2elinear{marker}"
|
||||
created = chat_client.create_server(
|
||||
McpServerCreateBody(
|
||||
alias=alias,
|
||||
url=LINEAR_MCP_URL,
|
||||
allow_all_keys=False,
|
||||
auth_type="oauth2",
|
||||
oauth2_flow="authorization_code",
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: chat_client.delete_server(created.server_id))
|
||||
|
||||
stored = chat_client.server_info(created.server_id)
|
||||
assert stored.auth_type == "oauth2"
|
||||
assert stored.oauth2_flow == "authorization_code"
|
||||
assert stored.allow_all_keys is False
|
||||
|
||||
key = chat_client.proxy.generate_key(
|
||||
KeyGenerateBody(
|
||||
user_id="e2e-test-user",
|
||||
object_permission=ObjectPermission(mcp_servers=[created.server_id]),
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: chat_client.proxy.delete_key(key))
|
||||
|
||||
seeded = chat_client.seed_user_token(alias, key, LINEAR_STORAGE_STATE)
|
||||
assert f"{alias}-{LINEAR_READONLY_TOOL}" in seeded, (
|
||||
f"the authorize dance listed {seeded}, expected it to include {alias}-{LINEAR_READONLY_TOOL}"
|
||||
)
|
||||
|
||||
response = chat_client.chat_with_mcp(
|
||||
AuthHeaders.model_validate({"x-litellm-api-key": f"Bearer {key}"}),
|
||||
ChatBody(
|
||||
model=CHEAP_ANTHROPIC_MODEL,
|
||||
messages=[ChatMessage(role="user", content=LINEAR_PROMPT)],
|
||||
tools=[
|
||||
McpChatTool(
|
||||
server_url=f"litellm_proxy/mcp/{alias}",
|
||||
server_label=alias,
|
||||
require_approval="never",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
assert message is not None and message.content, f"completion returned no answer: {response}"
|
||||
meta = message.provider_specific_fields
|
||||
assert meta is not None, f"no MCP metadata on the completion: {response}"
|
||||
listed = {t.function.name for t in (meta.mcp_list_tools or []) if t.function}
|
||||
assert f"{alias}-{LINEAR_READONLY_TOOL}" in listed, (
|
||||
f"the gateway listed {sorted(listed)}, expected the stored token to surface {alias}-{LINEAR_READONLY_TOOL}"
|
||||
)
|
||||
results = [r for r in (meta.mcp_call_results or []) if r.name == f"{alias}-{LINEAR_READONLY_TOOL}"]
|
||||
assert results and results[0].result, (
|
||||
f"Linear tool {alias}-{LINEAR_READONLY_TOOL} was not executed with a result: {meta.mcp_call_results}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("mcp.list_tools.oauth.succeeds")
|
||||
@pytest.mark.covers("mcp.call_tool.oauth.succeeds")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.MCP,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.ANTHROPIC,),
|
||||
models=(CHEAP_ANTHROPIC_MODEL,),
|
||||
capabilities=(Capability.FUNCTION_CALLING,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
)
|
||||
def test_chat_completion_uses_linear_with_authorization_bearer_header(
|
||||
self, chat_client: ChatMcpClient, resources: ResourceManager
|
||||
) -> None:
|
||||
marker = unique_marker()
|
||||
alias = f"e2elinear{marker}"
|
||||
created = chat_client.create_server(
|
||||
McpServerCreateBody(
|
||||
alias=alias,
|
||||
url=LINEAR_MCP_URL,
|
||||
allow_all_keys=False,
|
||||
auth_type="oauth2",
|
||||
oauth2_flow="authorization_code",
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: chat_client.delete_server(created.server_id))
|
||||
|
||||
stored = chat_client.server_info(created.server_id)
|
||||
assert stored.auth_type == "oauth2"
|
||||
assert stored.oauth2_flow == "authorization_code"
|
||||
assert stored.allow_all_keys is False
|
||||
|
||||
key = chat_client.proxy.generate_key(
|
||||
KeyGenerateBody(
|
||||
user_id="e2e-test-user",
|
||||
object_permission=ObjectPermission(mcp_servers=[created.server_id]),
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: chat_client.proxy.delete_key(key))
|
||||
|
||||
seeded = chat_client.seed_user_token(alias, key, LINEAR_STORAGE_STATE)
|
||||
assert f"{alias}-{LINEAR_READONLY_TOOL}" in seeded, (
|
||||
f"the authorize dance listed {seeded}, expected it to include {alias}-{LINEAR_READONLY_TOOL}"
|
||||
)
|
||||
|
||||
response = chat_client.chat_with_mcp(
|
||||
AuthHeaders.model_validate({"authorization": f"Bearer {key}"}),
|
||||
ChatBody(
|
||||
model=CHEAP_ANTHROPIC_MODEL,
|
||||
messages=[ChatMessage(role="user", content=LINEAR_PROMPT)],
|
||||
tools=[
|
||||
McpChatTool(
|
||||
server_url=f"litellm_proxy/mcp/{alias}",
|
||||
server_label=alias,
|
||||
require_approval="never",
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
message = response.choices[0].message
|
||||
assert message is not None and message.content, f"completion returned no answer: {response}"
|
||||
meta = message.provider_specific_fields
|
||||
assert meta is not None, f"no MCP metadata on the completion: {response}"
|
||||
listed = {t.function.name for t in (meta.mcp_list_tools or []) if t.function}
|
||||
assert f"{alias}-{LINEAR_READONLY_TOOL}" in listed, (
|
||||
f"the gateway listed {sorted(listed)}, expected the stored token to surface {alias}-{LINEAR_READONLY_TOOL}"
|
||||
)
|
||||
results = [r for r in (meta.mcp_call_results or []) if r.name == f"{alias}-{LINEAR_READONLY_TOOL}"]
|
||||
assert results and results[0].result, (
|
||||
f"Linear tool {alias}-{LINEAR_READONLY_TOOL} was not executed with a result: {meta.mcp_call_results}"
|
||||
)
|
||||
|
|
@ -1,216 +0,0 @@
|
|||
"""Real OAuth consent, immediate MCP operations and cold-restart persistence.
|
||||
|
||||
Aggregate SSO uses the SDK's normal authentication. The per-server variant is
|
||||
explicitly a configured two-header client, not an Authorization-only OAuth host.
|
||||
The observed variants forward to the same real Linear upstream and compare its
|
||||
bearer at the forwarding boundary; direct variants retain unmodified discovery.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from e2e_config import LINEAR_MCP_URL, LINEAR_READONLY_TOOL, LINEAR_STORAGE_STATE, unique_marker
|
||||
from e2e_http import AuthHeaders, NoBody, get_external, unwrap
|
||||
from e2e_metadata import Domain, Route, Subject, meta
|
||||
from idp import Identity, Keycloak
|
||||
from lifecycle import ResourceManager
|
||||
from models import (
|
||||
McpOauthCredentials,
|
||||
McpServerCreateBody,
|
||||
ObjectPermission,
|
||||
TeamMemberAddBody,
|
||||
TeamMemberEntry,
|
||||
TeamUpdateBody,
|
||||
)
|
||||
from oauth_chat_client import ChatMcpClient, InMemoryTokenStorage, OauthToolRun, build_chat_client
|
||||
from oauth_gateway import OAuthGateway, OAuthObservation, owned_gateway, stored_oauth
|
||||
from provider_edge import LiveEdge, start_provider_edge
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
pytestmark = [pytest.mark.e2e, pytest.mark.mcp_oauth_live, pytest.mark.provider_live]
|
||||
|
||||
|
||||
class OAuthMetadata(BaseModel):
|
||||
issuer: str
|
||||
authorization_endpoint: str
|
||||
token_endpoint: str
|
||||
registration_endpoint: str
|
||||
|
||||
|
||||
class LinearTeam(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
|
||||
|
||||
class LinearTeams(BaseModel):
|
||||
teams: tuple[LinearTeam, ...]
|
||||
|
||||
|
||||
def assert_tool_result(run: OauthToolRun, tool: str) -> None:
|
||||
assert tool in run.tools
|
||||
assert run.is_error is False
|
||||
try:
|
||||
result: Final = LinearTeams.model_validate_json(run.text)
|
||||
except ValidationError:
|
||||
raise AssertionError("list_teams did not return the expected teams payload") from None
|
||||
assert result.teams, "the test workspace must contain at least one team"
|
||||
assert all(team.id and team.name for team in result.teams), "team results must contain identifiers and names"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def oauth_gateway(idp: Keycloak, tmp_path_factory: pytest.TempPathFactory) -> Iterator[OAuthGateway]:
|
||||
assert LINEAR_STORAGE_STATE and Path(LINEAR_STORAGE_STATE).is_file(), (
|
||||
"E2E_LINEAR_STORAGE_STATE must name a captured Linear login; see mcp/linear_session_capture.py"
|
||||
)
|
||||
assert os.environ.get("E2E_FIXTURE_MODE", "live") == "live", "OAuth acceptance cannot use replay"
|
||||
with ExitStack() as cleanup:
|
||||
yield owned_gateway(idp, tmp_path_factory.mktemp("mcp-oauth"), cleanup)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy(oauth_gateway: OAuthGateway) -> ProxyClient:
|
||||
return oauth_gateway.proxy
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client(proxy: ProxyClient) -> ChatMcpClient:
|
||||
return build_chat_client(proxy)
|
||||
|
||||
|
||||
class TestMcpOauthHappyPath:
|
||||
@pytest.mark.covers("mcp.list_tools.oauth.succeeds")
|
||||
@pytest.mark.covers("mcp.call_tool.oauth.succeeds")
|
||||
@pytest.mark.covers("mcp.call_tool.oauth.persists_across_processes")
|
||||
@meta(
|
||||
Subject(
|
||||
domain=Domain.MCP,
|
||||
route=Route.MCP,
|
||||
)
|
||||
)
|
||||
@pytest.mark.parametrize("route", ("aggregate_sso", "explicit_header_jwt"))
|
||||
@pytest.mark.parametrize("observed", (False, True), ids=("direct", "observed"))
|
||||
def test_consent_list_call_and_cold_restart(
|
||||
self,
|
||||
client: ChatMcpClient,
|
||||
resources: ResourceManager,
|
||||
jwt_identity: Identity,
|
||||
idp: Keycloak,
|
||||
oauth_gateway: OAuthGateway,
|
||||
route: Literal["aggregate_sso", "explicit_header_jwt"],
|
||||
observed: bool,
|
||||
) -> None:
|
||||
alias: Final = f"e2elinear{unique_marker()}"
|
||||
tool: Final = f"{alias}-{LINEAR_READONLY_TOOL}"
|
||||
token: Final = idp.access_token(jwt_identity)
|
||||
observation: Final = OAuthObservation(gateway_token=token)
|
||||
edge: Final = (
|
||||
start_provider_edge(
|
||||
LiveEdge(observe_request=observation.observe),
|
||||
mounts=MappingProxyType(
|
||||
{"linear": "https://mcp.linear.app", ".well-known": "https://mcp.linear.app/.well-known"}
|
||||
),
|
||||
)
|
||||
if observed
|
||||
else None
|
||||
)
|
||||
if edge is not None:
|
||||
resources.defer(edge.shutdown)
|
||||
metadata: Final = (
|
||||
unwrap(
|
||||
get_external(
|
||||
"https://mcp.linear.app/.well-known/oauth-authorization-server",
|
||||
response_type=OAuthMetadata,
|
||||
)
|
||||
)
|
||||
if observed
|
||||
else None
|
||||
)
|
||||
created: Final = client.create_server(
|
||||
McpServerCreateBody(
|
||||
alias=alias,
|
||||
server_name=alias,
|
||||
url=f"{edge.edge.api_base('linear')}/mcp" if edge is not None else LINEAR_MCP_URL,
|
||||
transport="http",
|
||||
allow_all_keys=False,
|
||||
auth_type="oauth2",
|
||||
oauth2_flow="authorization_code",
|
||||
per_server_oauth_discovery=route == "explicit_header_jwt",
|
||||
issuer=metadata.issuer if metadata else None,
|
||||
authorization_url=metadata.authorization_endpoint if metadata else None,
|
||||
token_url=metadata.token_endpoint if metadata else None,
|
||||
registration_url=metadata.registration_endpoint if metadata else None,
|
||||
credentials=McpOauthCredentials(upstream_resource=LINEAR_MCP_URL) if observed else None,
|
||||
)
|
||||
)
|
||||
resources.defer(lambda: client.delete_server(created.server_id))
|
||||
assert client.server_user_credentials(created.server_id) == (), (
|
||||
"scenario must start without upstream credentials"
|
||||
)
|
||||
unwrap(
|
||||
client.proxy.transport.post(
|
||||
"/team/member_add",
|
||||
headers=client.proxy.transport.master,
|
||||
json=TeamMemberAddBody(
|
||||
team_id=jwt_identity.group, member=TeamMemberEntry(user_id=jwt_identity.user_id, role="user")
|
||||
),
|
||||
response_type=NoBody,
|
||||
)
|
||||
)
|
||||
client.proxy.update_team(
|
||||
TeamUpdateBody(
|
||||
team_id=jwt_identity.group,
|
||||
object_permission=ObjectPermission(mcp_servers=[created.server_id]),
|
||||
)
|
||||
)
|
||||
headers: Final = {"x-litellm-api-key": f"Bearer {token}"} if route == "explicit_header_jwt" else {}
|
||||
resources.defer(
|
||||
lambda: client.revoke_user_token(
|
||||
created.server_id,
|
||||
AuthHeaders(authorization=f"Bearer {idp.access_token(jwt_identity)}"),
|
||||
)
|
||||
)
|
||||
identity: Final = jwt_identity if route == "aggregate_sso" else None
|
||||
first: Final = client.list_and_call(
|
||||
alias,
|
||||
headers,
|
||||
InMemoryTokenStorage(),
|
||||
LINEAR_STORAGE_STATE,
|
||||
tool,
|
||||
{},
|
||||
base_url=oauth_gateway.base_url,
|
||||
identity=identity,
|
||||
)
|
||||
assert_tool_result(first, tool)
|
||||
credentials: Final = client.server_user_credentials(created.server_id)
|
||||
assert len(credentials) == 1
|
||||
assert credentials[0].user_id == jwt_identity.user_id
|
||||
assert credentials[0].credential_type == "oauth2"
|
||||
first_stored_oauth: Final = stored_oauth(jwt_identity.user_id, created.server_id)
|
||||
if observed:
|
||||
observation.assert_forwarded(first_stored_oauth)
|
||||
oauth_gateway.restart()
|
||||
fresh_token: Final = idp.access_token(jwt_identity)
|
||||
observation.gateway_token = fresh_token
|
||||
second: Final = client.list_and_call(
|
||||
alias,
|
||||
{"Authorization": f"Bearer {fresh_token}"} if identity is None else {},
|
||||
InMemoryTokenStorage(),
|
||||
LINEAR_STORAGE_STATE if identity is not None else None,
|
||||
tool,
|
||||
{},
|
||||
base_url=oauth_gateway.base_url,
|
||||
identity=identity,
|
||||
allow_upstream_consent=False,
|
||||
)
|
||||
assert_tool_result(second, tool)
|
||||
second_stored_oauth: Final = stored_oauth(jwt_identity.user_id, created.server_id)
|
||||
if observed:
|
||||
observation.assert_forwarded(second_stored_oauth)
|
||||
|
|
@ -45,8 +45,6 @@ class KeyLoggingCallbackVars(BaseModel):
|
|||
langfuse_public_key: str | None = Field(default=None, repr=False)
|
||||
langfuse_secret_key: str | None = Field(default=None, repr=False)
|
||||
langfuse_host: str | None = Field(default=None, repr=False)
|
||||
wandb_api_key: str | None = Field(default=None, repr=False)
|
||||
weave_project_id: str | None = None
|
||||
|
||||
|
||||
class KeyLoggingCallback(BaseModel):
|
||||
|
|
@ -364,6 +362,7 @@ class ChatBody(BaseModel):
|
|||
thinking: ThinkingParam | None = None
|
||||
service_tier: str | None = None
|
||||
prompt_cache_key: str | None = None
|
||||
prompt_cache_retention: str | None = None
|
||||
tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None
|
||||
tool_choice: str | None = None
|
||||
guardrails: list[str] | None = None
|
||||
|
|
@ -732,28 +731,14 @@ class McpInfo(BaseModel):
|
|||
logo_url: str | None = None
|
||||
|
||||
|
||||
class McpOauthCredentials(BaseModel):
|
||||
upstream_resource: str
|
||||
|
||||
|
||||
class McpServerCreateBody(BaseModel):
|
||||
"""POST /v1/mcp/server. For a gateway-managed OAuth server, `auth_type` is
|
||||
`oauth2` and `oauth2_flow` is `authorization_code`; the upstream endpoints
|
||||
are discovered and registered via DCR when left unset. `allow_all_keys`
|
||||
false scopes the server to keys granted it through object_permission."""
|
||||
"""POST /v1/mcp/server. `allow_all_keys` false scopes the server to keys
|
||||
granted it through object_permission."""
|
||||
|
||||
alias: str
|
||||
url: str
|
||||
transport: str = "http"
|
||||
allow_all_keys: bool = True
|
||||
auth_type: str | None = None
|
||||
oauth2_flow: Literal["client_credentials", "authorization_code"] | None = None
|
||||
per_server_oauth_discovery: bool | None = None
|
||||
issuer: str | None = None
|
||||
authorization_url: str | None = None
|
||||
token_url: str | None = None
|
||||
registration_url: str | None = None
|
||||
credentials: McpOauthCredentials | None = Field(default=None, repr=False)
|
||||
server_name: str | None = None
|
||||
description: str | None = None
|
||||
mcp_info: McpInfo | None = None
|
||||
|
|
@ -793,26 +778,6 @@ class McpServerListResponse(RootModel[list[McpServerRow]]):
|
|||
"""GET /v1/mcp/server answers with a bare array of servers."""
|
||||
|
||||
|
||||
class McpServerUserCredentialRow(BaseModel):
|
||||
user_id: str
|
||||
credential_type: Literal["oauth2", "byok"]
|
||||
expires_at: str | None = None
|
||||
connected_at: str | None = None
|
||||
updated_at: str
|
||||
|
||||
|
||||
class McpServerUserCredentialListResponse(RootModel[tuple[McpServerUserCredentialRow, ...]]):
|
||||
"""GET /v1/mcp/server/{server_id}/user-credentials answers with a bare array."""
|
||||
|
||||
|
||||
class McpOauthUserCredentialStatus(BaseModel):
|
||||
server_id: str
|
||||
has_credential: bool
|
||||
expires_at: str | None = None
|
||||
is_expired: bool = False
|
||||
connected_at: str | None = None
|
||||
|
||||
|
||||
class ToolsetTool(BaseModel):
|
||||
server_id: str
|
||||
tool_name: str
|
||||
|
|
@ -1392,9 +1357,11 @@ class LiteLLMParamsBody(BaseModel):
|
|||
litellm_credential_name: str | None = None
|
||||
api_base: str | None = Field(default=None, repr=False)
|
||||
api_version: str | None = None
|
||||
responses_api_version: str | None = None
|
||||
realtime_protocol: str | None = None
|
||||
allowed_openai_params: list[str] | None = None
|
||||
drop_params: bool | None = None
|
||||
thought_signature_in_tool_call_id: bool | None = None
|
||||
aws_access_key_id: str | None = Field(default=None, repr=False)
|
||||
aws_secret_access_key: str | None = Field(default=None, repr=False)
|
||||
aws_region_name: str | None = None
|
||||
|
|
@ -1448,7 +1415,7 @@ class LiteLLMParamsBody(BaseModel):
|
|||
order: int | None = None
|
||||
|
||||
|
||||
ModelMode = Literal["chat", "batch", "realtime", "image_generation"]
|
||||
ModelMode = Literal["chat", "embedding", "batch", "realtime", "image_generation"]
|
||||
|
||||
|
||||
class TokenCountModelBody(BaseModel, frozen=True):
|
||||
|
|
@ -1457,6 +1424,7 @@ class TokenCountModelBody(BaseModel, frozen=True):
|
|||
|
||||
|
||||
class ModelInfoBody(BaseModel):
|
||||
enforce_supported_endpoints: bool | None = None
|
||||
# id is left unset so the proxy assigns a unique model_id per deployment.
|
||||
# Pinning it to the model_name made re-registrations of a fixed-name model
|
||||
# (e.g. the batch suite's openai-batch) collide on the model_id unique
|
||||
|
|
|
|||
59
tests/e2e/perf/UI_QUERIES.md
Normal file
59
tests/e2e/perf/UI_QUERIES.md
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
# Admin UI journey latency on the 1TB SpendLogs restore
|
||||
|
||||
Run `test_ui_journeys.py` against a dedicated proxy, its Admin UI and a disposable database restored from the metadata-only performance seed. A headless Chromium logs into the dashboard and performs the journeys below, timing each one from the user's click or navigation until the data is on screen. No LLM is called. The test does not look at which routes, SQL or tables served the page, so it stays valid whether the backend gets faster queries or new aggregate tables
|
||||
|
||||
## Journeys
|
||||
|
||||
| name | what the user does | timer stops when |
|
||||
| --- | --- | --- |
|
||||
| `logs_open_24h` | opens Logs (default Last 24 Hours) | request rows are rendered and the refresh control is idle |
|
||||
| `logs_last_7_days` | switches the time range to Last 7 Days | same |
|
||||
| `logs_next_page` | clicks the next page control | same |
|
||||
| `logs_filter_key_hash` | filters by the seeded key hash and applies | same |
|
||||
| `logs_filter_key_alias` | picks the first key alias in the filter drawer and applies | same |
|
||||
| `logs_filter_team` | picks the first team in the filter drawer and applies | same |
|
||||
| `logs_filter_model` | picks the first model in the filter drawer and applies | same |
|
||||
| `usage_open` | opens Usage | summary tiles show dollar values with no skeletons left |
|
||||
| `usage_last_30_days` | picks Last 30 days in the Usage date picker | same |
|
||||
| `usage_team_view` | switches the Usage view to Team Usage | same |
|
||||
| `usage_customer_view` | switches the Usage view to Customer Usage | same |
|
||||
| `usage_model_activity_30_days` | on the Model Activity tab, picks Last 30 days | same |
|
||||
| `cache_open` | opens Caching | the Cache Hits tile is visible and the Cache Hits vs API Requests chart has bars |
|
||||
| `models_open` | opens Models + Endpoints | model rows are rendered and the loading state is gone |
|
||||
| `keys_open` | opens Virtual Keys | key rows are rendered and the loading state is gone |
|
||||
|
||||
The mix follows what customers report as slow: the Logs page on `LiteLLM_SpendLogs`, the Usage page on the `LiteLLM_Daily*Spend` rollups (30 day ranges, Customer and Model views), and the Models and Keys pages.
|
||||
|
||||
## Seeded rows
|
||||
|
||||
The snapshot carries the SpendLogs rows and their Daily* rollups. `setup/ui_query_fixture.py` checks that `LiteLLM_DailyUserSpend` and `LiteLLM_DailyTeamSpend` have rows in the week before the anchor, then fills the tables the snapshot does not have so the Keys, Models and filter journeys see realistic counts: 100 teams (`perf-team-N`), 2,000 users (`perf-user-N`), 5,000 keys whose token hashes and `perf-key-N` aliases match the seeded log rows, and 200 DB models pointed at the perf mock provider. Every insert is `ON CONFLICT DO NOTHING`, so reruns on the same restore are no-ops. Nothing here runs outside the disposable restore
|
||||
|
||||
Each journey is measured once cold, then `E2E_UI_JOURNEY_REPEATS` more times (default 9) from the same starting state, inside one logged-in tab whose clock is pinned to the seed anchor so relative windows land on seeded rows. The browser timezone is UTC to match the seed
|
||||
|
||||
## Safety
|
||||
|
||||
The seeded source is never the target. `E2E_UI_PERF_DISPOSABLE=1`, `E2E_RESET_SPEND_LOGS=0` and a hostname matching `db-perf-perf-<id>.litellm.svc` are all required, and in-flight SQL is cancelled only on that disposable restore between journeys (see `setup/ui_query_isolation.py`)
|
||||
|
||||
## Running
|
||||
|
||||
The componentized perf stack serves the dashboard behind the same router as the proxy, so `LITELLM_PROXY_URL` covers `/ui`. Set `E2E_UI_BASE_URL` only if the UI lives elsewhere. The admin login uses `E2E_UI_USERNAME` (default `admin`) and `E2E_UI_PASSWORD` (default the master key)
|
||||
|
||||
```bash
|
||||
export E2E_PERF_TESTS=1 E2E_UI_PERF_DISPOSABLE=1 E2E_RESET_SPEND_LOGS=0
|
||||
export DATABASE_URL=postgresql://...@db-perf-perf-<id>.litellm.svc:5432/litellm
|
||||
export E2E_DATABASE_METRICS_URLS=http://<backend-pod>:4000/metrics
|
||||
export E2E_ARTIFACT_DIR=/tmp/e2e-artifacts
|
||||
uv run pytest tests/e2e/perf/test_ui_journeys.py --reruns 0 --junitxml="$E2E_ARTIFACT_DIR/junit.xml" -q
|
||||
```
|
||||
|
||||
`E2E_UI_JOURNEY_TIMEOUT_SECONDS` (default 120) bounds one journey. A journey that never settles is recorded as a `timeout` sample and a journey whose page rendered empty or threw in the browser as a `ui` sample. The test fails when any sample failed, or when the p50 or p90 across all samples is over the journey's `Ceiling` in `UI_JOURNEY_LIMITS` (`setup/ui_journeys.py`, 10s/20s for Logs journeys, 20s/40s for Usage and Caching, 5s/10s for the Models and Keys tables, generous until the first 1TB run sets real numbers). The artifacts are written before the assertion so a failed run still ships its samples
|
||||
|
||||
## Artifacts
|
||||
|
||||
Each JUnit testcase `test_ui_journey_latency[<name>]` carries `perf_ui_journeys`, `perf_ui_failures`, `perf_ui_seconds_p50`, `perf_ui_seconds_p90`, `perf_ui_seconds_max`, `perf_ui_first_seconds`, `perf_ui_repeated_seconds_p50`, `perf_ui_repeated_seconds_max` and `perf_ui_database_failures`. `ui-journey-<name>.json` holds every sample with its on-screen row count and `ui-isolation-<name>.jsonl` the SQL isolation evidence
|
||||
|
||||
The Buildkite `ui_queries` lane restores the pinned snapshot, waits for migrations and indexes, runs these journeys with no retries and uploads the JUnit and JSON artifacts. Validate selectors on a small restore first, then take numbers from the completed 1TB snapshot
|
||||
|
||||
To measure a released version on the same snapshot, create the litellm-perf build with `perf_gateway_version=<tag>` (for example `v1.106.1`) next to `litellm_sha`. The gateway, backend and the restore's schema upgrade then come from `ghcr.io/berriai/litellm-{gateway,backend,migrations}:<tag>`, where the migrations image runs `prisma migrate deploy` and builds the request-log indexes the way a customer upgrade does, while the runner and these journeys come from `litellm_sha`. The fixture skips its upgrade-worker wait when `_LiteLLMPerfUpgrade` is absent. The snapshot's schema is v1.105.1 and the restore only migrates forward, so older tags cannot be measured
|
||||
|
||||
Each sample in `ui-journey-<name>.json` also records the `status` and `request` (method and path) of the data request the page issued, the first non-2xx one when there is one, so a failed sample names what the gateway answered. A failed sample with a 4xx or 5xx status is counted as `perf_ui_error_http_<status>` instead of `perf_ui_error_ui`. Each journey is also bounded by a wall clock of `E2E_UI_JOURNEY_TIMEOUT_SECONDS x (repeats + 2)`, so one hung browser call ends the journey with a `wall_clock` sample instead of eating the lane.
|
||||
15
tests/e2e/perf/cases/chat_completions_2rps.py
Normal file
15
tests/e2e/perf/cases/chat_completions_2rps.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions
|
||||
route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from .direct_vs_bridge import direct_vs_bridge
|
||||
|
||||
CASE: Final = direct_vs_bridge(
|
||||
"chat_completions_2rps",
|
||||
"chat_completions",
|
||||
direct="gpt-6-luna",
|
||||
bridge="gpt-6-luna-responses",
|
||||
)
|
||||
31
tests/e2e/perf/cases/direct_vs_bridge.py
Normal file
31
tests/e2e/perf/cases/direct_vs_bridge.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on
|
||||
that provider's own API (direct) and half on a deployment it has to translate to another provider
|
||||
API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as
|
||||
`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the
|
||||
translation's cost."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey
|
||||
|
||||
from .mixed_size_requests_2rps import CASE as MIXED
|
||||
|
||||
|
||||
def _scoped(key: VirtualKey, direct: str) -> VirtualKey:
|
||||
return replace(key, models=(direct,)) if key.models else key
|
||||
|
||||
|
||||
def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase:
|
||||
return replace(
|
||||
MIXED,
|
||||
name=name,
|
||||
teams=(
|
||||
replace(MIXED.team("team-a"), models=(direct, bridge)),
|
||||
replace(MIXED.team("team-b"), models=(bridge,)),
|
||||
),
|
||||
virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys),
|
||||
model_groups=(direct, bridge),
|
||||
traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}),
|
||||
)
|
||||
50
tests/e2e/perf/cases/event_loop_stalls.py
Normal file
50
tests/e2e/perf/cases/event_loop_stalls.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""Where the gateway holds its event loop for 100 ms or more: the mixed case's callers, model groups, guardrails
|
||||
and body mix (short chats, 300k to 1M-token contexts, tool calls, half streamed) at 1 rps for 120 requests against
|
||||
a gateway running the stdlib asyncio loop in debug mode, which logs every loop step over 0.1 s with the task that
|
||||
ran it. No latency, CPU or datastore ceilings: the one check is that no step over 0.1 s comes from a location that
|
||||
is not on the allowlist below, and that the listed ones do not show up more often than they do today. A stream
|
||||
that stops yielding (the 46925 fix), a provider handler that re-walks a 1M-token body on the loop (46967), or a
|
||||
new hook that deep-copies the request inline all land here as a new row. Runs on its own stack: the gateway needs
|
||||
`PYTHONASYNCIODEBUG=1` and `LITELLM_EVENT_LOOP=asyncio` (uvloop reports nothing and crashed under debug locally),
|
||||
and the runner `E2E_PERF_EVENT_LOOP=1`, so the nightly build, which sets neither, skips it. The allowlist counts
|
||||
are twice the highest count any of ten back-to-back local runs showed on that row (rows that showed nothing in
|
||||
those ten but did in the earlier before/after legs for 46925 and 46967, or in the first scheduled-lane run, get 2),
|
||||
so each run sits at or under half of every cap and a listed location doubling in frequency is a failure, not a flake. The
|
||||
`garbage collection` bucket is the gen-2 passes the collector logs in debug mode (0.6 to 0.7 s each, one every
|
||||
minute or two of this traffic, plus the ramp while the heap fills on a local whole-file run), which asyncio would
|
||||
otherwise charge to a random task; its count is the signal for a change that makes much more garbage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from setup.event_loop import EventLoopStalls, KnownStall
|
||||
|
||||
from cases.mixed_size_requests_2rps import CASE as MIXED
|
||||
|
||||
CASE: Final = replace(
|
||||
MIXED,
|
||||
name="event_loop_stalls",
|
||||
traffic=replace(MIXED.traffic, rps=1.0, total_requests=120, concurrency=10),
|
||||
limits=(),
|
||||
event_loop=EventLoopStalls(
|
||||
allowlist=(
|
||||
KnownStall("garbage collection", max_count=4),
|
||||
KnownStall("LifespanOn.main", max_count=4),
|
||||
KnownStall("RequestResponseCycle.run_asgi", max_count=2),
|
||||
KnownStall("with_proxy_compaction_executor", max_count=2),
|
||||
KnownStall("Logging.async_success_handler", max_count=2),
|
||||
KnownStall("Logging.dispatch_success_handlers", max_count=2),
|
||||
KnownStall("_TrackedPrismaEngine", max_count=2),
|
||||
KnownStall("prisma/actions.py", max_count=2),
|
||||
KnownStall("SpendCounterReseed", max_count=2),
|
||||
KnownStall("Subscriber._run", max_count=2),
|
||||
KnownStall("RedisBatch.flush", max_count=2),
|
||||
KnownStall("run_coroutine_job", max_count=2),
|
||||
KnownStall("user_api_key_auth.py", max_count=2),
|
||||
KnownStall("route_streaming_logging_to_handler", max_count=2),
|
||||
KnownStall("TaskHandle._run_coro", max_count=2),
|
||||
),
|
||||
),
|
||||
)
|
||||
16
tests/e2e/perf/cases/messages_2rps.py
Normal file
16
tests/e2e/perf/cases/messages_2rps.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route,
|
||||
`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the
|
||||
Bedrock mock's Converse route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from .direct_vs_bridge import direct_vs_bridge
|
||||
|
||||
CASE: Final = direct_vs_bridge(
|
||||
"messages_2rps",
|
||||
"messages",
|
||||
direct="claude-sonnet-5",
|
||||
bridge="claude-sonnet-5-bedrock-converse",
|
||||
)
|
||||
|
|
@ -62,13 +62,24 @@ CASE: Final = PerformanceTestCase(
|
|||
),
|
||||
limits=(
|
||||
Limit("unexpected_status_count", max=3),
|
||||
Limit("gateway_added_seconds", percentile=50, max=0.58),
|
||||
Limit("gateway_added_seconds", percentile=99, max=6.1),
|
||||
Limit("time_to_first_token_seconds", percentile=50, max=2.7),
|
||||
Limit("time_to_first_token_seconds", percentile=99, max=8.8),
|
||||
Limit("gateway_added_seconds", percentile=50, max=0.29),
|
||||
Limit("gateway_added_seconds", percentile=99, max=2.7),
|
||||
Limit("litellm_overhead_seconds", percentile=50, max=0.052),
|
||||
Limit("litellm_overhead_seconds", percentile=99, max=1.8),
|
||||
Limit("litellm_response_seconds", percentile=50, max=0.93),
|
||||
Limit("litellm_response_seconds", percentile=99, max=2.7),
|
||||
Limit("prometheus_redis_latency_mean_seconds", max=0.002),
|
||||
Limit("postgres_active_sessions", max=0.0094),
|
||||
Limit("postgres_slowest_statement_seconds", max=0.042),
|
||||
Limit("postgres_peak_connections", max=23),
|
||||
Limit("time_to_first_token_seconds", percentile=50, max=1.56),
|
||||
Limit("time_to_first_token_seconds", percentile=99, max=4.5),
|
||||
Limit("memory_growth_megabytes", max=600),
|
||||
Limit("cpu_seconds_per_request", max=1.26),
|
||||
Limit("postgres_calls_per_request", max=3.5),
|
||||
Limit("redis_calls_per_request", max=21.0),
|
||||
Limit("cpu_seconds_per_request", max=0.57),
|
||||
Limit("postgres_calls_per_request", max=2.64),
|
||||
Limit("postgres_max_statement_per_request", max=0.43),
|
||||
Limit("postgres_max_background_statement_per_request", max=0.86),
|
||||
Limit("redis_calls_per_request", max=15.4),
|
||||
),
|
||||
background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
|
||||
)
|
||||
|
|
|
|||
24
tests/e2e/perf/cases/mixed_size_requests_ramp.py
Normal file
24
tests/e2e/perf/cases/mixed_size_requests_ramp.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
"""The mixed-size traffic stepped up until the pod stops keeping up: the same callers, guardrails, mocks and body
|
||||
mix (short chats, 300k to 1M-token contexts, tool calls, half streamed) as `mixed_size_requests_2rps`, 2 rps more
|
||||
every 90 s from 2 to 14 on the same one 4-worker pod. Where the 2 rps case prices one big body on its own, this one
|
||||
reports `sustained_rps`, the last step the pod held with big bodies overlapping in every worker, so a handler that
|
||||
re-walks a 1M-token body on the event loop or a worker that holds several of them in memory at once moves the knee
|
||||
down and fails the floor."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from setup.performance_test_case import Limit, Ramp, ramped
|
||||
|
||||
from cases.mixed_size_requests_2rps import CASE as MIXED
|
||||
|
||||
RAMP: Final = Ramp(first_rps=2.0, last_rps=14.0, step_rps=2.0, step_seconds=90.0, held_p99_seconds=5.0)
|
||||
|
||||
CASE: Final = replace(
|
||||
MIXED,
|
||||
name="mixed_size_requests_ramp",
|
||||
traffic=ramped(MIXED.traffic, RAMP, seed=9, concurrency=300),
|
||||
limits=(Limit("sustained_rps", min=8),),
|
||||
)
|
||||
15
tests/e2e/perf/cases/responses_2rps.py
Normal file
15
tests/e2e/perf/cases/responses_2rps.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route,
|
||||
`claude-sonnet-5` is bridged to the Anthropic mock's Messages route."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
from .direct_vs_bridge import direct_vs_bridge
|
||||
|
||||
CASE: Final = direct_vs_bridge(
|
||||
"responses_2rps",
|
||||
"responses",
|
||||
direct="gpt-6-luna",
|
||||
bridge="claude-sonnet-5",
|
||||
)
|
||||
|
|
@ -64,11 +64,21 @@ CASE: Final = PerformanceTestCase(
|
|||
),
|
||||
limits=(
|
||||
Limit("unexpected_status_count", max=3),
|
||||
Limit("gateway_added_seconds", percentile=50, max=0.29),
|
||||
Limit("time_to_first_token_seconds", percentile=99, max=3.7),
|
||||
Limit("gateway_added_seconds", percentile=50, max=0.165),
|
||||
Limit("litellm_overhead_seconds", percentile=50, max=0.006),
|
||||
Limit("litellm_overhead_seconds", percentile=99, max=0.35),
|
||||
Limit("litellm_response_seconds", percentile=50, max=0.91),
|
||||
Limit("litellm_response_seconds", percentile=99, max=1.6),
|
||||
Limit("prometheus_redis_latency_mean_seconds", max=0.0026),
|
||||
Limit("postgres_active_sessions", max=0.0052),
|
||||
Limit("postgres_slowest_statement_seconds", max=0.011),
|
||||
Limit("postgres_peak_connections", max=23),
|
||||
Limit("time_to_first_token_seconds", percentile=99, max=2.05),
|
||||
Limit("memory_growth_megabytes", max=100),
|
||||
Limit("postgres_calls_per_request", max=1.8),
|
||||
Limit("redis_calls_per_request", max=18),
|
||||
Limit("cpu_seconds_per_request", max=0.29),
|
||||
Limit("postgres_calls_per_request", max=1.31),
|
||||
Limit("postgres_max_statement_per_request", max=0.27),
|
||||
Limit("postgres_max_background_statement_per_request", max=0.9),
|
||||
Limit("redis_calls_per_request", max=13.8),
|
||||
Limit("cpu_seconds_per_request", max=0.15),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
24
tests/e2e/perf/cases/small_bodies_ramp.py
Normal file
24
tests/e2e/perf/cases/small_bodies_ramp.py
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
"""The small-bodies traffic stepped up until the pod stops keeping up: the same callers, guardrails, mocks and
|
||||
2.5k to 4k-token bodies as `small_bodies_5rps`, 10 rps more every 90 s from 10 to 70 on the same one 4-worker pod.
|
||||
The 5 rps case prices one request with the pod idle; this one reports `sustained_rps`, the last step the pod
|
||||
held (gateway added p99 under 5 s, the queue still draining), so a change that moves the knee down, a lock held
|
||||
across an await, a pool that fills, a flush that stalls the loop while dozens of requests are in flight, fails the
|
||||
floor instead of hiding under a fixed rate the pod still manages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from setup.performance_test_case import Limit, Ramp, ramped
|
||||
|
||||
from cases.small_bodies_5rps import CASE as SMALL
|
||||
|
||||
RAMP: Final = Ramp(first_rps=10.0, last_rps=70.0, step_rps=10.0, step_seconds=90.0, held_p99_seconds=5.0)
|
||||
|
||||
CASE: Final = replace(
|
||||
SMALL,
|
||||
name="small_bodies_ramp",
|
||||
traffic=ramped(SMALL.traffic, RAMP, seed=13, concurrency=400),
|
||||
limits=(Limit("sustained_rps", min=20),),
|
||||
)
|
||||
|
|
@ -1,7 +1,10 @@
|
|||
"""Postgres and Redis work the gateway did during the traffic phase, split into what a request waits on and
|
||||
what runs after the response. Both come from the servers' own accounting, so nothing here depends on how the
|
||||
gateway talks to them: Postgres statement counts from `pg_stat_statements` (read before and after the run) and
|
||||
Redis commands from a `MONITOR` connection held open for the run. `DATABASE_URL` and `REDIS_HOST` are the
|
||||
Redis commands from a `MONITOR` connection held open for the run, Postgres execution time per statement from the
|
||||
same `pg_stat_statements` rows (`postgres_active_sessions`, the average-active-sessions number RDS Performance
|
||||
Insights calls DB load, and the slowest statement's mean) and the connection count from `pg_stat_activity` sampled
|
||||
once a second (`postgres_peak_connections`). `DATABASE_URL` and `REDIS_HOST` are the
|
||||
runner's own env, as in the other suites; without them the counters are None and any Limit on them reports as
|
||||
missing. `E2E_DATABASE_ADMIN_URL`, when set, is used once to create the `pg_stat_statements` extension. Only the
|
||||
runtime user's own statements are read: other users' rows come back with a NULL queryid and hidden text."""
|
||||
|
|
@ -19,6 +22,8 @@ from typing import TYPE_CHECKING, Final, Literal
|
|||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from setup.series import Recorder, Series
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import redis
|
||||
|
||||
|
|
@ -26,13 +31,18 @@ Bucket = Literal["awaited", "background", "plumbing", "unclassified"]
|
|||
|
||||
_COMMITS_QUERY: Final = "SELECT xact_commit FROM pg_stat_database WHERE datname = current_database()"
|
||||
_STATEMENTS_QUERY: Final = (
|
||||
"SELECT queryid, calls, query FROM pg_stat_statements "
|
||||
"SELECT queryid, calls, query, total_exec_time FROM pg_stat_statements "
|
||||
"WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) "
|
||||
"AND userid = (SELECT usesysid FROM pg_user WHERE usename = current_user) AND queryid IS NOT NULL"
|
||||
)
|
||||
_CREATE_EXTENSION: Final = "CREATE EXTENSION IF NOT EXISTS pg_stat_statements"
|
||||
_CONNECTIONS_QUERY: Final = (
|
||||
"SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() "
|
||||
"AND backend_type = 'client backend' AND pid <> pg_backend_pid()"
|
||||
)
|
||||
_CONNECTIONS_SAMPLE_SECONDS: Final = 1.0
|
||||
_COMMITS_ROW: Final = TypeAdapter(tuple[int])
|
||||
_STATEMENT_ROWS: Final = TypeAdapter(list[tuple[int, int, str]])
|
||||
_STATEMENT_ROWS: Final = TypeAdapter(list[tuple[int, int, str, float]])
|
||||
_INFO: Final = TypeAdapter(dict[str, object])
|
||||
_INT: Final = TypeAdapter(int)
|
||||
_MONITOR_LINE: Final = TypeAdapter(dict[str, object])
|
||||
|
|
@ -64,6 +74,11 @@ class StatementCalls:
|
|||
queryid: int
|
||||
query: str
|
||||
calls: int
|
||||
total_exec_ms: float = 0.0
|
||||
|
||||
@property
|
||||
def mean_exec_seconds(self) -> float | None:
|
||||
return self.total_exec_ms / 1000 / self.calls if self.calls > 0 else None
|
||||
|
||||
@property
|
||||
def bucket(self) -> Bucket:
|
||||
|
|
@ -76,6 +91,11 @@ class StatementCalls:
|
|||
return "plumbing"
|
||||
return "awaited" if verb == "SELECT" and tables[0] in _AWAITED_TABLES else "background"
|
||||
|
||||
@property
|
||||
def table(self) -> str:
|
||||
tables: Final = _TABLE.findall(self.query)
|
||||
return tables[0] if tables else ""
|
||||
|
||||
@property
|
||||
def summary(self) -> str:
|
||||
return re.sub(r"\s+", " ", self.query)[:110]
|
||||
|
|
@ -131,16 +151,95 @@ def statement_deltas(before: DbCounters | None, after: DbCounters | None) -> tup
|
|||
"""Calls per statement during the run: `after` minus `before` by queryid, new statements whole."""
|
||||
if before is None or after is None or before.statements is None or after.statements is None:
|
||||
return None
|
||||
earlier: Final = {statement.queryid: statement.calls for statement in before.statements}
|
||||
earlier: Final = {statement.queryid: statement for statement in before.statements}
|
||||
return tuple(
|
||||
StatementCalls(
|
||||
queryid=statement.queryid, query=statement.query, calls=statement.calls - earlier.get(statement.queryid, 0)
|
||||
)
|
||||
_statement_delta(statement, earlier.get(statement.queryid))
|
||||
for statement in after.statements
|
||||
if statement.calls > earlier.get(statement.queryid, 0)
|
||||
if statement.calls > (0 if statement.queryid not in earlier else earlier[statement.queryid].calls)
|
||||
)
|
||||
|
||||
|
||||
def _statement_delta(statement: StatementCalls, earlier: StatementCalls | None) -> StatementCalls:
|
||||
return StatementCalls(
|
||||
queryid=statement.queryid,
|
||||
query=statement.query,
|
||||
calls=statement.calls - (earlier.calls if earlier is not None else 0),
|
||||
total_exec_ms=statement.total_exec_ms - (earlier.total_exec_ms if earlier is not None else 0.0),
|
||||
)
|
||||
|
||||
|
||||
def active_sessions(statements: tuple[StatementCalls, ...] | None, wall_seconds: float) -> float | None:
|
||||
"""Seconds Postgres spent executing the run's statements per second of wall time, the average active
|
||||
sessions figure RDS Performance Insights reports as DB load."""
|
||||
if statements is None or wall_seconds <= 0:
|
||||
return None
|
||||
return sum(statement.total_exec_ms for statement in statements) / 1000 / wall_seconds
|
||||
|
||||
|
||||
def slowest_statement(statements: tuple[StatementCalls, ...] | None) -> StatementCalls | None:
|
||||
"""The run's statement with the highest mean execution time."""
|
||||
timed: Final = tuple(statement for statement in statements or () if statement.mean_exec_seconds is not None)
|
||||
return max(timed, key=lambda statement: statement.mean_exec_seconds or 0.0, default=None)
|
||||
|
||||
|
||||
class ConnectionsSampler:
|
||||
"""`pg_stat_activity` client backends on the gateway's database once a second on a background thread, minus
|
||||
this sampler's own connection, so the peak is what the gateway (and the test's own reads) held open.
|
||||
`series` keeps every count with its time for a ramped run's per-step peaks."""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
self._url: Final = url
|
||||
self._stop: Final = threading.Event()
|
||||
self._thread: Final = threading.Thread(target=self._run, name="postgres-connections", daemon=True)
|
||||
self._counts: Final[Recorder[int]] = Recorder()
|
||||
|
||||
@property
|
||||
def peak(self) -> int | None:
|
||||
return max((sample.value for sample in self._counts.series.samples), default=None)
|
||||
|
||||
@property
|
||||
def series(self) -> Series[int]:
|
||||
return self._counts.series
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=10)
|
||||
|
||||
def _run(self) -> None:
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(self._url, autocommit=True) as conn:
|
||||
while not self._stop.is_set():
|
||||
(count,) = _COMMITS_ROW.validate_python(conn.execute(_CONNECTIONS_QUERY).fetchone())
|
||||
self._counts.record(count)
|
||||
_ = self._stop.wait(_CONNECTIONS_SAMPLE_SECONDS)
|
||||
|
||||
|
||||
class NoConnectionsSampler:
|
||||
peak: None = None
|
||||
series: None = None
|
||||
|
||||
def start(self) -> None:
|
||||
return None
|
||||
|
||||
def stop(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connections_sampler() -> Generator[ConnectionsSampler | NoConnectionsSampler]:
|
||||
url: Final = os.environ.get("DATABASE_URL")
|
||||
sampler: Final = ConnectionsSampler(url) if url is not None else NoConnectionsSampler()
|
||||
sampler.start()
|
||||
try:
|
||||
yield sampler
|
||||
finally:
|
||||
sampler.stop()
|
||||
|
||||
|
||||
def calls_in(bucket: Bucket, items: tuple[StatementCalls, ...] | tuple[RedisCommand, ...] | None) -> int | None:
|
||||
if items is None:
|
||||
return None
|
||||
|
|
@ -166,7 +265,10 @@ def read_db_counters() -> DbCounters | None:
|
|||
return DbCounters(commits=commits, statements=None)
|
||||
return DbCounters(
|
||||
commits=commits,
|
||||
statements=tuple(StatementCalls(queryid=queryid, query=query, calls=calls) for queryid, calls, query in rows),
|
||||
statements=tuple(
|
||||
StatementCalls(queryid=queryid, query=query, calls=calls, total_exec_ms=total_ms)
|
||||
for queryid, calls, query, total_ms in rows
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
323
tests/e2e/perf/setup/event_loop.py
Normal file
323
tests/e2e/perf/setup/event_loop.py
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
"""Event-loop holds, read from the gateway's own asyncio debug output. A gateway started with
|
||||
`PYTHONASYNCIODEBUG=1` on the stdlib loop (`LITELLM_EVENT_LOOP=asyncio`, uvloop never reports them) logs every
|
||||
callback or task step over `slow_callback_duration` (0.1 s) as
|
||||
`Executing <Task ... coro=<name() running at file:line> ...> took X seconds`. A case with `event_loop` set reads
|
||||
the gateway log for its traffic window, groups those lines by coroutine and location, and fails on a location that
|
||||
is not on its allowlist of known holds or shows up more often than the allowlist gives it. asyncio names the task,
|
||||
not the frame that held the loop, so every request handler lands under `RequestResponseCycle.run_asgi`; the
|
||||
allowlist caps how often a known location may appear instead of ignoring it. The log comes from the gateway pods
|
||||
(`pods/log` with the runner's service account, pods picked by `E2E_PERF_GATEWAY_POD_SELECTOR`) or, on a local run,
|
||||
from the whole file named by `E2E_PERF_GATEWAY_LOG_FILE` (no time window, so startup holds are in it too).
|
||||
A garbage collection pass pauses the loop too, and asyncio charges it to whatever task was running, so the gateway
|
||||
logs every pass over the same threshold (`Garbage collection gen=2 collected=N took X seconds`, debug mode only);
|
||||
a hold that such a pass accounts for is reported under `garbage collection` instead of the task it landed on."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import asdict, dataclass, replace
|
||||
from datetime import datetime
|
||||
from itertools import groupby
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import quote
|
||||
|
||||
from e2e_http import AuthHeaders, NetworkError, Success, get_external, get_text_external
|
||||
from pydantic import BaseModel
|
||||
|
||||
from setup.podcpu import SELECTOR_ENV, in_cluster
|
||||
|
||||
OPT_IN_ENV: Final = "E2E_PERF_EVENT_LOOP"
|
||||
LOG_FILE_ENV: Final = "E2E_PERF_GATEWAY_LOG_FILE"
|
||||
_API: Final = "https://kubernetes.default.svc/api/v1/namespaces"
|
||||
_LINE: Final = re.compile(r"Executing <(?P<desc>.*?)> took (?P<seconds>[0-9.]+) seconds")
|
||||
_CORO: Final = re.compile(r"coro=<(?P<name>[^ ]+)")
|
||||
_AT: Final = re.compile(r" at (?P<where>[^ >]+:\d+)")
|
||||
_GC_LINE: Final = re.compile(
|
||||
r"Garbage collection gen=(?P<generation>\d+) collected=\d+ took (?P<seconds>[0-9.]+) seconds"
|
||||
)
|
||||
GC_BUCKET: Final = "garbage collection"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class KnownStall:
|
||||
"""A hold the gateway has today: `where` is a substring of the coroutine name or its file:line, `max_count` the
|
||||
most times it may show up in one run before the case fails."""
|
||||
|
||||
where: str
|
||||
max_count: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EventLoopStalls:
|
||||
allowlist: tuple[KnownStall, ...]
|
||||
max_hold_seconds: float = 0.1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GcPass:
|
||||
generation: int
|
||||
seconds: float
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Stall:
|
||||
coroutine: str
|
||||
where: str
|
||||
seconds: float
|
||||
gc_seconds: float = 0.0
|
||||
gc_generation: int | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StallRow:
|
||||
coroutine: str
|
||||
where: str
|
||||
count: int
|
||||
total_seconds: float
|
||||
max_seconds: float
|
||||
allowed_by: KnownStall | None
|
||||
"""Every hold in the row, longest first."""
|
||||
holds: tuple[float, ...] = ()
|
||||
|
||||
@property
|
||||
def offending(self) -> bool:
|
||||
return self.allowed_by is None or self.count > self.allowed_by.max_count
|
||||
|
||||
@property
|
||||
def over_allowance(self) -> int:
|
||||
return self.count if self.allowed_by is None else max(0, self.count - self.allowed_by.max_count)
|
||||
|
||||
@property
|
||||
def note(self) -> str:
|
||||
allowed: Final = (
|
||||
f"{self.count} of {self.allowed_by.max_count} allowed" if self.allowed_by else "not on the allowlist"
|
||||
)
|
||||
return f"{allowed}, max {self.max_seconds:.2f} s"
|
||||
|
||||
|
||||
def _short(path: str) -> str:
|
||||
if "site-packages/" in path:
|
||||
return path.rsplit("site-packages/", 1)[1]
|
||||
if "/litellm/" in path:
|
||||
return "litellm/" + path.rsplit("/litellm/", 1)[1]
|
||||
return path
|
||||
|
||||
|
||||
def _parse_line(line: str) -> Stall | None:
|
||||
match: Final = _LINE.search(line)
|
||||
if match is None:
|
||||
return None
|
||||
desc: Final = match.group("desc")
|
||||
coro: Final = _CORO.search(desc)
|
||||
at: Final = _AT.search(desc)
|
||||
return Stall(
|
||||
coroutine=coro.group("name") if coro else desc[:80],
|
||||
where=_short(at.group("where")) if at else "",
|
||||
seconds=float(match.group("seconds")),
|
||||
)
|
||||
|
||||
|
||||
def _parse_event(line: str) -> Stall | GcPass | None:
|
||||
match: Final = _GC_LINE.search(line)
|
||||
if match is None:
|
||||
return _parse_line(line)
|
||||
return GcPass(generation=int(match.group("generation")), seconds=float(match.group("seconds")))
|
||||
|
||||
|
||||
def _with_passes(stall: Stall, passes: tuple[GcPass, ...]) -> Stall:
|
||||
fitting: Final = tuple(gc_pass for gc_pass in passes if gc_pass.seconds <= stall.seconds)
|
||||
if not fitting:
|
||||
return stall
|
||||
longest: Final = max(fitting, key=lambda gc_pass: gc_pass.seconds)
|
||||
return replace(stall, gc_seconds=longest.seconds, gc_generation=longest.generation)
|
||||
|
||||
|
||||
def parse_stalls(lines: Iterable[str]) -> tuple[Stall, ...]:
|
||||
"""Holds in log order, each carrying the collector passes logged since the previous hold: a pass runs inside the
|
||||
task asyncio then blames and is only logged when it is itself over the threshold, so its hold line follows it."""
|
||||
events: Final = tuple(event for event in map(_parse_event, lines) if event is not None)
|
||||
holds: Final = tuple((index, event) for index, event in enumerate(events) if isinstance(event, Stall))
|
||||
return tuple(
|
||||
_with_passes(stall, tuple(event for event in events[previous + 1 : index] if isinstance(event, GcPass)))
|
||||
for previous, (index, stall) in zip((-1, *(index for index, _ in holds)), holds)
|
||||
)
|
||||
|
||||
|
||||
def _reported(spec: EventLoopStalls, stall: Stall) -> Stall:
|
||||
"""A hold the collector accounts for, what is left under the threshold, is the collector's, not the task's."""
|
||||
if stall.gc_generation is None:
|
||||
return stall
|
||||
if stall.seconds - stall.gc_seconds >= spec.max_hold_seconds:
|
||||
return stall
|
||||
return Stall(coroutine=GC_BUCKET, where=f"gen={stall.gc_generation}", seconds=stall.seconds)
|
||||
|
||||
|
||||
def _allowed_by(spec: EventLoopStalls, coroutine: str, where: str) -> KnownStall | None:
|
||||
return next((known for known in spec.allowlist if known.where in coroutine or known.where in where), None)
|
||||
|
||||
|
||||
def summarize(spec: EventLoopStalls, stalls: Iterable[Stall]) -> tuple[StallRow, ...]:
|
||||
"""One row per (coroutine, location) over the hold threshold, worst total first."""
|
||||
reported: Final = tuple(_reported(spec, stall) for stall in stalls)
|
||||
over: Final = sorted(
|
||||
(stall for stall in reported if stall.seconds >= spec.max_hold_seconds), key=lambda s: (s.coroutine, s.where)
|
||||
)
|
||||
rows: Final = tuple(
|
||||
StallRow(
|
||||
coroutine=coroutine,
|
||||
where=where,
|
||||
count=len(group),
|
||||
total_seconds=sum(stall.seconds for stall in group),
|
||||
max_seconds=max(stall.seconds for stall in group),
|
||||
allowed_by=_allowed_by(spec, coroutine, where),
|
||||
holds=tuple(sorted((stall.seconds for stall in group), reverse=True)),
|
||||
)
|
||||
for (coroutine, where), group in (
|
||||
(key, tuple(items)) for key, items in groupby(over, key=lambda s: (s.coroutine, s.where))
|
||||
)
|
||||
)
|
||||
return tuple(sorted(rows, key=lambda row: -row.total_seconds))
|
||||
|
||||
|
||||
def offenders(rows: Iterable[StallRow]) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
f"{row.coroutine} {row.where}: {row.count}x, max {row.max_seconds:.2f} s"
|
||||
+ (
|
||||
f" (allowlist {row.allowed_by.where} permits {row.allowed_by.max_count})"
|
||||
if row.allowed_by
|
||||
else " (not on the allowlist)"
|
||||
)
|
||||
for row in rows
|
||||
if row.offending
|
||||
)
|
||||
|
||||
|
||||
TOP_HOLDS: Final = 10
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Slice:
|
||||
"""One `_breakdown` row in the shape the other metrics use (label, value, share) plus the allowlist verdict the
|
||||
dashboard shows under the label and colours by. `share` is None where the rows are not parts of a total."""
|
||||
|
||||
label: str
|
||||
value: float
|
||||
share: float | None
|
||||
note: str | None
|
||||
over: bool
|
||||
|
||||
|
||||
def _label(row: StallRow) -> str:
|
||||
return f"{row.coroutine} {row.where}".strip()
|
||||
|
||||
|
||||
def _slice(row: StallRow, value: float, total: float) -> Slice:
|
||||
return Slice(label=_label(row), value=value, share=value / total, note=row.note, over=row.offending)
|
||||
|
||||
|
||||
def _breakdown(name: str, slices: Iterable[Slice]) -> tuple[tuple[str, str], ...]:
|
||||
rows: Final = tuple(asdict(row) for row in slices)
|
||||
return ((f"perf_event_loop_{name}_breakdown", json.dumps(rows)),) if rows else ()
|
||||
|
||||
|
||||
def _longest_holds(table: tuple[StallRow, ...]) -> tuple[Slice, ...]:
|
||||
holds: Final = sorted(((seconds, row) for row in table for seconds in row.holds), key=lambda hold: -hold[0])
|
||||
return tuple(
|
||||
Slice(label=_label(row), value=seconds, share=None, note=None, over=row.offending)
|
||||
for seconds, row in holds[:TOP_HOLDS]
|
||||
)
|
||||
|
||||
|
||||
def junit_properties(spec: EventLoopStalls, rows: Iterable[StallRow]) -> tuple[tuple[str, float | str], ...]:
|
||||
"""The stall table as `perf_event_loop_*` properties for the dashboard: every hold over the threshold with the
|
||||
allowlist's total as its limit, the holds past their allowance (limit 0, the case's one assertion), the longest
|
||||
and the summed hold, each with a `_breakdown` the dashboard opens on click: holds and seconds per location, the
|
||||
locations past their cap by how far over they are, and the ten longest single holds with where they landed."""
|
||||
table: Final = tuple(rows)
|
||||
holds: Final = sum(row.count for row in table)
|
||||
offending: Final = sum(row.over_allowance for row in table)
|
||||
total_seconds: Final = sum(row.total_seconds for row in table)
|
||||
return (
|
||||
("perf_event_loop_holds", holds),
|
||||
("perf_event_loop_holds_limit", sum(known.max_count for known in spec.allowlist)),
|
||||
("perf_event_loop_offending_holds", offending),
|
||||
("perf_event_loop_offending_holds_limit", 0),
|
||||
("perf_event_loop_hold_seconds_max", max((row.max_seconds for row in table), default=0.0)),
|
||||
("perf_event_loop_hold_seconds_total", total_seconds),
|
||||
*_breakdown("holds", (_slice(row, row.count, holds) for row in table)),
|
||||
*_breakdown(
|
||||
"offending_holds",
|
||||
(_slice(row, row.over_allowance, offending) for row in table if row.over_allowance),
|
||||
),
|
||||
*_breakdown("hold_seconds_total", (_slice(row, row.total_seconds, total_seconds) for row in table)),
|
||||
*_breakdown("hold_seconds_max", _longest_holds(table)),
|
||||
)
|
||||
|
||||
|
||||
def render_stalls(spec: EventLoopStalls, rows: Iterable[StallRow]) -> str:
|
||||
header: Final = (
|
||||
f" event-loop holds >= {spec.max_hold_seconds:.2f} s in the gateway log (count total_s max_s where)"
|
||||
)
|
||||
body: Final = tuple(
|
||||
f" {'FAIL' if row.offending else 'ok '} {row.count:5d} {row.total_seconds:8.2f} {row.max_seconds:6.2f} "
|
||||
f"{row.coroutine} {row.where}"
|
||||
for row in rows
|
||||
)
|
||||
return "\n".join((header, *body)) if body else header + "\n none"
|
||||
|
||||
|
||||
class _PodName(BaseModel):
|
||||
name: str
|
||||
|
||||
|
||||
class _Pod(BaseModel):
|
||||
metadata: _PodName
|
||||
|
||||
|
||||
class _PodList(BaseModel):
|
||||
items: list[_Pod]
|
||||
|
||||
|
||||
def _pod_log_lines(url: str, headers: AuthHeaders, ca_bundle: str) -> tuple[str, ...]:
|
||||
text: Final = get_text_external(url, headers=headers, timeout=60.0, ca_bundle=ca_bundle)
|
||||
if isinstance(text, NetworkError):
|
||||
raise RuntimeError(f"gateway pod log unreadable: {text.message}")
|
||||
return tuple(text.splitlines())
|
||||
|
||||
|
||||
def _pod_logs(since: datetime) -> tuple[str, ...] | None:
|
||||
cluster: Final = in_cluster()
|
||||
selector: Final = os.environ.get(SELECTOR_ENV, "")
|
||||
if cluster is None or not selector:
|
||||
return None
|
||||
token, ca_bundle, namespace = cluster
|
||||
headers: Final = AuthHeaders(authorization=f"Bearer {token}")
|
||||
pods: Final = get_external(
|
||||
f"{_API}/{namespace}/pods?labelSelector={quote(selector)}",
|
||||
response_type=_PodList,
|
||||
headers=headers,
|
||||
timeout=60.0,
|
||||
ca_bundle=ca_bundle,
|
||||
)
|
||||
if not isinstance(pods, Success):
|
||||
raise RuntimeError(f"gateway pod list unreadable: {pods}")
|
||||
since_param: Final = quote(since.strftime("%Y-%m-%dT%H:%M:%SZ"))
|
||||
return tuple(
|
||||
line
|
||||
for pod in pods.data.items
|
||||
for line in _pod_log_lines(
|
||||
f"{_API}/{namespace}/pods/{pod.metadata.name}/log?sinceTime={since_param}", headers, ca_bundle
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def read_gateway_log(since: datetime) -> tuple[str, ...] | None:
|
||||
"""Gateway log lines for the traffic window, or None when neither a pod nor a local file is reachable."""
|
||||
local: Final = os.environ.get(LOG_FILE_ENV, "")
|
||||
if local:
|
||||
return tuple(Path(local).read_text(errors="replace").splitlines())
|
||||
return _pod_logs(since)
|
||||
|
|
@ -4,7 +4,8 @@ time to headers, time to first byte, time to first token (streams), total time,
|
|||
and whether it matched. Runs in its own process (`setup/run.py` starts it), never in the gateway's.
|
||||
|
||||
Reads `PERF_PLAN_FILE` (JSON list of wire requests), writes `PERF_RECORDS_FILE` (one JSON object per
|
||||
line), paces to `PERF_RPS` over `PERF_USERS` users, and quits once the last request is recorded.
|
||||
line), paces to `PERF_RPS` over `PERF_USERS` users (or, when the plan carries `send_at_s`, sends each
|
||||
request at its scheduled second), and quits once the last request is recorded.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -19,9 +20,10 @@ from typing import Final, cast
|
|||
import gevent
|
||||
from locust import FastHttpUser, task
|
||||
from locust.contrib.fasthttp import FastResponse, ResponseContextManager
|
||||
from setup.wire import Endpoint, WireRecord, WireRequest, request_body
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from setup.wire import Endpoint, PlanProgress, WireRecord, WireRequest, request_body, response_id
|
||||
|
||||
_PLAN_ADAPTER: Final = TypeAdapter(list[WireRequest])
|
||||
_JSON: Final = TypeAdapter[object](object)
|
||||
_PLAN: Final = _PLAN_ADAPTER.validate_json(Path(os.environ["PERF_PLAN_FILE"]).read_bytes())
|
||||
|
|
@ -33,7 +35,10 @@ _INTERVAL_S: Final = _USERS / _RPS
|
|||
_JITTER: Final = 0.25
|
||||
_RNG: Final = random.Random(_PLAN[0].marker if _PLAN else "empty-plan")
|
||||
_RECORDS: Final = _RECORDS_PATH.open("a", encoding="utf-8")
|
||||
_written = 0
|
||||
_PROGRESS: Final = PlanProgress(len(_PLAN))
|
||||
_SCHEDULED: Final = bool(_PLAN) and _PLAN[0].send_at_s is not None
|
||||
_RUN_STARTED: Final = time.monotonic()
|
||||
_STOP_FILE: Final = os.environ.get("PERF_STOP_FILE")
|
||||
BLOCK_REASON: Final = "perf-test-block"
|
||||
_TOKEN_MAX_AGE_S: Final = 240.0
|
||||
_tokens: dict[str, tuple[str, float]] = {}
|
||||
|
|
@ -135,6 +140,15 @@ def _header(response: FastResponse, name: str) -> str | None:
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _ms_header(response: FastResponse | None, name: str) -> float | None:
|
||||
"""A gateway timing header as a number; the gateway writes the literal `None` when it has no value."""
|
||||
value: Final = _header(response, name) if response is not None else None
|
||||
try:
|
||||
return float(value) if value is not None else None
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _quit(user: FastHttpUser) -> None:
|
||||
"""End the run once every record is written. Quitting from a separate greenlet: `runner.quit`
|
||||
kills the user greenlets, and the user calling it would be killed mid-shutdown."""
|
||||
|
|
@ -155,10 +169,9 @@ def _judge(item: WireRequest, status: int, body_text: str) -> tuple[bool, str |
|
|||
|
||||
|
||||
def _record(record: WireRecord) -> None:
|
||||
global _written
|
||||
_ = _RECORDS.write(record.model_dump_json() + "\n")
|
||||
_RECORDS.flush()
|
||||
_written += 1
|
||||
_PROGRESS.completed += 1
|
||||
|
||||
|
||||
def _authorization(user: FastHttpUser, item: WireRequest) -> str:
|
||||
|
|
@ -201,17 +214,21 @@ def _authorization(user: FastHttpUser, item: WireRequest) -> str:
|
|||
class PlanUser(FastHttpUser):
|
||||
"""One simulated client; all users share the plan and together hold the planned rate. Each user starts at
|
||||
a random point of its first interval and jitters every later interval by up to `_JITTER` either way, so the
|
||||
users never fire in lockstep; the mean interval is unchanged and `rps` stays the total rate."""
|
||||
users never fire in lockstep; the mean interval is unchanged and `rps` stays the total rate. A scheduled
|
||||
plan (a ramp) is paced by the plan instead: a user takes the next request and sleeps until its second."""
|
||||
|
||||
network_timeout = 120.0
|
||||
network_timeout = float(os.environ.get("PERF_REQUEST_TIMEOUT", "120"))
|
||||
connection_timeout = 30.0
|
||||
|
||||
def on_start(self) -> None:
|
||||
self._last_run = 0.0
|
||||
self._last_wait = 0.0
|
||||
gevent.sleep(_RNG.uniform(0.0, _INTERVAL_S))
|
||||
if not _SCHEDULED:
|
||||
gevent.sleep(_RNG.uniform(0.0, _INTERVAL_S))
|
||||
|
||||
def wait_time(self) -> float:
|
||||
if _SCHEDULED:
|
||||
return 0.1
|
||||
target: Final = _INTERVAL_S * _RNG.uniform(1.0 - _JITTER, 1.0 + _JITTER)
|
||||
now: Final = time.monotonic()
|
||||
run_s: Final = now - self._last_run - self._last_wait if self._last_run else 0.0
|
||||
|
|
@ -221,11 +238,14 @@ class PlanUser(FastHttpUser):
|
|||
|
||||
@task
|
||||
def send(self) -> None:
|
||||
item: Final = next(_NEXT, None)
|
||||
take: Final = _PROGRESS.take(stop=_STOP_FILE is not None and Path(_STOP_FILE).exists())
|
||||
item: Final = next(_NEXT, None) if take else None
|
||||
if item is None:
|
||||
if _written >= len(_PLAN):
|
||||
if _PROGRESS.drained:
|
||||
_quit(self)
|
||||
return
|
||||
if item.send_at_s is not None:
|
||||
gevent.sleep(max(0.0, _RUN_STARTED + item.send_at_s - time.monotonic()))
|
||||
started_at: Final = time.time()
|
||||
started: Final = time.perf_counter()
|
||||
first_byte_s: float | None = None
|
||||
|
|
@ -237,6 +257,7 @@ class PlanUser(FastHttpUser):
|
|||
error: str | None = None
|
||||
header_s = 0.0
|
||||
response: FastResponse | None = None
|
||||
identifier: str | None = None # rebind-ok: the ID arrives in a JSON body or the first SSE event
|
||||
try:
|
||||
headers: Final = {"Authorization": _authorization(self, item), "Content-Type": "application/json"}
|
||||
context: ResponseContextManager
|
||||
|
|
@ -265,6 +286,7 @@ class PlanUser(FastHttpUser):
|
|||
buffer += line
|
||||
events, buffer = _events(buffer)
|
||||
for event in events:
|
||||
identifier = identifier or response_id(event)
|
||||
if first_token_s is None and token_in_event(item.endpoint, event):
|
||||
first_token_s = now
|
||||
text += delta_of_event(item.endpoint, event)
|
||||
|
|
@ -272,7 +294,9 @@ class PlanUser(FastHttpUser):
|
|||
body_text = context.text or ""
|
||||
first_byte_s = time.perf_counter() - started
|
||||
if status == 200:
|
||||
text = text_of_json(item.endpoint, _as_dict(_JSON.validate_json(body_text)))
|
||||
payload: Final = _as_dict(_JSON.validate_json(body_text))
|
||||
identifier = response_id(payload)
|
||||
text = text_of_json(item.endpoint, payload)
|
||||
ok, error = _judge(item, status, body_text)
|
||||
if ok:
|
||||
context.success()
|
||||
|
|
@ -302,7 +326,10 @@ class PlanUser(FastHttpUser):
|
|||
mock_delay_s=float(delay_header) if delay_header else None,
|
||||
call_id=_header(response, "x-litellm-call-id") if response is not None else None,
|
||||
error=error,
|
||||
response_id=identifier,
|
||||
litellm_overhead_ms=_ms_header(response, "x-litellm-overhead-duration-ms"),
|
||||
litellm_response_ms=_ms_header(response, "x-litellm-response-duration-ms"),
|
||||
)
|
||||
)
|
||||
if _written >= len(_PLAN):
|
||||
if _PROGRESS.drained:
|
||||
_quit(self)
|
||||
|
|
|
|||
|
|
@ -4,27 +4,35 @@ calls, the workers' memory) into the case's metrics, and the case's limits into
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import statistics
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import asdict, dataclass
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from e2e_http import NoBody, get_external, post_external, unwrap
|
||||
from memory_readings import RssCapture, RssReading, WorkerKey
|
||||
from pydantic import BaseModel
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from setup.datastore import (
|
||||
DbCounters,
|
||||
RedisCommand,
|
||||
RedisCounters,
|
||||
StatementCalls,
|
||||
active_sessions,
|
||||
calls_in,
|
||||
slowest_statement,
|
||||
statement_deltas,
|
||||
)
|
||||
from setup.mocks.reply_common import Provider
|
||||
from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase
|
||||
from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase, Ramp
|
||||
from setup.ramp import StepReading, render_steps, sustained_rps
|
||||
from setup.series import Series
|
||||
from setup.telemetry import GatewayTelemetry
|
||||
from setup.wire import WireRecord
|
||||
|
||||
|
||||
|
|
@ -115,6 +123,20 @@ class Observation:
|
|||
redis_after: RedisCounters | None = None
|
||||
redis_commands: tuple[RedisCommand, ...] | None = None
|
||||
gateway_cpu_seconds: float | None = None
|
||||
telemetry_before: GatewayTelemetry | None = None
|
||||
telemetry_after: GatewayTelemetry | None = None
|
||||
peak_connections: int | None = None
|
||||
ramp: Ramp | None = None
|
||||
cpu_series: Series[float] | None = None
|
||||
connection_series: Series[int] | None = None
|
||||
telemetry_series: Series[GatewayTelemetry] | None = None
|
||||
|
||||
@property
|
||||
def telemetry(self) -> GatewayTelemetry | None:
|
||||
"""What the gateway's `/metrics` counted during the run."""
|
||||
if self.telemetry_before is None or self.telemetry_after is None:
|
||||
return None
|
||||
return self.telemetry_after.minus(self.telemetry_before)
|
||||
|
||||
def per_request(self, before: int | None, after: int | None) -> float | None:
|
||||
if before is None or after is None or not self.records:
|
||||
|
|
@ -128,6 +150,21 @@ class Observation:
|
|||
def statements(self) -> tuple[StatementCalls, ...] | None:
|
||||
return statement_deltas(self.db_before, self.db_after)
|
||||
|
||||
@property
|
||||
def busiest_awaited_statement(self) -> StatementCalls | None:
|
||||
"""The awaited statement with the most calls: a lookup added to every request shows up here at
|
||||
1.0 per request while cache refreshes stay fractional, which the awaited total cannot tell apart."""
|
||||
awaited: Final = tuple(item for item in self.statements or () if item.bucket == "awaited")
|
||||
return max(awaited, key=_item_count, default=None)
|
||||
|
||||
def busiest_background_statement(self, exempt_tables: frozenset[str]) -> StatementCalls | None:
|
||||
"""The background statement with the most calls, skipping the case's per-flush exemptions: a rollup
|
||||
written once per request reads 1.0 here while batched writers stay below it."""
|
||||
background: Final = tuple(
|
||||
item for item in self.statements or () if item.bucket == "background" and item.table not in exempt_tables
|
||||
)
|
||||
return max(background, key=_item_count, default=None)
|
||||
|
||||
@property
|
||||
def retried_markers(self) -> frozenset[str]:
|
||||
counts: Final = Counter(seen.marker for seen in self.mock_requests if seen.marker is not None)
|
||||
|
|
@ -156,21 +193,42 @@ class Observation:
|
|||
"""Requests the plan expected to succeed that came back with any other status (0 is a client timeout)."""
|
||||
return sum(1 for record in self.records if not record.blocked and record.status != 200)
|
||||
|
||||
def latency_pairs(self, metric: LatencyMetric) -> list[tuple[WireRecord, float]]:
|
||||
"""Each first-attempt request with its value of one latency metric. `gateway_added_seconds` is the
|
||||
seconds the gateway added (total time minus the mock's own delay for a non-streamed request, time to the
|
||||
first token minus the mock's delay for a stream), `time_to_first_token_seconds` is the client's wait for
|
||||
the first token with the mock delay included, and the two `litellm_*` metrics are the gateway's own
|
||||
`x-litellm-overhead-duration-ms` and `x-litellm-response-duration-ms` headers in seconds."""
|
||||
if metric == "gateway_added_seconds":
|
||||
non_stream: Final = [(record, record.total_s - delay) for record, delay in self._clean(stream=False)]
|
||||
stream: Final = [
|
||||
(record, record.first_token_s - delay)
|
||||
for record, delay in self._clean(stream=True)
|
||||
if record.first_token_s is not None
|
||||
]
|
||||
return non_stream + stream
|
||||
if metric == "time_to_first_token_seconds":
|
||||
return [(r, r.first_token_s) for r, _ in self._clean(stream=True) if r.first_token_s is not None]
|
||||
if metric == "litellm_overhead_seconds":
|
||||
return [
|
||||
(r, r.litellm_overhead_ms / 1000) for r in self._first_attempts() if r.litellm_overhead_ms is not None
|
||||
]
|
||||
return [(r, r.litellm_response_ms / 1000) for r in self._first_attempts() if r.litellm_response_ms is not None]
|
||||
|
||||
def _first_attempts(self) -> list[WireRecord]:
|
||||
return [record for record, _ in self._clean(stream=False)] + [record for record, _ in self._clean(stream=True)]
|
||||
|
||||
def gateway_added_seconds(self) -> list[float]:
|
||||
"""Seconds the gateway added to each first-attempt request: total time minus the mock's own delay for a
|
||||
non-streamed request, time to the first token minus the mock's delay to its first byte for a stream."""
|
||||
non_stream: Final = [record.total_s - delay for record, delay in self._clean(stream=False)]
|
||||
stream: Final = [
|
||||
record.first_token_s - delay
|
||||
for record, delay in self._clean(stream=True)
|
||||
if record.first_token_s is not None
|
||||
]
|
||||
return non_stream + stream
|
||||
return [value for _, value in self.latency_pairs("gateway_added_seconds")]
|
||||
|
||||
def litellm_overhead_seconds(self) -> list[float]:
|
||||
return [value for _, value in self.latency_pairs("litellm_overhead_seconds")]
|
||||
|
||||
def litellm_response_seconds(self) -> list[float]:
|
||||
return [value for _, value in self.latency_pairs("litellm_response_seconds")]
|
||||
|
||||
def time_to_first_token_seconds(self) -> list[float]:
|
||||
"""Seconds from the client sending each first-attempt streamed request to its first token, mock delay
|
||||
included, comparable across versions because the mocks generate the same replies after the same wait."""
|
||||
return [record.first_token_s for record, _ in self._clean(stream=True) if record.first_token_s is not None]
|
||||
return [value for _, value in self.latency_pairs("time_to_first_token_seconds")]
|
||||
|
||||
def bodies_off_reply(self) -> int:
|
||||
"""Served requests whose text is not the reply the mock generated for them (plus the post-call rewrite
|
||||
|
|
@ -189,26 +247,236 @@ class Observation:
|
|||
post: Final = Counter(call.marker for call in self.guardrail_calls if call.input_type == "response")
|
||||
return sum(1 for record in self.served if pre[record.marker] != 1 or post[record.marker] != 1)
|
||||
|
||||
def steps(self) -> tuple[StepReading, ...]:
|
||||
"""One reading per step of a ramped run: the gateway's added latency over the step's requests, the rate
|
||||
at which requests completed inside the step's window, the step's unexpected statuses, and what the pod
|
||||
was doing in that window (CPU per completed request, Redis latency, Postgres connections)."""
|
||||
ramp: Final = self.ramp
|
||||
if ramp is None or not self.records:
|
||||
return ()
|
||||
started: Final = min(record.started_at for record in self.records)
|
||||
added: Final = _by_step(ramp, self.latency_pairs("gateway_added_seconds"))
|
||||
first_token: Final = _by_step(ramp, self.latency_pairs("time_to_first_token_seconds"))
|
||||
completed: Final = Counter(
|
||||
int((record.started_at + record.total_s - started) // ramp.step_seconds) for record in self.records
|
||||
)
|
||||
unexpected: Final = Counter(
|
||||
ramp.step_of(record.seq) for record in self.records if not record.blocked and record.status != 200
|
||||
)
|
||||
return tuple(
|
||||
StepReading(
|
||||
rps=rate,
|
||||
planned=count,
|
||||
completed_rps=completed[index] / ramp.step_seconds,
|
||||
gateway_added_p99=percentile(added.get(index, ()), 99),
|
||||
unexpected=unexpected[index],
|
||||
held_p99_seconds=ramp.held_p99_seconds,
|
||||
gateway_added_p50=percentile(added.get(index, ()), 50),
|
||||
time_to_first_token_p99=percentile(first_token.get(index, ()), 99),
|
||||
**self._window_readings(started + index * ramp.step_seconds, ramp.step_seconds, completed[index]),
|
||||
)
|
||||
for index, (rate, count) in enumerate(zip(ramp.rates, ramp.counts, strict=True))
|
||||
)
|
||||
|
||||
def _window_readings(self, start: float, seconds: float, completed: int) -> WindowReadings:
|
||||
end: Final = start + seconds
|
||||
cpu: Final = None if self.cpu_series is None else sum(self.cpu_series.between(start, end))
|
||||
conns: Final = None if self.connection_series is None else self.connection_series.between(start, end)
|
||||
before: Final = None if self.telemetry_series is None else self.telemetry_series.last_before(start)
|
||||
after: Final = None if self.telemetry_series is None else self.telemetry_series.last_before(end)
|
||||
return WindowReadings(
|
||||
cpu_seconds_per_request=None if cpu is None or completed == 0 else cpu / completed,
|
||||
redis_latency_mean_seconds=None
|
||||
if before is None or after is None
|
||||
else after.minus(before).redis_latency.mean,
|
||||
postgres_peak_connections=max(conns, default=None) if conns else None,
|
||||
)
|
||||
|
||||
|
||||
class WindowReadings(TypedDict):
|
||||
cpu_seconds_per_request: ReadOnly[float | None]
|
||||
redis_latency_mean_seconds: ReadOnly[float | None]
|
||||
postgres_peak_connections: ReadOnly[int | None]
|
||||
|
||||
|
||||
def _by_step(ramp: Ramp, pairs: Iterable[tuple[WireRecord, float]]) -> Mapping[int, tuple[float, ...]]:
|
||||
ordered: Final = sorted(pairs, key=lambda pair: pair[0].seq)
|
||||
return MappingProxyType(
|
||||
{
|
||||
step: tuple(value for _, value in group)
|
||||
for step, group in groupby(ordered, key=lambda pair: ramp.step_of(pair[0].seq))
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _text_sha1(text: str, *, intervened: bool) -> str:
|
||||
original: Final = text.removesuffix(" [post]") if intervened else text
|
||||
return hashlib.sha1(original.encode()).hexdigest()
|
||||
|
||||
|
||||
def observe(metric: Metric, percentile_rank: int | None, observation: Observation) -> float | None:
|
||||
LatencyMetric = Literal[
|
||||
"gateway_added_seconds", "time_to_first_token_seconds", "litellm_overhead_seconds", "litellm_response_seconds"
|
||||
]
|
||||
TOP_CONTRIBUTORS: Final = 5
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Contributor:
|
||||
"""One slice of a metric: a statement, a Redis command, an endpoint, a worker or a status, with its own
|
||||
value in the metric's unit and, where the metric is a total, its share of that total."""
|
||||
|
||||
label: str
|
||||
value: float
|
||||
share: float | None = None
|
||||
|
||||
|
||||
def _top(items: Iterable[Contributor]) -> tuple[Contributor, ...]:
|
||||
return tuple(sorted(items, key=lambda item: item.value, reverse=True)[:TOP_CONTRIBUTORS])
|
||||
|
||||
|
||||
def _endpoint_label(record: WireRecord) -> str:
|
||||
return f"{record.endpoint}{' stream' if record.stream else ''}"
|
||||
|
||||
|
||||
def _latency_breakdown(metric: LatencyMetric, rank: int, observation: Observation) -> tuple[Contributor, ...]:
|
||||
pairs: Final = observation.latency_pairs(metric)
|
||||
by_endpoint: dict[str, list[float]] = defaultdict(list)
|
||||
for record, value in pairs:
|
||||
by_endpoint[_endpoint_label(record)].append(value)
|
||||
return _top(
|
||||
Contributor(label, percentile(values, rank) or 0.0, len(values) / len(pairs))
|
||||
for label, values in by_endpoint.items()
|
||||
)
|
||||
|
||||
|
||||
def _statements_for(
|
||||
metric: Metric, statements: tuple[StatementCalls, ...], exempt_tables: frozenset[str]
|
||||
) -> tuple[StatementCalls, ...]:
|
||||
if metric in ("postgres_calls_per_request", "postgres_max_statement_per_request"):
|
||||
return tuple(item for item in statements if item.bucket == "awaited")
|
||||
if metric == "postgres_background_calls_per_request":
|
||||
return tuple(item for item in statements if item.bucket == "background")
|
||||
if metric == "postgres_max_background_statement_per_request":
|
||||
return tuple(item for item in statements if item.bucket == "background" and item.table not in exempt_tables)
|
||||
return tuple(item for item in statements if item.calls > 0)
|
||||
|
||||
|
||||
def _statement_breakdown(
|
||||
metric: Metric, observation: Observation, exempt_tables: frozenset[str]
|
||||
) -> tuple[Contributor, ...]:
|
||||
chosen: Final = _statements_for(metric, observation.statements or (), exempt_tables)
|
||||
requests: Final = max(1, len(observation.records))
|
||||
total_calls: Final = sum(item.calls for item in chosen)
|
||||
total_ms: Final = sum(item.total_exec_ms for item in chosen)
|
||||
if metric == "postgres_slowest_statement_seconds":
|
||||
return _top(
|
||||
Contributor(
|
||||
item.summary, item.mean_exec_seconds or 0.0, item.total_exec_ms / total_ms if total_ms else None
|
||||
)
|
||||
for item in chosen
|
||||
)
|
||||
if metric == "postgres_active_sessions":
|
||||
return _top(
|
||||
Contributor(
|
||||
item.summary, item.total_exec_ms / 1000 / observation.wall_seconds, item.total_exec_ms / total_ms
|
||||
)
|
||||
for item in chosen
|
||||
if total_ms
|
||||
)
|
||||
return _top(
|
||||
Contributor(item.summary, item.calls / requests, item.calls / total_calls) for item in chosen if total_calls
|
||||
)
|
||||
|
||||
|
||||
def _redis_breakdown(metric: Metric, observation: Observation) -> tuple[Contributor, ...]:
|
||||
bucket: Final = "awaited" if metric == "redis_calls_per_request" else "background"
|
||||
chosen: Final = tuple(item for item in observation.redis_commands or () if item.bucket == bucket)
|
||||
requests: Final = max(1, len(observation.records))
|
||||
total: Final = sum(item.count for item in chosen)
|
||||
return _top(
|
||||
Contributor(f"{item.command} {item.key}".strip(), item.count / requests, item.count / total)
|
||||
for item in chosen
|
||||
if total
|
||||
)
|
||||
|
||||
|
||||
def _memory_breakdown(observation: Observation) -> tuple[Contributor, ...]:
|
||||
before: Final = {reading.worker: reading.ram_usage_mb for reading in observation.rss_before.readings}
|
||||
return _top(
|
||||
Contributor(f"{reading.replica} pid {reading.worker_pid}", reading.ram_usage_mb - before[reading.worker])
|
||||
for reading in observation.rss_after.readings
|
||||
if reading.worker in before
|
||||
)
|
||||
|
||||
|
||||
def _status_breakdown(observation: Observation) -> tuple[Contributor, ...]:
|
||||
unexpected: Final = [r for r in observation.records if not r.blocked and r.status != 200]
|
||||
counts: Final = Counter(f"{_endpoint_label(record)} status {record.status}" for record in unexpected)
|
||||
return _top(Contributor(label, float(count), count / len(unexpected)) for label, count in counts.items())
|
||||
|
||||
|
||||
def breakdown(
|
||||
metric: Metric, percentile_rank: int | None, observation: Observation, exempt_tables: frozenset[str] = frozenset()
|
||||
) -> tuple[Contributor, ...]:
|
||||
"""The top contributors to one metric, so a spike on the chart names its statement, command, endpoint or
|
||||
worker. Gauges with no slices (`cpu_seconds_per_request`, `prometheus_redis_latency_mean_seconds`,
|
||||
`postgres_peak_connections`) have none."""
|
||||
if metric in (
|
||||
"gateway_added_seconds",
|
||||
"time_to_first_token_seconds",
|
||||
"litellm_overhead_seconds",
|
||||
"litellm_response_seconds",
|
||||
):
|
||||
return _latency_breakdown(metric, 50 if percentile_rank is None else percentile_rank, observation)
|
||||
if metric.startswith("postgres_") and metric != "postgres_peak_connections":
|
||||
return _statement_breakdown(metric, observation, exempt_tables)
|
||||
if metric in {"redis_calls_per_request", "redis_background_calls_per_request"}:
|
||||
return _redis_breakdown(metric, observation)
|
||||
if metric == "memory_growth_megabytes":
|
||||
return _memory_breakdown(observation)
|
||||
if metric == "unexpected_status_count":
|
||||
return _status_breakdown(observation)
|
||||
return ()
|
||||
|
||||
|
||||
def observe(
|
||||
metric: Metric, percentile_rank: int | None, observation: Observation, exempt_tables: frozenset[str] = frozenset()
|
||||
) -> float | None:
|
||||
"""The observed value of one metric, at a percentile when the metric is per request."""
|
||||
per_request: dict[str, list[float]] = {
|
||||
"gateway_added_seconds": observation.gateway_added_seconds(),
|
||||
"time_to_first_token_seconds": observation.time_to_first_token_seconds(),
|
||||
"litellm_overhead_seconds": observation.litellm_overhead_seconds(),
|
||||
"litellm_response_seconds": observation.litellm_response_seconds(),
|
||||
}
|
||||
if metric in per_request:
|
||||
return percentile(per_request[metric], 50 if percentile_rank is None else percentile_rank)
|
||||
if metric == "unexpected_status_count":
|
||||
return float(observation.unexpected_status_count())
|
||||
if metric == "prometheus_redis_latency_mean_seconds":
|
||||
counted: Final = observation.telemetry
|
||||
return None if counted is None else counted.redis_latency.mean
|
||||
if metric == "postgres_active_sessions":
|
||||
return active_sessions(observation.statements, observation.wall_seconds)
|
||||
if metric == "postgres_slowest_statement_seconds":
|
||||
slowest: Final = slowest_statement(observation.statements)
|
||||
return None if slowest is None else slowest.mean_exec_seconds
|
||||
if metric == "postgres_peak_connections":
|
||||
return None if observation.peak_connections is None else float(observation.peak_connections)
|
||||
if metric == "postgres_calls_per_request":
|
||||
return observation.count_per_request(calls_in("awaited", observation.statements))
|
||||
if metric == "postgres_background_calls_per_request":
|
||||
return observation.count_per_request(calls_in("background", observation.statements))
|
||||
if metric == "postgres_max_statement_per_request":
|
||||
if observation.statements is None:
|
||||
return None
|
||||
busiest: Final = observation.busiest_awaited_statement
|
||||
return observation.count_per_request(0 if busiest is None else busiest.calls)
|
||||
if metric == "postgres_max_background_statement_per_request":
|
||||
if observation.statements is None:
|
||||
return None
|
||||
busiest_background: Final = observation.busiest_background_statement(exempt_tables)
|
||||
return observation.count_per_request(0 if busiest_background is None else busiest_background.calls)
|
||||
if metric == "redis_calls_per_request":
|
||||
return observation.count_per_request(calls_in("awaited", observation.redis_commands))
|
||||
if metric == "redis_background_calls_per_request":
|
||||
|
|
@ -216,6 +484,8 @@ def observe(metric: Metric, percentile_rank: int | None, observation: Observatio
|
|||
if metric == "cpu_seconds_per_request":
|
||||
cpu: Final = observation.gateway_cpu_seconds
|
||||
return None if cpu is None or not observation.records else cpu / len(observation.records)
|
||||
if metric == "sustained_rps":
|
||||
return None if observation.ramp is None else sustained_rps(observation.steps())
|
||||
return worst_worker_growth_mb(observation.rss_before, observation.rss_after)
|
||||
|
||||
|
||||
|
|
@ -225,6 +495,9 @@ class Row:
|
|||
percentile: int | None
|
||||
observed: float | None
|
||||
limit: float | None
|
||||
floor: float | None = None
|
||||
detail: str = ""
|
||||
breakdown: tuple[Contributor, ...] = ()
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
|
|
@ -236,22 +509,43 @@ class Row:
|
|||
|
||||
@property
|
||||
def over(self) -> bool:
|
||||
return self.limit is not None and self.observed is not None and self.observed > self.limit
|
||||
"""Past the limit: above a max, or below a floor."""
|
||||
if self.observed is None:
|
||||
return False
|
||||
return (self.limit is not None and self.observed > self.limit) or (
|
||||
self.floor is not None and self.observed < self.floor
|
||||
)
|
||||
|
||||
@property
|
||||
def missing(self) -> bool:
|
||||
return self.limit is not None and self.observed is None
|
||||
return (self.limit is not None or self.floor is not None) and self.observed is None
|
||||
|
||||
@property
|
||||
def bound(self) -> str:
|
||||
if self.limit is not None:
|
||||
return f"{self.limit:.4f}"
|
||||
return "" if self.floor is None else f">= {self.floor:.4f}"
|
||||
|
||||
|
||||
REPORTED: Final[tuple[tuple[Metric, int | None], ...]] = (
|
||||
("unexpected_status_count", None),
|
||||
("gateway_added_seconds", 50),
|
||||
("gateway_added_seconds", 99),
|
||||
("litellm_overhead_seconds", 50),
|
||||
("litellm_overhead_seconds", 99),
|
||||
("litellm_response_seconds", 50),
|
||||
("litellm_response_seconds", 99),
|
||||
("prometheus_redis_latency_mean_seconds", None),
|
||||
("postgres_active_sessions", None),
|
||||
("postgres_slowest_statement_seconds", None),
|
||||
("postgres_peak_connections", None),
|
||||
("time_to_first_token_seconds", 50),
|
||||
("time_to_first_token_seconds", 99),
|
||||
("memory_growth_megabytes", None),
|
||||
("postgres_calls_per_request", None),
|
||||
("postgres_background_calls_per_request", None),
|
||||
("postgres_max_statement_per_request", None),
|
||||
("postgres_max_background_statement_per_request", None),
|
||||
("redis_calls_per_request", None),
|
||||
("redis_background_calls_per_request", None),
|
||||
("cpu_seconds_per_request", None),
|
||||
|
|
@ -265,6 +559,8 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[
|
|||
(limit.metric, limit.percentile): limit for limit in case.limits
|
||||
}
|
||||
keys: list[tuple[Metric, int | None]] = list(REPORTED)
|
||||
if observation.ramp is not None:
|
||||
keys.append(("sustained_rps", None))
|
||||
keys.extend(key for key in limits if key not in keys)
|
||||
rows: list[Row] = []
|
||||
for metric, rank in keys:
|
||||
|
|
@ -273,27 +569,53 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[
|
|||
Row(
|
||||
metric=metric,
|
||||
percentile=rank,
|
||||
observed=observe(metric, rank, observation),
|
||||
observed=observe(metric, rank, observation, case.background_cap_exempt_tables),
|
||||
limit=limit.max if limit else None,
|
||||
floor=limit.min if limit else None,
|
||||
detail=_detail(metric, observation, case.background_cap_exempt_tables),
|
||||
breakdown=breakdown(metric, rank, observation, case.background_cap_exempt_tables),
|
||||
)
|
||||
)
|
||||
exceeded: Final = [
|
||||
f"{row.label}: observed {row.observed:.4f} > limit {row.limit:.4f}"
|
||||
if row.over and row.observed is not None and row.limit is not None
|
||||
else f"{row.label}: limit set but nothing observed"
|
||||
for row in rows
|
||||
if row.over or row.missing
|
||||
]
|
||||
exceeded: Final = [_breach(row) for row in rows if row.over or row.missing]
|
||||
return rows, exceeded
|
||||
|
||||
|
||||
def _row_properties(row: Row) -> tuple[tuple[str, float], ...]:
|
||||
def _breach(row: Row) -> str:
|
||||
if row.observed is None:
|
||||
return f"{row.label}: limit set but nothing observed"
|
||||
if row.floor is not None:
|
||||
return f"{row.label}: observed {row.observed:.4f} < floor {row.floor:.4f}"
|
||||
return f"{row.label}: observed {row.observed:.4f} > limit {row.bound}{f' ({row.detail})' if row.detail else ''}"
|
||||
|
||||
|
||||
def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[str]) -> str:
|
||||
if metric == "postgres_max_statement_per_request":
|
||||
busiest: Final = observation.busiest_awaited_statement
|
||||
return "" if busiest is None else busiest.summary
|
||||
if metric == "postgres_max_background_statement_per_request":
|
||||
busiest_background: Final = observation.busiest_background_statement(exempt_tables)
|
||||
return "" if busiest_background is None else busiest_background.summary
|
||||
if metric == "postgres_slowest_statement_seconds":
|
||||
slowest: Final = slowest_statement(observation.statements)
|
||||
return "" if slowest is None else slowest.summary
|
||||
return ""
|
||||
|
||||
|
||||
def _row_properties(row: Row) -> tuple[tuple[str, float | str], ...]:
|
||||
observed: Final = ((row.property_name, row.observed),) if row.observed is not None else ()
|
||||
limit: Final = ((f"{row.property_name}_limit", row.limit),) if row.limit is not None else ()
|
||||
return observed + limit
|
||||
floor: Final = ((f"{row.property_name}_floor", row.floor),) if row.floor is not None else ()
|
||||
contributors: Final = (
|
||||
((f"{row.property_name}_breakdown", json.dumps([asdict(item) for item in row.breakdown])),)
|
||||
if row.breakdown
|
||||
else ()
|
||||
)
|
||||
return observed + limit + floor + contributors
|
||||
|
||||
|
||||
def junit_properties(rows: Iterable[Row]) -> tuple[tuple[str, float], ...]:
|
||||
def junit_properties(rows: Iterable[Row]) -> tuple[tuple[str, float | str], ...]:
|
||||
"""One numeric property per observed metric and per limit, plus a `_breakdown` JSON list of the top
|
||||
contributors for every metric that has slices, all under the `perf_` prefix the dashboard ingests."""
|
||||
return tuple(chain.from_iterable(map(_row_properties, rows)))
|
||||
|
||||
|
||||
|
|
@ -313,9 +635,17 @@ def render(case: PerformanceTestCase, observation: Observation, rows: list[Row])
|
|||
]
|
||||
for row in rows:
|
||||
observed = "n/a" if row.observed is None else f"{row.observed:.4f}"
|
||||
limit = "" if row.limit is None else f"{row.limit:.4f}"
|
||||
flag = " OVER" if row.over else " MISSING" if row.missing else ""
|
||||
lines.append(f" {row.label:32} {observed:>12} {limit:>12}{flag}")
|
||||
flag = (
|
||||
" UNDER"
|
||||
if row.over and row.floor is not None
|
||||
else " OVER"
|
||||
if row.over
|
||||
else " MISSING"
|
||||
if row.missing
|
||||
else ""
|
||||
)
|
||||
lines.append(f" {row.label:32} {observed:>12} {row.bound:>12}{flag}")
|
||||
lines.extend(render_steps(observation.steps()))
|
||||
lines.extend(_diagnostics(observation))
|
||||
lines.extend(_datastore_lines(observation))
|
||||
for record in failures:
|
||||
|
|
@ -332,6 +662,7 @@ def _diagnostics(observation: Observation) -> list[str]:
|
|||
return [
|
||||
" diagnostics (not limits):",
|
||||
f" bodies off reply {observation.bodies_off_reply()}",
|
||||
f" mock routes {_mock_routes(observation)}",
|
||||
f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}",
|
||||
f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}",
|
||||
f" retried share {len(observation.retried_markers) / count:.4f}",
|
||||
|
|
@ -342,6 +673,12 @@ def _diagnostics(observation: Observation) -> list[str]:
|
|||
]
|
||||
|
||||
|
||||
def _mock_routes(observation: Observation) -> str:
|
||||
"""Which provider route each request landed on, so a direct-vs-bridge case shows its split."""
|
||||
counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests)
|
||||
return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a"
|
||||
|
||||
|
||||
def _datastore_lines(observation: Observation) -> list[str]:
|
||||
"""What the per-request Postgres and Redis counts are made of, so a red run names the statement or
|
||||
command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit."""
|
||||
|
|
|
|||
|
|
@ -9,10 +9,14 @@ so a case is a literal a reviewer can read top to bottom.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import bisect
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import accumulate
|
||||
from typing import Final, Literal
|
||||
|
||||
from setup.event_loop import EventLoopStalls
|
||||
|
||||
Endpoint = Literal["chat_completions", "responses", "messages"]
|
||||
MockProvider = Literal["openai", "anthropic", "bedrock"]
|
||||
GuardrailMode = Literal["pre_call", "post_call"]
|
||||
|
|
@ -23,9 +27,18 @@ Metric = Literal[
|
|||
"memory_growth_megabytes",
|
||||
"postgres_calls_per_request",
|
||||
"postgres_background_calls_per_request",
|
||||
"postgres_max_statement_per_request",
|
||||
"postgres_max_background_statement_per_request",
|
||||
"cpu_seconds_per_request",
|
||||
"redis_calls_per_request",
|
||||
"redis_background_calls_per_request",
|
||||
"litellm_overhead_seconds",
|
||||
"litellm_response_seconds",
|
||||
"prometheus_redis_latency_mean_seconds",
|
||||
"postgres_active_sessions",
|
||||
"postgres_slowest_statement_seconds",
|
||||
"postgres_peak_connections",
|
||||
"sustained_rps",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -87,12 +100,50 @@ SHORT: Final = RequestShape("short", input_tokens=(5_000, 100_000), tools=True)
|
|||
LONG: Final = RequestShape("long", input_tokens=(300_000, 1_000_000), tools=True)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Ramp:
|
||||
"""Rate stepped up through the run: `first_rps`, then `step_rps` more every `step_seconds`, up to `last_rps`.
|
||||
Each step is scored on its own and the run reports `sustained_rps`, the last step the pod held before the
|
||||
first one it did not: a step holds when its `gateway_added_seconds` p99 stayed under `held_p99_seconds`, it
|
||||
completed at least `HELD_SHARE` of its rate and no more than `HELD_UNEXPECTED_SHARE` of its requests came back
|
||||
with an unexpected status (`setup/ramp.py`). Requests are sent on a schedule drawn from the plan's seed, not
|
||||
from per-user intervals, so the rate changes on the step boundary."""
|
||||
|
||||
first_rps: float
|
||||
last_rps: float
|
||||
step_rps: float
|
||||
step_seconds: float
|
||||
held_p99_seconds: float
|
||||
|
||||
@property
|
||||
def rates(self) -> tuple[float, ...]:
|
||||
count: Final = round((self.last_rps - self.first_rps) / self.step_rps) + 1
|
||||
return tuple(self.first_rps + index * self.step_rps for index in range(count))
|
||||
|
||||
@property
|
||||
def counts(self) -> tuple[int, ...]:
|
||||
return tuple(round(rate * self.step_seconds) for rate in self.rates)
|
||||
|
||||
@property
|
||||
def total_requests(self) -> int:
|
||||
return sum(self.counts)
|
||||
|
||||
@property
|
||||
def seconds(self) -> float:
|
||||
return len(self.rates) * self.step_seconds
|
||||
|
||||
def step_of(self, seq: int) -> int:
|
||||
"""Zero-based step of the request with 1-based plan number `seq`."""
|
||||
return bisect.bisect_left(tuple(accumulate(self.counts)), seq)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Traffic:
|
||||
"""The plan: `total_requests` drawn from `seed`, sent at `rps` by `concurrency` simulated
|
||||
users. Shares are fractions of the requests: `endpoints` and `shapes` must each sum to 1,
|
||||
`intervene_share` picks non-streaming requests the guardrail rewrites, `block_share` picks
|
||||
requests it blocks."""
|
||||
users, or stepped up through a `ramp` (then `rps` is its first step and `total_requests` its
|
||||
total, see `ramped`). Shares are fractions of the requests: `endpoints` and `shapes` must each
|
||||
sum to 1, `intervene_share` picks non-streaming requests the guardrail rewrites, `block_share`
|
||||
picks requests it blocks."""
|
||||
|
||||
seed: int
|
||||
rps: float
|
||||
|
|
@ -103,20 +154,33 @@ class Traffic:
|
|||
intervene_share: float = 0.0
|
||||
block_share: float = 0.0
|
||||
concurrency: int = 20
|
||||
ramp: Ramp | None = None
|
||||
|
||||
@property
|
||||
def expected_seconds(self) -> float:
|
||||
return self.total_requests / self.rps
|
||||
return self.total_requests / self.rps if self.ramp is None else self.ramp.seconds
|
||||
|
||||
|
||||
def ramped(base: Traffic, ramp: Ramp, *, seed: int, concurrency: int) -> Traffic:
|
||||
"""`base`'s request mix sent along `ramp` instead of at one rate. `concurrency` caps the requests in
|
||||
flight, so size it past the deepest step the ramp is expected to reach."""
|
||||
return replace(
|
||||
base, seed=seed, rps=ramp.first_rps, total_requests=ramp.total_requests, concurrency=concurrency, ramp=ramp
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Limit:
|
||||
"""One pass/fail rule: `metric` (at `percentile`, or the whole-run value) must not exceed `max`.
|
||||
Metrics without a Limit are still reported."""
|
||||
"""One pass/fail rule: `metric` (at `percentile`, or the whole-run value) must not exceed `max`, or must
|
||||
reach `min` for a floor such as `sustained_rps`. Metrics without a Limit are still reported."""
|
||||
|
||||
metric: Metric
|
||||
max: float
|
||||
max: float | None = None
|
||||
percentile: int | None = None
|
||||
min: float | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
assert (self.max is None) != (self.min is None), f"{self.metric}: a Limit is either a max or a min"
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
|
|
@ -148,6 +212,10 @@ class PerformanceTestCase:
|
|||
traffic: Traffic
|
||||
limits: tuple[Limit, ...]
|
||||
mock_latency: MockLatency = field(default_factory=lambda: MockLatency())
|
||||
background_cap_exempt_tables: frozenset[str] = frozenset()
|
||||
"""Tables whose background write is one upsert per writer flush and, at this case's request rate, lands
|
||||
above the `postgres_max_background_statement_per_request` cap without being per request."""
|
||||
event_loop: EventLoopStalls | None = None
|
||||
|
||||
def team(self, name: str) -> Team:
|
||||
return next(team for team in self.teams if team.name == name)
|
||||
|
|
@ -187,6 +255,14 @@ class PerformanceTestCase:
|
|||
assert 0 <= self.traffic.stream_share <= 1, f"{self.name}: stream_share out of range"
|
||||
assert self.traffic.intervene_share + self.traffic.block_share <= 1, f"{self.name}: guardrail shares exceed 1"
|
||||
assert self.traffic.total_requests > 0 and self.traffic.rps > 0, f"{self.name}: empty traffic"
|
||||
ramp: Final = self.traffic.ramp
|
||||
assert ramp is None or (ramp.first_rps, ramp.total_requests) == (
|
||||
self.traffic.rps,
|
||||
self.traffic.total_requests,
|
||||
), f"{self.name}: traffic rps and total_requests must be the ramp's, build it with ramped()"
|
||||
assert ramp is None or (0 < ramp.first_rps <= ramp.last_rps and ramp.step_rps > 0 and ramp.step_seconds > 0), (
|
||||
f"{self.name}: ramp {ramp} does not step up"
|
||||
)
|
||||
assert self.model_groups, f"{self.name}: no model groups"
|
||||
assert 0 <= self.mock_latency.jitter_seconds <= self.mock_latency.seconds or self.mock_latency.seconds == 0, (
|
||||
f"{self.name}: mock latency jitter {self.mock_latency.jitter_seconds} s exceeds {self.mock_latency.seconds} s"
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from dataclasses import dataclass
|
|||
from typing import Final, Literal, TypeVar
|
||||
|
||||
from setup.performance_test_case import Endpoint, PerformanceTestCase, RequestShape, Team
|
||||
from setup.ramp import send_offsets
|
||||
|
||||
Flag = Literal["n", "i", "b"]
|
||||
CallerKind = Literal["jwt", "key"]
|
||||
|
|
@ -39,6 +40,7 @@ class PlannedRequest:
|
|||
flag: Flag
|
||||
shape: RequestShape
|
||||
input_tokens: int
|
||||
send_at_s: float | None = None
|
||||
|
||||
@property
|
||||
def blocked(self) -> bool:
|
||||
|
|
@ -98,6 +100,7 @@ def build_plan(case: PerformanceTestCase) -> tuple[PlannedRequest, ...]:
|
|||
case.validate()
|
||||
rng: Final = random.Random(case.traffic.seed)
|
||||
callers: Final = _callers_by_group(case)
|
||||
offsets: Final = None if case.traffic.ramp is None else send_offsets(case.traffic.ramp, case.traffic.seed)
|
||||
planned: list[PlannedRequest] = []
|
||||
for seq in range(1, case.traffic.total_requests + 1):
|
||||
endpoint = _pick(rng, case.traffic.endpoints)
|
||||
|
|
@ -117,6 +120,7 @@ def build_plan(case: PerformanceTestCase) -> tuple[PlannedRequest, ...]:
|
|||
flag=flag,
|
||||
shape=shape,
|
||||
input_tokens=rng.randint(low, high),
|
||||
send_at_s=None if offsets is None else offsets[seq - 1],
|
||||
)
|
||||
)
|
||||
return tuple(planned)
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ from urllib.parse import quote
|
|||
from e2e_http import AuthHeaders, Success, get_external
|
||||
from pydantic import BaseModel
|
||||
|
||||
from setup.series import Recorder, Series
|
||||
|
||||
SELECTOR_ENV: Final = "E2E_PERF_GATEWAY_POD_SELECTOR"
|
||||
_SERVICE_ACCOUNT: Final = Path("/var/run/secrets/kubernetes.io/serviceaccount")
|
||||
_INTERVAL_S: Final = 5.0
|
||||
|
|
@ -49,7 +51,8 @@ def cores(quantity: str) -> float:
|
|||
|
||||
|
||||
class GatewayCpuSampler:
|
||||
"""Integrates the matching pods' core use over the time it is running; `seconds` is CPU seconds."""
|
||||
"""Integrates the matching pods' core use over the time it is running; `seconds` is CPU seconds and
|
||||
`series` the CPU seconds of each sampling interval, stamped when the interval ended."""
|
||||
|
||||
def __init__(self, selector: str, namespace: str, token: str, ca_bundle: str) -> None:
|
||||
self._url: Final = (
|
||||
|
|
@ -60,12 +63,15 @@ class GatewayCpuSampler:
|
|||
self._ca_bundle: Final = ca_bundle
|
||||
self._stop: Final = threading.Event()
|
||||
self._thread: Final = threading.Thread(target=self._run, name="gateway-cpu-sampler", daemon=True)
|
||||
self._cpu_seconds = 0.0
|
||||
self._samples = 0
|
||||
self._intervals: Final[Recorder[float]] = Recorder()
|
||||
|
||||
@property
|
||||
def seconds(self) -> float | None:
|
||||
return self._cpu_seconds if self._samples > 0 else None
|
||||
return sum(sample.value for sample in self._intervals.series.samples) if len(self._intervals) else None
|
||||
|
||||
@property
|
||||
def series(self) -> Series[float]:
|
||||
return self._intervals.series
|
||||
|
||||
def _read_cores(self) -> float | None:
|
||||
result: Final = get_external(
|
||||
|
|
@ -81,8 +87,7 @@ class GatewayCpuSampler:
|
|||
now: float = time.monotonic()
|
||||
current: float | None = self._read_cores()
|
||||
if current is not None:
|
||||
self._cpu_seconds += current * (now - last)
|
||||
self._samples += 1
|
||||
self._intervals.record(current * (now - last))
|
||||
last = now # rebind-ok: the sampling clock advances every tick
|
||||
|
||||
def start(self) -> None:
|
||||
|
|
@ -95,6 +100,7 @@ class GatewayCpuSampler:
|
|||
|
||||
class NoSampler:
|
||||
seconds: Final[None] = None
|
||||
series: Final[None] = None
|
||||
|
||||
def start(self) -> None:
|
||||
return None
|
||||
|
|
@ -103,7 +109,7 @@ class NoSampler:
|
|||
return None
|
||||
|
||||
|
||||
def _in_cluster() -> tuple[str, str, str] | None:
|
||||
def in_cluster() -> tuple[str, str, str] | None:
|
||||
token: Final = _SERVICE_ACCOUNT / "token"
|
||||
ca: Final = _SERVICE_ACCOUNT / "ca.crt"
|
||||
namespace: Final = _SERVICE_ACCOUNT / "namespace"
|
||||
|
|
@ -115,7 +121,7 @@ def _in_cluster() -> tuple[str, str, str] | None:
|
|||
@contextmanager
|
||||
def gateway_cpu_sampler() -> Generator[GatewayCpuSampler | NoSampler]:
|
||||
selector: Final = os.environ.get(SELECTOR_ENV, "")
|
||||
cluster: Final = _in_cluster()
|
||||
cluster: Final = in_cluster()
|
||||
sampler: Final = (
|
||||
GatewayCpuSampler(selector, cluster[2], cluster[0], cluster[1]) if selector and cluster else NoSampler()
|
||||
)
|
||||
|
|
|
|||
116
tests/e2e/perf/setup/ramp.py
Normal file
116
tests/e2e/perf/setup/ramp.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
"""A ramped run step by step: the send schedule its plan carries (`send_offsets`), one `StepReading` per step
|
||||
and `sustained_rps`, the last step the pod held before the first it did not. `Ramp` itself sits in
|
||||
`performance_test_case.py` with the rest of the case schema."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, takewhile
|
||||
from typing import Final
|
||||
|
||||
from setup.performance_test_case import Ramp
|
||||
|
||||
HELD_SHARE: Final = 0.9
|
||||
"""A step that completed fewer than this share of its rate is a queue that stopped draining."""
|
||||
HELD_UNEXPECTED_SHARE: Final = 0.01
|
||||
"""A step with more than this share of unexpected statuses broke, whatever its latency."""
|
||||
_JITTER: Final = 0.25
|
||||
|
||||
|
||||
def _step_offsets(ramp: Ramp, index: int, count: int, rng: random.Random) -> Iterator[float]:
|
||||
for slot in range(count):
|
||||
yield (index + (slot + 0.5 + rng.uniform(-_JITTER, _JITTER)) / count) * ramp.step_seconds
|
||||
|
||||
|
||||
def send_offsets(ramp: Ramp, seed: int) -> tuple[float, ...]:
|
||||
"""Seconds after the run starts to send each request of the plan, in plan order: a step's requests spread
|
||||
evenly over its window, each moved by up to a quarter of a slot either way, drawn from `seed` so a rerun
|
||||
sends at the same seconds."""
|
||||
rng: Final = random.Random(f"ramp-{seed}")
|
||||
return tuple(chain.from_iterable(_step_offsets(ramp, index, count, rng) for index, count in enumerate(ramp.counts)))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StepReading:
|
||||
rps: float
|
||||
planned: int
|
||||
completed_rps: float
|
||||
gateway_added_p99: float | None
|
||||
unexpected: int
|
||||
held_p99_seconds: float
|
||||
gateway_added_p50: float | None = None
|
||||
time_to_first_token_p99: float | None = None
|
||||
cpu_seconds_per_request: float | None = None
|
||||
redis_latency_mean_seconds: float | None = None
|
||||
postgres_peak_connections: int | None = None
|
||||
"""The hold verdict reads the first six. The rest say what the pod was doing on that step: CPU per request
|
||||
from the pod sampler, Redis latency from the gateway's `/metrics`, connections from `pg_stat_activity`,
|
||||
all cut to the step's window, so a latency jump can be told apart as CPU, waiting or neither."""
|
||||
|
||||
@property
|
||||
def held(self) -> bool:
|
||||
return (
|
||||
self.gateway_added_p99 is not None
|
||||
and self.gateway_added_p99 <= self.held_p99_seconds
|
||||
and self.completed_rps >= HELD_SHARE * self.rps
|
||||
and self.unexpected <= HELD_UNEXPECTED_SHARE * self.planned
|
||||
)
|
||||
|
||||
|
||||
def sustained_rps(steps: Iterable[StepReading]) -> float:
|
||||
"""The rate of the last step that held before the first that did not, 0 when the first step did not hold."""
|
||||
held: Final = tuple(takewhile(lambda step: step.held, steps))
|
||||
return held[-1].rps if held else 0.0
|
||||
|
||||
|
||||
def step_properties(steps: Iterable[StepReading]) -> tuple[tuple[str, float], ...]:
|
||||
"""The per-step readings as JUnit properties (`perf_ramp_step_<rps>_<reading>`, every reading the step has)
|
||||
plus the hold bound, so the dashboard can draw the run's curves next to the sustained rate."""
|
||||
rows: Final = tuple(steps)
|
||||
hold: Final = (("perf_ramp_hold_p99_seconds", rows[0].held_p99_seconds),) if rows else ()
|
||||
return (*hold, *chain.from_iterable(_step_properties(step) for step in rows))
|
||||
|
||||
|
||||
def _step_properties(step: StepReading) -> Iterator[tuple[str, float]]:
|
||||
prefix: Final = f"perf_ramp_step_{step.rps:g}"
|
||||
readings: Final[tuple[tuple[str, float | None], ...]] = (
|
||||
("gateway_added_p99", step.gateway_added_p99),
|
||||
("completed_rps", step.completed_rps),
|
||||
("unexpected", float(step.unexpected)),
|
||||
("gateway_added_p50", step.gateway_added_p50),
|
||||
("time_to_first_token_p99", step.time_to_first_token_p99),
|
||||
("cpu_seconds_per_request", step.cpu_seconds_per_request),
|
||||
("redis_latency_mean_seconds", step.redis_latency_mean_seconds),
|
||||
(
|
||||
"postgres_peak_connections",
|
||||
None if step.postgres_peak_connections is None else float(step.postgres_peak_connections),
|
||||
),
|
||||
)
|
||||
return ((f"{prefix}_{name}", value) for name, value in readings if value is not None)
|
||||
|
||||
|
||||
def render_steps(steps: Iterable[StepReading]) -> tuple[str, ...]:
|
||||
rows: Final = tuple(steps)
|
||||
if not rows:
|
||||
return ()
|
||||
header: Final = (
|
||||
f" steps: {'rps':>6} {'planned':>8} {'completed rps':>14} {'gateway added p50':>18} {'p99':>8}"
|
||||
f" {'ttft p99':>9} {'unexpected':>11} {'cpu s/req':>10} {'redis mean':>11} {'pg conns':>9}"
|
||||
)
|
||||
return (header, *(_step_line(step) for step in rows))
|
||||
|
||||
|
||||
def _seconds(value: float | None) -> str:
|
||||
return "n/a" if value is None else f"{value:.4f}"
|
||||
|
||||
|
||||
def _step_line(step: StepReading) -> str:
|
||||
conns: Final = "n/a" if step.postgres_peak_connections is None else str(step.postgres_peak_connections)
|
||||
return (
|
||||
f" {step.rps:6.1f} {step.planned:8d} {step.completed_rps:14.2f} {_seconds(step.gateway_added_p50):>18}"
|
||||
f" {_seconds(step.gateway_added_p99):>8} {_seconds(step.time_to_first_token_p99):>9} {step.unexpected:11d}"
|
||||
f" {_seconds(step.cpu_seconds_per_request):>10} {_seconds(step.redis_latency_mean_seconds):>11} {conns:>9}"
|
||||
f" {'held' if step.held else 'broke'}"
|
||||
)
|
||||
|
|
@ -9,9 +9,11 @@ import shutil
|
|||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from typing import BinaryIO, Final
|
||||
|
||||
from e2e_config import PROXY_BASE_URL
|
||||
from pydantic import TypeAdapter
|
||||
|
|
@ -61,6 +63,7 @@ def wire_request(item: PlannedRequest, setup: CaseSetup) -> WireRequest:
|
|||
token_source=setup.token_source(item.caller),
|
||||
stream=item.stream,
|
||||
blocked=item.blocked,
|
||||
send_at_s=item.send_at_s,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -73,7 +76,59 @@ def _keep_records_artifact(records_path: Path) -> None:
|
|||
_ = shutil.copyfile(records_path, Path(artifact_dir) / "perf-records.jsonl")
|
||||
|
||||
|
||||
def run_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...], setup: CaseSetup) -> RunOutcome:
|
||||
class RecordStream:
|
||||
def __init__(self, source: BinaryIO) -> None:
|
||||
self.source: Final = source
|
||||
self.partial = b""
|
||||
|
||||
def read(self) -> tuple[WireRecord, ...]:
|
||||
chunks: Final = (self.partial + self.source.read()).split(b"\n")
|
||||
self.partial = chunks[-1]
|
||||
return tuple(_RECORD_ADAPTER.validate_json(line) for line in chunks[:-1] if line)
|
||||
|
||||
|
||||
def run_until(
|
||||
command: list[str],
|
||||
env: dict[str, str],
|
||||
records_path: Path,
|
||||
stop_path: Path,
|
||||
stop_when: Callable[[tuple[WireRecord, ...]], bool],
|
||||
timeout: int,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
with records_path.open("rb") as records, records_path.with_suffix(".log").open("w+") as output:
|
||||
with subprocess.Popen(command, env=env, stdout=output, stderr=subprocess.STDOUT, text=True) as process:
|
||||
deadline: Final = time.monotonic() + timeout
|
||||
reader: Final = RecordStream(records)
|
||||
try:
|
||||
while process.poll() is None:
|
||||
if stop_when(reader.read()):
|
||||
stop_path.touch()
|
||||
if time.monotonic() >= deadline:
|
||||
raise subprocess.TimeoutExpired(command, timeout)
|
||||
try:
|
||||
_ = process.wait(timeout=1)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
finally:
|
||||
stop_path.touch()
|
||||
try:
|
||||
_ = process.wait(timeout=35)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
_ = process.wait(timeout=10)
|
||||
_keep_records_artifact(records_path)
|
||||
_ = output.seek(0)
|
||||
return subprocess.CompletedProcess(command, process.returncode, output.read(), "")
|
||||
|
||||
|
||||
def run_plan(
|
||||
case: PerformanceTestCase,
|
||||
plan: tuple[PlannedRequest, ...],
|
||||
setup: CaseSetup,
|
||||
*,
|
||||
stop_when: Callable[[tuple[WireRecord, ...]], bool] | None = None,
|
||||
request_timeout: float = 120.0,
|
||||
) -> RunOutcome:
|
||||
"""Replay `plan` with Locust in a child process and collect its records. The run-time ceiling
|
||||
is well past the planned duration so Locust's own clock never cuts a slow run short; the
|
||||
locustfile quits itself once the last record is written."""
|
||||
|
|
@ -82,6 +137,7 @@ def run_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...], setup:
|
|||
with tempfile.TemporaryDirectory(prefix="e2e-perf-") as directory:
|
||||
plan_path: Final = Path(directory) / "plan.json"
|
||||
records_path: Final = Path(directory) / "records.jsonl"
|
||||
stop_path: Final = Path(directory) / "stop"
|
||||
_ = plan_path.write_bytes(_PLAN_ADAPTER.dump_json(wire))
|
||||
records_path.touch()
|
||||
env: Final = {
|
||||
|
|
@ -91,33 +147,33 @@ def run_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...], setup:
|
|||
"PERF_RECORDS_FILE": str(records_path),
|
||||
"PERF_RPS": str(case.traffic.rps),
|
||||
"PERF_USERS": str(case.traffic.concurrency),
|
||||
"PERF_REQUEST_TIMEOUT": str(request_timeout),
|
||||
**({"PERF_STOP_FILE": str(stop_path)} if stop_when is not None else {}),
|
||||
}
|
||||
completed: Final = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-P",
|
||||
"-m",
|
||||
"locust",
|
||||
"--headless",
|
||||
"--only-summary",
|
||||
"--locustfile",
|
||||
str(_LOCUSTFILE),
|
||||
"--host",
|
||||
PROXY_BASE_URL,
|
||||
"--users",
|
||||
str(case.traffic.concurrency),
|
||||
"--spawn-rate",
|
||||
str(case.traffic.concurrency),
|
||||
"--run-time",
|
||||
f"{ceiling_s}s",
|
||||
"--exit-code-on-error",
|
||||
"0",
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=ceiling_s + 120,
|
||||
check=False,
|
||||
command: Final = [
|
||||
sys.executable,
|
||||
"-P",
|
||||
"-m",
|
||||
"locust",
|
||||
"--headless",
|
||||
"--only-summary",
|
||||
"--locustfile",
|
||||
str(_LOCUSTFILE),
|
||||
"--host",
|
||||
PROXY_BASE_URL,
|
||||
"--users",
|
||||
str(case.traffic.concurrency),
|
||||
"--spawn-rate",
|
||||
str(case.traffic.concurrency),
|
||||
"--run-time",
|
||||
f"{ceiling_s}s",
|
||||
"--exit-code-on-error",
|
||||
"0",
|
||||
]
|
||||
completed: Final = (
|
||||
run_until(command, env, records_path, stop_path, stop_when, ceiling_s + 120)
|
||||
if stop_when is not None
|
||||
else subprocess.run(command, env=env, capture_output=True, text=True, timeout=ceiling_s + 120, check=False)
|
||||
)
|
||||
records: Final = tuple(
|
||||
_RECORD_ADAPTER.validate_json(line)
|
||||
|
|
|
|||
48
tests/e2e/perf/setup/series.py
Normal file
48
tests/e2e/perf/setup/series.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
"""Readings a background sampler takes while the traffic runs, each stamped with the wall clock
|
||||
(`time.time()`, the clock the Locust records carry in `started_at`), so a ramped run can cut them into its
|
||||
steps' windows after the fact."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Generic, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Sample(Generic[T]):
|
||||
at: float
|
||||
value: T
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Series(Generic[T]):
|
||||
samples: tuple[Sample[T], ...]
|
||||
|
||||
def between(self, start: float, end: float) -> tuple[T, ...]:
|
||||
"""The values stamped inside `[start, end)`, in time order."""
|
||||
return tuple(sample.value for sample in self.samples if start <= sample.at < end)
|
||||
|
||||
def last_before(self, at: float) -> T | None:
|
||||
"""The newest value stamped before `at`, None when the series starts later."""
|
||||
earlier: Final = tuple(sample.value for sample in self.samples if sample.at < at)
|
||||
return earlier[-1] if earlier else None
|
||||
|
||||
|
||||
class Recorder(Generic[T]):
|
||||
"""What a sampler thread appends to; `series` is a frozen copy for the observation."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._samples: list[Sample[T]] = []
|
||||
|
||||
def record(self, value: T) -> None:
|
||||
self._samples.append(Sample(at=time.time(), value=value))
|
||||
|
||||
@property
|
||||
def series(self) -> Series[T]:
|
||||
return Series(samples=tuple(self._samples))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._samples)
|
||||
110
tests/e2e/perf/setup/telemetry.py
Normal file
110
tests/e2e/perf/setup/telemetry.py
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
"""The gateway's own `/metrics`, read before and after a run (and every few seconds during a ramped run, see
|
||||
`telemetry_sampler`) for the numbers nothing outside the gateway can see. Today that is `litellm_redis_latency`, the histogram customers' Grafana panels show when Redis
|
||||
slows the gateway down. The gateway's latency and request counts are read per request from its response
|
||||
headers instead (`setup/wire.py`), which gives percentiles where a histogram delta only gives a mean.
|
||||
|
||||
RSS and CPU are not here: a multi-worker gateway serves `/metrics` through the prometheus multiprocess
|
||||
collector, which drops `process_resident_memory_bytes` and `process_cpu_seconds_total`, so those stay
|
||||
on psutil (`proxy_usage.py`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import threading
|
||||
from collections.abc import Generator, Iterable
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
from e2e_http import NoBody
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from prometheus_client.samples import Sample
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
from setup.series import Recorder, Series
|
||||
|
||||
_SAMPLE_SECONDS: Final = 10.0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HistogramTotals:
|
||||
"""`_sum` and `_count` of one histogram across every label set, enough for a mean over a window."""
|
||||
|
||||
total: float
|
||||
count: float
|
||||
|
||||
def minus(self, earlier: HistogramTotals) -> HistogramTotals:
|
||||
return HistogramTotals(total=self.total - earlier.total, count=self.count - earlier.count)
|
||||
|
||||
@property
|
||||
def mean(self) -> float | None:
|
||||
return self.total / self.count if self.count > 0 else None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GatewayTelemetry:
|
||||
redis_latency: HistogramTotals
|
||||
"""`litellm_redis_latency`, seconds per Redis call the gateway made."""
|
||||
|
||||
def minus(self, earlier: GatewayTelemetry) -> GatewayTelemetry:
|
||||
return GatewayTelemetry(redis_latency=self.redis_latency.minus(earlier.redis_latency))
|
||||
|
||||
|
||||
def _value(samples: Iterable[Sample], name: str) -> float:
|
||||
return sum(sample.value for sample in samples if sample.name == name and math.isfinite(sample.value))
|
||||
|
||||
|
||||
def _histogram(samples: tuple[Sample, ...], family: str) -> HistogramTotals:
|
||||
return HistogramTotals(total=_value(samples, f"{family}_sum"), count=_value(samples, f"{family}_count"))
|
||||
|
||||
|
||||
def parse_gateway_telemetry(text: str) -> GatewayTelemetry:
|
||||
samples: Final = tuple(chain.from_iterable(family.samples for family in text_string_to_metric_families(text)))
|
||||
return GatewayTelemetry(redis_latency=_histogram(samples, "litellm_redis_latency"))
|
||||
|
||||
|
||||
def read_gateway_telemetry(proxy: ProxyClient) -> GatewayTelemetry | None:
|
||||
"""One `/metrics` scrape, or None when the gateway does not serve it (no `prometheus` callback), in which
|
||||
case every limit on a gateway-reported metric fails as missing rather than reading a silent zero."""
|
||||
scrape: Final = proxy.probe("/metrics", params=NoBody())
|
||||
return parse_gateway_telemetry(scrape.body) if scrape.status_code == 200 else None
|
||||
|
||||
|
||||
class TelemetrySampler:
|
||||
"""`/metrics` every `_SAMPLE_SECONDS` on a background thread while traffic runs, so a ramped run can read
|
||||
the Redis latency of each step's window as the difference between the scrapes around it."""
|
||||
|
||||
def __init__(self, proxy: ProxyClient) -> None:
|
||||
self._proxy: Final = proxy
|
||||
self._stop: Final = threading.Event()
|
||||
self._thread: Final = threading.Thread(target=self._run, name="gateway-telemetry", daemon=True)
|
||||
self._scrapes: Final[Recorder[GatewayTelemetry]] = Recorder()
|
||||
|
||||
@property
|
||||
def series(self) -> Series[GatewayTelemetry]:
|
||||
return self._scrapes.series
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.is_set():
|
||||
scrape: GatewayTelemetry | None = read_gateway_telemetry(self._proxy)
|
||||
if scrape is not None:
|
||||
self._scrapes.record(scrape)
|
||||
_ = self._stop.wait(_SAMPLE_SECONDS)
|
||||
|
||||
def start(self) -> None:
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=30)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def telemetry_sampler(proxy: ProxyClient) -> Generator[TelemetrySampler]:
|
||||
sampler: Final = TelemetrySampler(proxy)
|
||||
sampler.start()
|
||||
try:
|
||||
yield sampler
|
||||
finally:
|
||||
sampler.stop()
|
||||
430
tests/e2e/perf/setup/ui_journeys.py
Normal file
430
tests/e2e/perf/setup/ui_journeys.py
Normal file
|
|
@ -0,0 +1,430 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
import statistics
|
||||
import time
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from playwright.sync_api import Error as PlaywrightError
|
||||
from playwright.sync_api import Locator, Page, Response, expect, sync_playwright
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
MONEY: Final = re.compile(r"^\$[\d,]+\.\d{2}$")
|
||||
LOADING_LOGS: Final = "Loading request logs…"
|
||||
LOADING_MODELS: Final = "Loading models…"
|
||||
LOADING_KEYS: Final = "Loading keys..."
|
||||
|
||||
|
||||
class Sample(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
seconds: float
|
||||
rows: int = 0
|
||||
error: str = ""
|
||||
status: int = 0
|
||||
request: str = ""
|
||||
|
||||
|
||||
class Timings(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
first_seconds: float | None
|
||||
repeated_samples: int
|
||||
repeated_p50_seconds: float | None
|
||||
repeated_p95_seconds: float | None
|
||||
repeated_p99_seconds: float | None
|
||||
repeated_max_seconds: float | None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Journey:
|
||||
"""One dashboard interaction. ``start`` brings the page to the state the user acts from (untimed),
|
||||
``act`` is the click or navigation being measured and ``settled`` blocks until the data the user
|
||||
was waiting for is on screen, returning the number of rows or tiles that rendered."""
|
||||
|
||||
name: str
|
||||
surface: str
|
||||
start: Callable[[Page, float], None]
|
||||
act: Callable[[Page], None]
|
||||
settled: Callable[[Page, float], int]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Dashboard:
|
||||
base_url: str
|
||||
username: str
|
||||
password: str
|
||||
anchor: datetime
|
||||
|
||||
|
||||
def summarize(samples: tuple[Sample, ...]) -> Timings:
|
||||
assert samples, "No UI journeys were measured"
|
||||
repeated: Final = tuple(sorted(sample.seconds for sample in samples[1:] if not sample.error))
|
||||
return Timings(
|
||||
first_seconds=samples[0].seconds if not samples[0].error else None,
|
||||
repeated_samples=len(repeated),
|
||||
repeated_p50_seconds=statistics.median(repeated) if repeated else None,
|
||||
repeated_p95_seconds=repeated[math.ceil(len(repeated) * 0.95) - 1] if len(repeated) >= 20 else None,
|
||||
repeated_p99_seconds=repeated[math.ceil(len(repeated) * 0.99) - 1] if len(repeated) >= 100 else None,
|
||||
repeated_max_seconds=max(repeated) if repeated else None,
|
||||
)
|
||||
|
||||
|
||||
def error_code(sample: Sample) -> str:
|
||||
if not sample.error:
|
||||
return ""
|
||||
if sample.status >= 400:
|
||||
return f"http_{sample.status}"
|
||||
if sample.error.startswith("wall_clock"):
|
||||
return "wall_clock"
|
||||
return "timeout" if sample.error == "timeout" else "ui"
|
||||
|
||||
|
||||
def ui_junit_properties(samples: tuple[Sample, ...]) -> tuple[tuple[str, float], ...]:
|
||||
codes: Final = tuple(error_code(sample) for sample in samples if sample.error)
|
||||
timings: Final = summarize(samples)
|
||||
successful: Final = tuple(sorted(sample.seconds for sample in samples if not sample.error))
|
||||
latency: Final = (
|
||||
("seconds_p50", statistics.median(successful) if successful else None),
|
||||
("seconds_p90", successful[math.ceil(len(successful) * 0.90) - 1] if successful else None),
|
||||
("seconds_max", max(successful) if successful else None),
|
||||
("first_seconds", timings.first_seconds),
|
||||
("repeated_seconds_p50", timings.repeated_p50_seconds),
|
||||
("repeated_seconds_p95", timings.repeated_p95_seconds),
|
||||
("repeated_seconds_p99", timings.repeated_p99_seconds),
|
||||
("repeated_seconds_max", timings.repeated_max_seconds),
|
||||
)
|
||||
return (
|
||||
("perf_ui_journeys", len(samples)),
|
||||
("perf_ui_failures", len(codes)),
|
||||
("perf_ui_samples", len(successful)),
|
||||
("perf_ui_repeated_samples", timings.repeated_samples),
|
||||
*((f"perf_ui_error_{code}", codes.count(code)) for code in sorted(frozenset(codes))),
|
||||
*((f"perf_ui_{name}", value) for name, value in latency if value is not None),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Ceiling:
|
||||
p50_seconds: float
|
||||
p90_seconds: float
|
||||
|
||||
|
||||
LOGS_CEILING: Final = Ceiling(p50_seconds=10, p90_seconds=20)
|
||||
CHART_CEILING: Final = Ceiling(p50_seconds=20, p90_seconds=40)
|
||||
TABLE_CEILING: Final = Ceiling(p50_seconds=5, p90_seconds=10)
|
||||
UI_JOURNEY_LIMITS: Final = MappingProxyType(
|
||||
{
|
||||
"logs_open_24h": LOGS_CEILING,
|
||||
"logs_last_7_days": LOGS_CEILING,
|
||||
"logs_next_page": LOGS_CEILING,
|
||||
"logs_filter_key_hash": LOGS_CEILING,
|
||||
"logs_filter_key_alias": LOGS_CEILING,
|
||||
"logs_filter_team": LOGS_CEILING,
|
||||
"logs_filter_model": LOGS_CEILING,
|
||||
"usage_open": CHART_CEILING,
|
||||
"usage_last_30_days": CHART_CEILING,
|
||||
"usage_team_view": CHART_CEILING,
|
||||
"usage_customer_view": CHART_CEILING,
|
||||
"usage_model_activity_30_days": CHART_CEILING,
|
||||
"cache_open": CHART_CEILING,
|
||||
"models_open": TABLE_CEILING,
|
||||
"keys_open": TABLE_CEILING,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def assert_within_ceiling(journey_name: str, samples: tuple[Sample, ...]) -> None:
|
||||
failed: Final = tuple(sample.error for sample in samples if sample.error)
|
||||
assert not failed, f"{len(failed)} of {len(samples)} {journey_name} samples failed: {failed[0]}"
|
||||
metrics: Final = dict(ui_junit_properties(samples))
|
||||
ceiling: Final = UI_JOURNEY_LIMITS[journey_name]
|
||||
assert metrics["perf_ui_seconds_p50"] <= ceiling.p50_seconds, (
|
||||
f"{journey_name} p50 {metrics['perf_ui_seconds_p50']:.1f}s over {ceiling.p50_seconds}s"
|
||||
)
|
||||
assert metrics["perf_ui_seconds_p90"] <= ceiling.p90_seconds, (
|
||||
f"{journey_name} p90 {metrics['perf_ui_seconds_p90']:.1f}s over {ceiling.p90_seconds}s"
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def dashboard_page(dashboard: Dashboard, timeout: float) -> Generator[Page]:
|
||||
"""A logged-in admin tab whose clock reads the seed anchor, so relative windows such as
|
||||
Last 24 Hours land on seeded rows. Live Tail is off so its 15 second refetch cannot land
|
||||
inside a sample. Playwright keeps a UTC timezone to match the seed."""
|
||||
with sync_playwright() as playwright:
|
||||
browser: Final = playwright.chromium.launch()
|
||||
context: Final = browser.new_context(
|
||||
base_url=dashboard.base_url, timezone_id="UTC", viewport={"width": 1600, "height": 1000}
|
||||
)
|
||||
context.set_default_timeout(timeout * 1000)
|
||||
context.add_init_script('sessionStorage.setItem("isLiveTail", "false")')
|
||||
page: Final = context.new_page()
|
||||
page.clock.set_fixed_time(dashboard.anchor)
|
||||
_login(page, dashboard)
|
||||
try:
|
||||
yield page
|
||||
finally:
|
||||
context.close()
|
||||
browser.close()
|
||||
|
||||
|
||||
def _login(page: Page, dashboard: Dashboard) -> None:
|
||||
_ = page.goto("/ui/login")
|
||||
page.get_by_placeholder("Enter your username").fill(dashboard.username)
|
||||
page.get_by_placeholder("Enter your password").fill(dashboard.password)
|
||||
page.get_by_role("button", name="Login", exact=True).click()
|
||||
page.wait_for_url(lambda url: "/ui" in url and "/login" not in url)
|
||||
expect(page.get_by_label("LiteLLM home")).to_be_visible()
|
||||
|
||||
|
||||
def _is_data_fetch(response: Response) -> bool:
|
||||
"""API calls only. The Admin UI's own assets and Next.js prefetches live under /ui/ and /_next/."""
|
||||
path: Final = urlsplit(response.url).path
|
||||
return response.request.resource_type in ("fetch", "xhr") and not path.startswith(("/ui/", "/_next/"))
|
||||
|
||||
|
||||
def _worst(responses: list[Response]) -> Response | None:
|
||||
return next((response for response in responses if response.status >= 400), responses[-1] if responses else None)
|
||||
|
||||
|
||||
def _with_response(sample: Sample, responses: list[Response]) -> Sample:
|
||||
worst: Final = _worst(responses)
|
||||
if worst is None:
|
||||
return sample
|
||||
return sample.model_copy(
|
||||
update={"status": worst.status, "request": f"{worst.request.method} {urlsplit(worst.url).path}"[:120]}
|
||||
)
|
||||
|
||||
|
||||
def measure(page: Page, journey: Journey, timeout: float) -> Sample:
|
||||
"""Wall time from the user's action to the data being on screen. Every sample starts from the
|
||||
same pre-action state so repeats measure the same journey, not a warmed client cache."""
|
||||
try:
|
||||
journey.start(page, timeout)
|
||||
except (PlaywrightError, AssertionError) as error:
|
||||
return Sample(seconds=0.0, error=f"start {type(error).__name__}: {error}"[:300])
|
||||
responses: Final[list[Response]] = [] # mutable-ok: filled by the playwright response listener
|
||||
|
||||
def record(response: Response) -> None:
|
||||
if _is_data_fetch(response):
|
||||
responses.append(response)
|
||||
|
||||
page.on("response", record)
|
||||
started: Final = time.monotonic()
|
||||
try:
|
||||
with page.expect_response(_is_data_fetch, timeout=timeout * 1000):
|
||||
journey.act(page)
|
||||
rows: Final = journey.settled(page, timeout)
|
||||
except PlaywrightTimeoutError:
|
||||
return _with_response(Sample(seconds=time.monotonic() - started, error="timeout"), responses)
|
||||
except (PlaywrightError, AssertionError) as error:
|
||||
return _with_response(
|
||||
Sample(seconds=time.monotonic() - started, error=f"{type(error).__name__}: {error}"[:300]), responses
|
||||
)
|
||||
finally:
|
||||
page.remove_listener("response", record)
|
||||
return _with_response(Sample(seconds=time.monotonic() - started, rows=rows), responses)
|
||||
|
||||
|
||||
def _visible(page: Page, test_id: str) -> Locator:
|
||||
return page.get_by_test_id(test_id).filter(visible=True).first
|
||||
|
||||
|
||||
def _logs_rows(page: Page) -> Locator:
|
||||
return page.locator("table").filter(visible=True).first.locator("tbody tr")
|
||||
|
||||
|
||||
def _logs_settled(page: Page, timeout: float) -> int:
|
||||
millis: Final = timeout * 1000
|
||||
rows: Final = _logs_rows(page).filter(has_not_text="No results")
|
||||
expect(rows.first).to_be_visible(timeout=millis)
|
||||
expect(_visible(page, "datatable-refresh")).to_be_enabled(timeout=millis)
|
||||
expect(page.get_by_text(LOADING_LOGS)).to_have_count(0, timeout=millis)
|
||||
return rows.count()
|
||||
|
||||
|
||||
def _no_skeletons(page: Page, timeout: float) -> None:
|
||||
expect(page.locator('[data-slot="skeleton"]')).to_have_count(0, timeout=timeout * 1000)
|
||||
|
||||
|
||||
def _usage_settled(page: Page, timeout: float) -> int:
|
||||
millis: Final = timeout * 1000
|
||||
expect(page.get_by_text("Total Spend", exact=True).first).to_be_visible(timeout=millis)
|
||||
_no_skeletons(page, timeout)
|
||||
money: Final = page.get_by_text(MONEY)
|
||||
expect(money.first).to_be_visible(timeout=millis)
|
||||
return money.count()
|
||||
|
||||
|
||||
def _cache_settled(page: Page, timeout: float) -> int:
|
||||
millis: Final = timeout * 1000
|
||||
expect(page.get_by_text("Cache Hits", exact=True)).to_be_visible(timeout=millis)
|
||||
bars: Final = page.locator(".recharts-bar-rectangle")
|
||||
expect(bars.first).to_be_visible(timeout=millis)
|
||||
return bars.count()
|
||||
|
||||
|
||||
def _table_settled(loading: str) -> Callable[[Page, float], int]:
|
||||
def settled(page: Page, timeout: float) -> int:
|
||||
millis: Final = timeout * 1000
|
||||
rows: Final = page.locator("[data-row-id]")
|
||||
expect(rows.first).to_be_visible(timeout=millis)
|
||||
expect(page.get_by_text(loading)).to_have_count(0, timeout=millis)
|
||||
return rows.count()
|
||||
|
||||
return settled
|
||||
|
||||
|
||||
def _open_logs(page: Page, timeout: float) -> None:
|
||||
_ = page.goto("/ui/logs")
|
||||
_ = _logs_settled(page, timeout)
|
||||
|
||||
|
||||
def _open_usage(page: Page, timeout: float) -> None:
|
||||
_ = page.goto("/ui/usage")
|
||||
_ = _usage_settled(page, timeout)
|
||||
|
||||
|
||||
def _filters_drawer(page: Page) -> Locator:
|
||||
_visible(page, "datatable-filters-trigger").click()
|
||||
drawer: Final = page.get_by_role("dialog", name="Filters")
|
||||
expect(drawer).to_be_visible()
|
||||
return drawer
|
||||
|
||||
|
||||
def _apply_filters(page: Page) -> None:
|
||||
page.get_by_role("dialog", name="Filters").get_by_role("button", name="Apply Filters").click()
|
||||
|
||||
|
||||
def _pick_first_option(page: Page, combobox: str) -> None:
|
||||
_filters_drawer(page).get_by_role("combobox", name=combobox).click()
|
||||
page.get_by_role("option").first.click()
|
||||
|
||||
|
||||
def _open_time_range(page: Page, timeout: float) -> None:
|
||||
_open_logs(page, timeout)
|
||||
page.get_by_role("button", name="Last 24 Hours").click()
|
||||
|
||||
|
||||
def _open_usage_range(page: Page, timeout: float) -> None:
|
||||
_open_usage(page, timeout)
|
||||
page.locator('[data-slot="advanced-date-picker-trigger"]').click()
|
||||
|
||||
|
||||
def _model_activity_settled(page: Page, timeout: float) -> int:
|
||||
millis: Final = timeout * 1000
|
||||
expect(page.locator('[role="tab"][data-active]', has_text="Model Activity")).to_be_visible(timeout=millis)
|
||||
_no_skeletons(page, timeout)
|
||||
money: Final = page.get_by_text(MONEY).filter(visible=True)
|
||||
expect(money.first).to_be_visible(timeout=millis)
|
||||
return money.count()
|
||||
|
||||
|
||||
def _open_model_activity_range(page: Page, timeout: float) -> None:
|
||||
_open_usage(page, timeout)
|
||||
page.get_by_role("tab", name="Model Activity").click()
|
||||
_ = _model_activity_settled(page, timeout)
|
||||
page.locator('[data-slot="advanced-date-picker-trigger"]').click()
|
||||
|
||||
|
||||
def _pick_usage_preset(preset: str) -> Callable[[Page], None]:
|
||||
def act(page: Page) -> None:
|
||||
page.locator('[data-slot="advanced-date-picker-preset"]', has_text=preset).click()
|
||||
|
||||
return act
|
||||
|
||||
|
||||
def _select_usage_view(page: Page, view: str) -> None:
|
||||
page.locator('[data-slot="select-trigger"]').filter(has_text="Usage").first.click()
|
||||
page.get_by_role("option", name=re.compile(rf"^{re.escape(view)}")).click()
|
||||
|
||||
|
||||
def _nothing(page: Page, timeout: float) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _goto(path: str) -> Callable[[Page], None]:
|
||||
def act(page: Page) -> None:
|
||||
_ = page.goto(path)
|
||||
|
||||
return act
|
||||
|
||||
|
||||
def _logs_filter(name: str, prepare: Callable[[Page], None]) -> Journey:
|
||||
def start(page: Page, timeout: float) -> None:
|
||||
_open_logs(page, timeout)
|
||||
prepare(page)
|
||||
|
||||
return Journey(name, "logs", start, _apply_filters, _logs_settled)
|
||||
|
||||
|
||||
def journeys(api_key: str) -> tuple[Journey, ...]:
|
||||
return (
|
||||
Journey("logs_open_24h", "logs", _nothing, _goto("/ui/logs"), _logs_settled),
|
||||
Journey(
|
||||
"logs_last_7_days",
|
||||
"logs",
|
||||
_open_time_range,
|
||||
lambda page: page.get_by_role("button", name="Last 7 Days", exact=True).click(),
|
||||
_logs_settled,
|
||||
),
|
||||
Journey(
|
||||
"logs_next_page", "logs", _open_logs, lambda page: _visible(page, "pagination-next").click(), _logs_settled
|
||||
),
|
||||
_logs_filter(
|
||||
"logs_filter_key_hash",
|
||||
lambda page: _filters_drawer(page).get_by_placeholder("Enter key hash…").fill(api_key),
|
||||
),
|
||||
_logs_filter("logs_filter_key_alias", lambda page: _pick_first_option(page, "Search a key alias")),
|
||||
_logs_filter("logs_filter_team", lambda page: _pick_first_option(page, "Search or select a team")),
|
||||
_logs_filter("logs_filter_model", lambda page: _pick_first_option(page, "Search a model")),
|
||||
Journey("usage_open", "usage", _nothing, _goto("/ui/usage"), _usage_settled),
|
||||
Journey("usage_last_30_days", "usage", _open_usage_range, _pick_usage_preset("Last 30 days"), _usage_settled),
|
||||
Journey(
|
||||
"usage_team_view", "usage", _open_usage, lambda page: _select_usage_view(page, "Team Usage"), _usage_settled
|
||||
),
|
||||
Journey(
|
||||
"usage_customer_view",
|
||||
"usage",
|
||||
_open_usage,
|
||||
lambda page: _select_usage_view(page, "Customer Usage"),
|
||||
_usage_settled,
|
||||
),
|
||||
Journey(
|
||||
"usage_model_activity_30_days",
|
||||
"usage",
|
||||
_open_model_activity_range,
|
||||
_pick_usage_preset("Last 30 days"),
|
||||
_model_activity_settled,
|
||||
),
|
||||
Journey("cache_open", "caching", _nothing, _goto("/ui/caching"), _cache_settled),
|
||||
Journey("models_open", "models", _nothing, _goto("/ui/models-and-endpoints"), _table_settled(LOADING_MODELS)),
|
||||
Journey("keys_open", "keys", _nothing, _goto("/ui/api-keys"), _table_settled(LOADING_KEYS)),
|
||||
)
|
||||
|
||||
|
||||
def database_failures(text: str) -> float:
|
||||
families: Final = tuple(text_string_to_metric_families(text))
|
||||
required: Final = frozenset(("litellm_postgres_failed_requests", "litellm_batch_write_to_db_failed_requests"))
|
||||
missing: Final = required - frozenset(family.name for family in families)
|
||||
all_samples: Final = tuple(chain.from_iterable(family.samples for family in families))
|
||||
has_activity: Final = any(
|
||||
sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0
|
||||
for sample in all_samples
|
||||
)
|
||||
assert not missing or has_activity, f"Missing PostgreSQL failure counters {missing}"
|
||||
names: Final = frozenset(f"{name}_total" for name in required)
|
||||
values: Final = tuple(sample.value for sample in all_samples if sample.name in names)
|
||||
assert all(math.isfinite(value) and value >= 0 for value in values), "Invalid PostgreSQL failure counter"
|
||||
return sum(values)
|
||||
186
tests/e2e/perf/setup/ui_query_fixture.py
Normal file
186
tests/e2e/perf/setup/ui_query_fixture.py
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timedelta
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
import psycopg
|
||||
from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
from setup.ui_query_isolation import validate_target
|
||||
from setup.upgrade_worker import read_state, release_indexes
|
||||
|
||||
|
||||
class UiFixture(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
anchor: AwareDatetime
|
||||
api_key: str
|
||||
"""`perf_ui_db_*` JUnit properties naming how much data the journeys ran against."""
|
||||
volume: tuple[tuple[str, int], ...] = ()
|
||||
|
||||
|
||||
class TableVolume(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
table: str
|
||||
rows: int
|
||||
bytes: int
|
||||
|
||||
|
||||
QUERIED_TABLES: Final = (
|
||||
"LiteLLM_SpendLogs",
|
||||
"LiteLLM_DailyUserSpend",
|
||||
"LiteLLM_DailyTeamSpend",
|
||||
"LiteLLM_DailyTagSpend",
|
||||
"LiteLLM_VerificationToken",
|
||||
"LiteLLM_TeamTable",
|
||||
"LiteLLM_UserTable",
|
||||
"LiteLLM_ProxyModelTable",
|
||||
)
|
||||
_VOLUME_SQL: Final = """
|
||||
SELECT c.relname, GREATEST(c.reltuples, 0)::bigint, pg_total_relation_size(c.oid)
|
||||
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public' AND c.relname = ANY(%(tables)s)
|
||||
"""
|
||||
|
||||
|
||||
SEEDED_TEAMS: Final = 100
|
||||
SEEDED_USERS: Final = 2000
|
||||
SEEDED_KEYS: Final = 5000
|
||||
SEEDED_MODELS: Final = 200
|
||||
DAILY_TABLES: Final = ("LiteLLM_DailyUserSpend", "LiteLLM_DailyTeamSpend")
|
||||
_SEED_SQL: Final = (
|
||||
"""
|
||||
INSERT INTO "LiteLLM_TeamTable" (team_id, team_alias, members_with_roles, metadata, model_spend, model_max_budget)
|
||||
SELECT 'perf-team-' || n, 'perf-team-' || n, '[]', '{}', '{}', '{}' FROM generate_series(0, %(teams)s - 1) AS n
|
||||
ON CONFLICT (team_id) DO NOTHING
|
||||
""",
|
||||
"""
|
||||
INSERT INTO "LiteLLM_UserTable" (user_id, user_alias, user_email, user_role, team_id, metadata, model_spend, model_max_budget)
|
||||
SELECT 'perf-user-' || n, 'perf-user-' || n, 'perf-user-' || n || '@example.com', 'internal_user',
|
||||
'perf-team-' || (n %% %(teams)s), '{}', '{}', '{}'
|
||||
FROM generate_series(0, %(users)s - 1) AS n
|
||||
ON CONFLICT (user_id) DO NOTHING
|
||||
""",
|
||||
"""
|
||||
INSERT INTO "LiteLLM_VerificationToken" (token, key_name, key_alias, team_id, user_id, aliases, config, permissions,
|
||||
metadata, model_spend, model_max_budget, created_at, updated_at)
|
||||
SELECT md5('perf-key:' || n) || md5('perf-key-tail:' || n), 'sk-...perf' || n, 'perf-key-' || n,
|
||||
'perf-team-' || (n %% %(teams)s), 'perf-user-' || (n %% %(users)s), '{}', '{}', '{}', '{}', '{}', '{}',
|
||||
%(anchor)s - make_interval(days => n %% 365), %(anchor)s
|
||||
FROM generate_series(0, %(keys)s - 1) AS n
|
||||
ON CONFLICT (token) DO NOTHING
|
||||
""",
|
||||
"""
|
||||
INSERT INTO "LiteLLM_ProxyModelTable" (model_id, model_name, litellm_params, model_info, created_by, updated_by)
|
||||
SELECT 'perf-model-' || n, 'perf-chat-' || (n %% 20),
|
||||
jsonb_build_object('model', 'openai/perf-chat', 'api_key', 'perf-seed',
|
||||
'api_base', 'os.environ/E2E_PERF_MOCK_OPENAI_URL'),
|
||||
jsonb_build_object('id', 'perf-model-' || n, 'db_model', true), 'perf-seed', 'perf-seed'
|
||||
FROM generate_series(0, %(models)s - 1) AS n
|
||||
ON CONFLICT (model_id) DO NOTHING
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def volume_properties(volumes: Iterable[TableVolume]) -> tuple[tuple[str, int], ...]:
|
||||
"""One `perf_ui_db_rows_<table>` and one `perf_ui_db_bytes_<table>` property per table the journeys read,
|
||||
so the dashboard can say what the latencies were measured against."""
|
||||
return tuple(
|
||||
chain.from_iterable(
|
||||
((f"perf_ui_db_rows_{v.table}", v.rows), (f"perf_ui_db_bytes_{v.table}", v.bytes)) for v in volumes
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def table_volumes(connection: psycopg.Connection, spend_log_rows: int) -> tuple[TableVolume, ...]:
|
||||
"""Planner row estimates and on-disk size (indexes included) of the queried tables. The restore's
|
||||
`LiteLLM_SpendLogs` count comes from the seed checkpoint, exact where the estimate can be stale."""
|
||||
adapter: Final = TypeAdapter(tuple[str, int, int])
|
||||
found: Final = tuple(
|
||||
adapter.validate_python(row)
|
||||
for row in connection.execute(_VOLUME_SQL, {"tables": list(QUERIED_TABLES)}).fetchall()
|
||||
)
|
||||
return tuple(
|
||||
TableVolume(table=table, rows=spend_log_rows if table == "LiteLLM_SpendLogs" else rows, bytes=size)
|
||||
for table, rows, size in sorted(found)
|
||||
)
|
||||
|
||||
|
||||
def prepare_fixture() -> UiFixture:
|
||||
validate_target(os.environ["DATABASE_URL"])
|
||||
_finish_upgrade()
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], connect_timeout=10) as connection:
|
||||
_ = connection.execute("SET statement_timeout='30s'")
|
||||
row: Final = connection.execute("SELECT spec->>'anchor', rows FROM litellm_perf_seed.checkpoint").fetchone()
|
||||
assert row is not None and TypeAdapter(int).validate_python(row[1]) > 0, "Missing completed seed rows"
|
||||
seed_ui_tables(connection, anchor=TypeAdapter(datetime).validate_python(row[0]))
|
||||
fixture: Final = UiFixture(
|
||||
anchor=TypeAdapter(datetime).validate_python(row[0]),
|
||||
api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest()
|
||||
+ hashlib.md5(b"perf-key-tail:0", usedforsecurity=False).hexdigest(),
|
||||
volume=volume_properties(table_volumes(connection, TypeAdapter(int).validate_python(row[1]))),
|
||||
)
|
||||
return fixture
|
||||
|
||||
|
||||
def seed_ui_tables(connection: psycopg.Connection, anchor: datetime) -> None:
|
||||
"""Give the Keys, Teams, Users and Models pages realistic row counts on the disposable restore.
|
||||
|
||||
Names follow the SpendLogs seed (perf-key-N aliases, perf-team-N, perf-user-N) so the Logs filters and
|
||||
Team Usage line up with real spend. The Daily* rollups come from the snapshot itself and are only checked.
|
||||
"""
|
||||
window: Final = {
|
||||
"since": (anchor - timedelta(days=7)).strftime("%Y-%m-%d"),
|
||||
"until": anchor.strftime("%Y-%m-%d"),
|
||||
}
|
||||
for table in DAILY_TABLES:
|
||||
row = connection.execute(f'SELECT count(*) FROM "{table}" WHERE date BETWEEN %(since)s AND %(until)s', window)
|
||||
assert _count(row.fetchone()) > 0, f"{table} has no rows in the week before the anchor"
|
||||
params: Final = {
|
||||
"teams": SEEDED_TEAMS,
|
||||
"users": SEEDED_USERS,
|
||||
"keys": SEEDED_KEYS,
|
||||
"models": SEEDED_MODELS,
|
||||
"anchor": anchor,
|
||||
}
|
||||
for statement in _SEED_SQL:
|
||||
_ = connection.execute(statement, params)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def _count(row: tuple[object, ...] | None) -> int:
|
||||
assert row is not None
|
||||
return TypeAdapter(int).validate_python(row[0])
|
||||
|
||||
|
||||
def _released_schema(connection: psycopg.Connection) -> bool:
|
||||
"""A released tag's migrations image upgrades the restore (schema and request-log indexes) before the
|
||||
stack starts, so there is no upgrade worker state to wait on."""
|
||||
row: Final = connection.execute("""SELECT to_regclass('public."_LiteLLMPerfUpgrade"') IS NULL""").fetchone()
|
||||
return row is None or TypeAdapter(bool).validate_python(row[0])
|
||||
|
||||
|
||||
def _finish_upgrade() -> None:
|
||||
if os.environ.get("E2E_DATABASE_UPGRADE") != "1":
|
||||
return
|
||||
with psycopg.connect(os.environ["DATABASE_URL"], autocommit=True, connect_timeout=10) as connection:
|
||||
if _released_schema(connection):
|
||||
return
|
||||
release_indexes(connection)
|
||||
deadline: Final = time.monotonic() + 6 * 3600
|
||||
while time.monotonic() < deadline:
|
||||
match read_state(connection).phase:
|
||||
case "failed":
|
||||
raise AssertionError("Snapshot upgrade failed")
|
||||
case "complete":
|
||||
return
|
||||
case _:
|
||||
pass
|
||||
time.sleep(5)
|
||||
raise AssertionError("Snapshot indexes did not finish before UI measurements")
|
||||
112
tests/e2e/perf/setup/ui_query_isolation.py
Normal file
112
tests/e2e/perf/setup/ui_query_isolation.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Generator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import psycopg
|
||||
import pytest
|
||||
from psycopg.rows import dict_row
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
|
||||
class ActiveQuery(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
pid: int
|
||||
leader_pid: int | None
|
||||
application_name: str
|
||||
state: str | None
|
||||
wait_event_type: str | None
|
||||
wait_event: str | None
|
||||
seconds: float | None
|
||||
query: str
|
||||
|
||||
|
||||
def validate_target(url: str) -> None:
|
||||
assert os.environ.get("E2E_UI_PERF_DISPOSABLE") == "1", "Disposable database confirmation required"
|
||||
assert os.environ.get("E2E_RESET_SPEND_LOGS") != "1", "Never truncate the snapshot baseline"
|
||||
assert re.fullmatch(r"db-perf-perf-[a-z0-9-]+\.litellm\.svc", urlparse(url).hostname or ""), (
|
||||
"SQL cancellation is restricted to the disposable perf-lane restore"
|
||||
)
|
||||
|
||||
|
||||
def active(connection: psycopg.Connection[dict[str, object]]) -> tuple[ActiveQuery, ...]:
|
||||
rows: Final = connection.execute("""SELECT pid, leader_pid, application_name, state,
|
||||
wait_event_type, wait_event, extract(epoch from clock_timestamp()-query_start)::float AS seconds,
|
||||
left(query, 1500) AS query FROM pg_stat_activity
|
||||
WHERE datname = current_database() AND pid <> pg_backend_pid()
|
||||
AND backend_type IN ('client backend', 'parallel worker')
|
||||
AND state IS DISTINCT FROM 'idle' ORDER BY pid""").fetchall()
|
||||
return TypeAdapter(tuple[ActiveQuery, ...]).validate_python(rows)
|
||||
|
||||
|
||||
def wait_idle(connection: psycopg.Connection[dict[str, object]], evidence: Path, *, cancel: bool) -> None:
|
||||
consecutive = 0
|
||||
for queries in (active(connection) for _ in range(30)):
|
||||
with evidence.open("a", encoding="utf-8") as output:
|
||||
_ = output.write(
|
||||
json.dumps(
|
||||
{"time": time.time(), "cancel": cancel, "active": tuple(query.model_dump() for query in queries)}
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
if queries:
|
||||
consecutive = 0 # rebind-ok: consecutive idle observations reset on activity
|
||||
for query in queries:
|
||||
if cancel and query.leader_pid is None:
|
||||
_ = connection.execute(
|
||||
"SELECT pg_cancel_backend(%s)"
|
||||
if query.state == "active"
|
||||
else "SELECT pg_terminate_backend(%s)",
|
||||
(query.pid,),
|
||||
)
|
||||
else:
|
||||
consecutive += 1 # rebind-ok: consecutive idle observations across polling iterations
|
||||
if consecutive == 3:
|
||||
return
|
||||
time.sleep(1)
|
||||
pytest.exit("Database did not become idle. No further UI measurements will run", returncode=2)
|
||||
|
||||
|
||||
def settled_counter(read: Callable[[], float], evidence: Path) -> float:
|
||||
previous: float | None = None
|
||||
consecutive = 0
|
||||
for value in (read() for _ in range(10)):
|
||||
with evidence.open("a", encoding="utf-8") as output:
|
||||
_ = output.write(json.dumps({"time": time.time(), "database_failures": value}) + "\n")
|
||||
consecutive = consecutive + 1 if value == previous else 1 # rebind-ok: polling state
|
||||
if consecutive == 3:
|
||||
return value
|
||||
previous = value # rebind-ok: last observed counter for stabilization
|
||||
time.sleep(1)
|
||||
pytest.exit("Database counters did not settle. No further UI measurements will run", returncode=2)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def isolated_database(name: str) -> Generator[None]:
|
||||
url: Final = os.environ["DATABASE_URL"]
|
||||
validate_target(url)
|
||||
evidence: Final = Path(os.environ["E2E_ARTIFACT_DIR"]) / f"ui-isolation-{name}.jsonl"
|
||||
with psycopg.Connection[dict[str, object]].connect(
|
||||
url,
|
||||
autocommit=True,
|
||||
connect_timeout=10,
|
||||
application_name="ui-isolation-monitor",
|
||||
options="-c statement_timeout=5000",
|
||||
row_factory=dict_row,
|
||||
) as connection:
|
||||
try:
|
||||
wait_idle(connection, evidence, cancel=False)
|
||||
yield
|
||||
finally:
|
||||
try:
|
||||
wait_idle(connection, evidence, cancel=True)
|
||||
except Exception as error:
|
||||
pytest.exit(f"SQL cleanup failed ({type(error).__name__}). Stopping UI measurements", returncode=2)
|
||||
223
tests/e2e/perf/setup/upgrade.py
Normal file
223
tests/e2e/perf/setup/upgrade.py
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
import psycopg
|
||||
from cases.small_bodies_5rps import CASE
|
||||
from e2e_config import MASTER_KEY
|
||||
from e2e_http import send_text_external
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
||||
|
||||
from setup.performance_test_case import PerformanceTestCase
|
||||
from setup.upgrade_worker import UpgradeState, fingerprints, missing_indexes, read_state, release_indexes
|
||||
from setup.wire import WireRecord
|
||||
|
||||
MAX_TRAFFIC_SECONDS: Final = 6 * 3600
|
||||
_ROWS: Final = TypeAdapter(tuple[tuple[str, str | None], ...])
|
||||
_ACTIVITY: Final = TypeAdapter(tuple[list[int], int])
|
||||
_COUNT: Final = TypeAdapter(tuple[int])
|
||||
_SERVICES: Final = ("postgres", "batch_write_to_db")
|
||||
|
||||
|
||||
def upgrade_case() -> PerformanceTestCase:
|
||||
return replace(
|
||||
CASE,
|
||||
name="database_upgrade_5rps",
|
||||
traffic=replace(CASE.traffic, total_requests=int(CASE.traffic.rps * MAX_TRAFFIC_SECONDS)),
|
||||
limits=tuple(
|
||||
limit
|
||||
for limit in CASE.limits
|
||||
if limit.metric
|
||||
in {
|
||||
"unexpected_status_count",
|
||||
"gateway_added_seconds",
|
||||
"time_to_first_token_seconds",
|
||||
"memory_growth_megabytes",
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class MetricsHeaders(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
authorization: str = Field(default=f"Bearer {MASTER_KEY}", alias="Authorization")
|
||||
|
||||
|
||||
def postgres_failures(text: str) -> float:
|
||||
families: Final = tuple(text_string_to_metric_families(text))
|
||||
names: Final = frozenset(family.name for family in families)
|
||||
required: Final = frozenset(f"litellm_{service}_failed_requests" for service in _SERVICES)
|
||||
samples: Final = tuple(sample for family in families for sample in family.samples)
|
||||
has_database_activity: Final = any(
|
||||
sample.name == "litellm_postgres_total_requests_total" and math.isfinite(sample.value) and sample.value > 0
|
||||
for sample in samples
|
||||
)
|
||||
assert required <= names or has_database_activity, f"Missing database service counters: {required - names}"
|
||||
failures: Final = tuple(sample.value for sample in samples if sample.name in {f"{name}_total" for name in required})
|
||||
assert all(math.isfinite(value) and value >= 0 for value in failures), "Invalid database failure counter"
|
||||
return sum(failures)
|
||||
|
||||
|
||||
def assert_database_metrics(urls: tuple[str, ...]) -> None:
|
||||
assert urls, "E2E_DATABASE_METRICS_URLS must name every gateway/backend /metrics endpoint"
|
||||
for url in urls:
|
||||
assert_database_metrics_url(url)
|
||||
|
||||
|
||||
def assert_database_metrics_url(url: str) -> None:
|
||||
result: Final = send_text_external("GET", url, headers=MetricsHeaders(), timeout=10)
|
||||
assert result.status_code == 200, f"Database metrics unavailable at {url}: {result.status_code}"
|
||||
assert postgres_failures(result.body) == 0, f"Failed PostgreSQL operations reported at {url}"
|
||||
|
||||
|
||||
def persisted(connection: psycopg.Connection[tuple[object, ...]], records: tuple[WireRecord, ...]) -> frozenset[str]:
|
||||
ids: Final = tuple(record.call_id for record in records if record.call_id)
|
||||
if not ids:
|
||||
return frozenset()
|
||||
rows: Final = _ROWS.validate_python(
|
||||
connection.execute(
|
||||
'SELECT request_id, litellm_call_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=ANY(%s)', (list(ids),)
|
||||
).fetchall()
|
||||
)
|
||||
return matched_response_ids(records, rows)
|
||||
|
||||
|
||||
def matched_response_ids(records: tuple[WireRecord, ...], rows: tuple[tuple[str, str | None], ...]) -> frozenset[str]:
|
||||
call_ids: Final = frozenset(call_id for _, call_id in rows if call_id)
|
||||
return frozenset(record.response_id for record in records if record.response_id and record.call_id in call_ids)
|
||||
|
||||
|
||||
def activity(connection: psycopg.Connection[tuple[object, ...]]) -> tuple[list[int], int]:
|
||||
return _ACTIVITY.validate_python(
|
||||
connection.execute("""SELECT
|
||||
ARRAY(SELECT pid FROM pg_stat_progress_create_index WHERE datname=current_database()),
|
||||
(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock')
|
||||
""").fetchone()
|
||||
)
|
||||
|
||||
|
||||
def assert_finished(connection: psycopg.Connection[tuple[object, ...]], state: UpgradeState) -> None:
|
||||
assert state.phase == "complete", f"Upgrade did not finish: {state.phase}"
|
||||
assert not missing_indexes(connection, state.required_indexes), "Required target indexes are missing or invalid"
|
||||
(missing,) = _COUNT.validate_python(
|
||||
connection.execute(
|
||||
"""
|
||||
SELECT count(*) FROM unnest(%s::text[]) name WHERE NOT EXISTS (
|
||||
SELECT 1 FROM "_prisma_migrations" WHERE migration_name=name
|
||||
AND finished_at IS NOT NULL AND rolled_back_at IS NULL)""",
|
||||
(list(state.required_migrations),),
|
||||
).fetchone()
|
||||
)
|
||||
assert missing == 0, f"{missing} target migrations were not applied"
|
||||
(unfinished,) = _COUNT.validate_python(
|
||||
connection.execute("""
|
||||
SELECT count(*) FROM "_prisma_migrations" WHERE finished_at IS NULL AND rolled_back_at IS NULL
|
||||
""").fetchone()
|
||||
)
|
||||
assert unfinished == 0, f"{unfinished} unfinished migrations"
|
||||
assert fingerprints(connection, tuple(state.baseline_rows)) == state.baseline_rows, (
|
||||
"Baseline rows changed or disappeared"
|
||||
)
|
||||
|
||||
|
||||
class UpgradeMonitor:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
state: Callable[[], UpgradeState],
|
||||
check_metrics: Callable[[], None],
|
||||
read_activity: Callable[[], tuple[list[int], int]],
|
||||
read_persisted: Callable[[tuple[WireRecord, ...]], frozenset[str]],
|
||||
start_indexes: Callable[[], None],
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
wall_clock: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
self.state: Final = state
|
||||
self.check_metrics: Final = check_metrics
|
||||
self.read_activity: Final = read_activity
|
||||
self.read_persisted: Final = read_persisted
|
||||
self.start_indexes: Final = start_indexes
|
||||
self.clock: Final = clock
|
||||
self.wall_clock: Final = wall_clock
|
||||
self.started: Final = clock()
|
||||
self.status_limit: Final = next(
|
||||
limit.max for limit in CASE.limits if limit.metric == "unexpected_status_count" and limit.max is not None
|
||||
)
|
||||
self.pending: tuple[WireRecord, ...] = ()
|
||||
self.processed = 0
|
||||
self.successes = 0
|
||||
self.persisted_count = 0
|
||||
self.overlap_writes = 0
|
||||
self.max_lock_waiters = 0
|
||||
self.previous_builds: frozenset[int] = frozenset()
|
||||
self.build_seen_at: float | None = None
|
||||
self.released = False
|
||||
self.completed_at: float | None = None
|
||||
|
||||
def poll(self, records: tuple[WireRecord, ...]) -> bool:
|
||||
self.check_metrics()
|
||||
current: Final = self.state()
|
||||
assert current.phase != "failed", "Target migration/index worker failed"
|
||||
assert self.clock() - self.started < MAX_TRAFFIC_SECONDS, "Upgrade exceeded its six-hour operational timeout"
|
||||
successful: Final = tuple(record for record in records if record.status == 200)
|
||||
self.processed += len(records)
|
||||
self.successes += len(successful)
|
||||
assert self.processed - self.successes <= self.status_limit, "Too many unexpected non-200 responses"
|
||||
assert all(record.ok and record.text for record in successful), "Failed or empty inference response"
|
||||
assert all(record.response_id and record.call_id for record in successful), (
|
||||
"Response missing spend-log identifiers"
|
||||
)
|
||||
builds, lock_waiters = self.read_activity()
|
||||
if not self.previous_builds.intersection(builds):
|
||||
self.build_seen_at = self.wall_clock() if builds else None
|
||||
self.max_lock_waiters = max(self.max_lock_waiters, lock_waiters)
|
||||
self.pending += successful
|
||||
found: Final = self.read_persisted(self.pending)
|
||||
self.persisted_count += len(found)
|
||||
still_building, _ = self.read_activity()
|
||||
since: Final = self.build_seen_at
|
||||
if since is not None and frozenset(builds).intersection(still_building):
|
||||
self.overlap_writes += sum(
|
||||
1 for record in self.pending if record.response_id in found and record.started_at >= since
|
||||
)
|
||||
self.previous_builds = frozenset(builds)
|
||||
self.pending = tuple(record for record in self.pending if record.response_id not in found)
|
||||
if self.successes and not self.released:
|
||||
self.start_indexes()
|
||||
self.released = True
|
||||
if current.phase == "complete" and self.completed_at is None:
|
||||
self.completed_at = self.clock()
|
||||
return self.completed_at is not None and self.clock() - self.completed_at >= 60 and self.successes >= 300
|
||||
|
||||
def assert_complete(self, state: UpgradeState) -> None:
|
||||
assert self.successes >= 300, f"Only {self.successes} requests completed"
|
||||
assert not self.pending, f"{len(self.pending)} successful requests have no matching spend log"
|
||||
assert self.persisted_count == self.successes, "Spend-log count does not match successful requests"
|
||||
assert not state.pending_indexes or self.overlap_writes > 0, "No persisted requests observed during index work"
|
||||
|
||||
|
||||
def upgrade_properties(state: UpgradeState, monitor: UpgradeMonitor) -> tuple[tuple[str, float], ...]:
|
||||
"""The upgrade's own numbers under the `perf_` prefix the dashboard ingests, next to the perf rows."""
|
||||
return (
|
||||
("perf_upgrade_schema_seconds", state.schema_seconds),
|
||||
("perf_upgrade_index_seconds", state.index_seconds),
|
||||
("perf_upgrade_overlap_writes", float(monitor.overlap_writes)),
|
||||
("perf_upgrade_max_lock_waiters", float(monitor.max_lock_waiters)),
|
||||
("perf_upgrade_pending_indexes", float(len(state.pending_indexes))),
|
||||
("perf_upgrade_unmatched_requests", float(len(monitor.pending))),
|
||||
)
|
||||
|
||||
|
||||
def monitor_for(connection: psycopg.Connection[tuple[object, ...]], urls: tuple[str, ...]) -> UpgradeMonitor:
|
||||
return UpgradeMonitor(
|
||||
state=lambda: read_state(connection),
|
||||
check_metrics=lambda: assert_database_metrics(urls),
|
||||
read_activity=lambda: activity(connection),
|
||||
read_persisted=lambda records: persisted(connection, records),
|
||||
start_indexes=lambda: release_indexes(connection),
|
||||
)
|
||||
146
tests/e2e/perf/setup/upgrade_worker.py
Normal file
146
tests/e2e/perf/setup/upgrade_worker.py
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
|
||||
import psycopg
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
|
||||
class UpgradeState(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
phase: Literal["schema", "schema_ready", "indexes", "complete", "failed"]
|
||||
started_at: float
|
||||
schema_seconds: float = 0
|
||||
index_seconds: float = 0
|
||||
required_indexes: tuple[str, ...]
|
||||
required_migrations: tuple[str, ...]
|
||||
pending_indexes: tuple[str, ...]
|
||||
baseline_rows: dict[str, str]
|
||||
|
||||
|
||||
_STATE: Final = TypeAdapter(tuple[UpgradeState])
|
||||
_BOOL: Final = TypeAdapter(tuple[bool])
|
||||
_STRINGS: Final = TypeAdapter(tuple[tuple[str], ...])
|
||||
_FINGERPRINTS: Final = TypeAdapter(tuple[tuple[str, str], ...])
|
||||
_FINGERPRINT: Final = """md5(ROW(request_id, api_key, spend, total_tokens, prompt_tokens,
|
||||
completion_tokens, model, "startTime", "endTime")::text)"""
|
||||
|
||||
|
||||
def connect() -> psycopg.Connection[tuple[object, ...]]:
|
||||
return psycopg.connect(
|
||||
os.environ["DATABASE_URL"],
|
||||
autocommit=True,
|
||||
connect_timeout=10,
|
||||
options="-c statement_timeout=10000 -c timezone=UTC",
|
||||
)
|
||||
|
||||
|
||||
def fingerprints(connection: psycopg.Connection[tuple[object, ...]], ids: tuple[str, ...]) -> dict[str, str]:
|
||||
rows: Final = connection.execute(
|
||||
f'SELECT request_id, {_FINGERPRINT} FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s)', (list(ids),)
|
||||
).fetchall()
|
||||
return dict(_FINGERPRINTS.validate_python(rows))
|
||||
|
||||
|
||||
def missing_indexes(connection: psycopg.Connection[tuple[object, ...]], names: tuple[str, ...]) -> tuple[str, ...]:
|
||||
rows: Final = connection.execute(
|
||||
"""SELECT name FROM unnest(%s::text[]) AS name WHERE NOT EXISTS (
|
||||
SELECT 1 FROM pg_index WHERE indexrelid = to_regclass(format('public.%%I', name))
|
||||
AND indisvalid AND indisready)""",
|
||||
(list(names),),
|
||||
).fetchall()
|
||||
return tuple(row[0] for row in _STRINGS.validate_python(rows))
|
||||
|
||||
|
||||
def read_state(connection: psycopg.Connection[tuple[object, ...]]) -> UpgradeState:
|
||||
(state,) = _STATE.validate_python(
|
||||
connection.execute('SELECT state FROM "_LiteLLMPerfUpgrade" WHERE id=1').fetchone()
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
def write_state(connection: psycopg.Connection[tuple[object, ...]], state: UpgradeState) -> None:
|
||||
_ = connection.execute(
|
||||
"""INSERT INTO "_LiteLLMPerfUpgrade" (id, state) VALUES (1, %s::jsonb)
|
||||
ON CONFLICT (id) DO UPDATE SET state = EXCLUDED.state""",
|
||||
(state.model_dump_json(),),
|
||||
)
|
||||
|
||||
|
||||
def release_indexes(connection: psycopg.Connection[tuple[object, ...]]) -> None:
|
||||
_ = connection.execute('UPDATE "_LiteLLMPerfUpgrade" SET traffic_ready=true WHERE id=1')
|
||||
|
||||
|
||||
def wait_for_traffic(connection: psycopg.Connection[tuple[object, ...]], deadline: float) -> bool:
|
||||
while time.monotonic() < deadline:
|
||||
(ready,) = _BOOL.validate_python(
|
||||
connection.execute('SELECT traffic_ready FROM "_LiteLLMPerfUpgrade" WHERE id=1').fetchone()
|
||||
)
|
||||
if ready:
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def main() -> int:
|
||||
assert os.environ.get("E2E_DATABASE_UPGRADE") == "1", "Only run against a disposable restored baseline"
|
||||
assert os.environ.get("DATABASE_PERF_CONFIRM_DISPOSABLE") == "1", "Disposable database confirmation required"
|
||||
from litellm_proxy_extras import request_log_indexes
|
||||
from litellm_proxy_extras.utils import ProxyExtrasDBManager
|
||||
|
||||
with connect() as connection:
|
||||
_ = connection.execute("""CREATE TABLE "_LiteLLMPerfUpgrade" (
|
||||
id integer PRIMARY KEY CHECK (id=1), state jsonb NOT NULL, traffic_ready boolean NOT NULL DEFAULT false)""")
|
||||
sample: Final = connection.execute("""
|
||||
(SELECT request_id FROM "LiteLLM_SpendLogs" ORDER BY request_id LIMIT 32)
|
||||
UNION (SELECT request_id FROM "LiteLLM_SpendLogs" ORDER BY request_id DESC LIMIT 32)""").fetchall()
|
||||
ids: Final = tuple(row[0] for row in _STRINGS.validate_python(sample))
|
||||
assert ids, "Restored baseline has no spend logs"
|
||||
names: Final = tuple(index.name for index in request_log_indexes.REQUEST_LOG_INDEXES)
|
||||
migrations: Final = Path(request_log_indexes.__file__).parent / "migrations"
|
||||
initial: Final = UpgradeState(
|
||||
phase="schema",
|
||||
started_at=time.time(),
|
||||
required_indexes=names,
|
||||
required_migrations=tuple(sorted(path.parent.name for path in migrations.glob("*/migration.sql"))),
|
||||
pending_indexes=missing_indexes(connection, names),
|
||||
baseline_rows=fingerprints(connection, ids),
|
||||
)
|
||||
assert initial.required_migrations and initial.required_indexes, "Target schema requirements are missing"
|
||||
write_state(connection, initial)
|
||||
try:
|
||||
assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True), (
|
||||
"Schema migration failed"
|
||||
)
|
||||
ready: Final = initial.model_copy(
|
||||
update={
|
||||
"phase": "schema_ready",
|
||||
"schema_seconds": time.time() - initial.started_at,
|
||||
"pending_indexes": missing_indexes(connection, names),
|
||||
}
|
||||
)
|
||||
write_state(connection, ready)
|
||||
assert wait_for_traffic(connection, time.monotonic() + 1800), "Traffic did not start within 30 minutes"
|
||||
write_state(connection, ready.model_copy(update={"phase": "indexes"}))
|
||||
index_start: Final = time.monotonic()
|
||||
assert ProxyExtrasDBManager.build_request_log_indexes(), "Request-log index build failed"
|
||||
complete: Final = ready.model_copy(
|
||||
update={"phase": "complete", "index_seconds": time.monotonic() - index_start}
|
||||
)
|
||||
write_state(connection, complete)
|
||||
_ = sys.stdout.write(complete.model_dump_json() + "\n")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
write_state(connection, read_state(connection).model_copy(update={"phase": "failed"}))
|
||||
_ = sys.stderr.write(json.dumps({"error": type(exc).__name__}) + "\n")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -9,12 +9,13 @@ from collections.abc import Mapping
|
|||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
||||
|
||||
from setup.corpus.session import Endpoint as CorpusEndpoint
|
||||
from setup.corpus.session import request_body as corpus_body
|
||||
|
||||
Endpoint = Literal["chat_completions", "responses", "messages"]
|
||||
_PAYLOAD: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
class TokenSource(BaseModel):
|
||||
|
|
@ -50,6 +51,8 @@ class WireRequest(BaseModel):
|
|||
token_source: TokenSource | None = None
|
||||
stream: bool
|
||||
blocked: bool
|
||||
send_at_s: float | None = None
|
||||
"""Seconds after the run starts to send this request, set when the case ramps; otherwise the users pace."""
|
||||
|
||||
|
||||
class WireRecord(BaseModel):
|
||||
|
|
@ -70,6 +73,35 @@ class WireRecord(BaseModel):
|
|||
mock_delay_s: float | None
|
||||
call_id: str | None
|
||||
error: str | None
|
||||
response_id: str | None = None
|
||||
litellm_overhead_ms: float | None = None
|
||||
litellm_response_ms: float | None = None
|
||||
|
||||
|
||||
def response_id(payload: Mapping[str, object]) -> str | None:
|
||||
nested: Final = payload.get("response") or payload.get("message")
|
||||
source: Final = _PAYLOAD.validate_python(nested) if isinstance(nested, Mapping) else payload
|
||||
identifier: Final = source.get("id")
|
||||
return identifier if isinstance(identifier, str) else None
|
||||
|
||||
|
||||
class PlanProgress:
|
||||
def __init__(self, total: int) -> None:
|
||||
self.total: Final = total
|
||||
self.issued = 0
|
||||
self.completed = 0
|
||||
self.stopped = False
|
||||
|
||||
def take(self, *, stop: bool) -> bool:
|
||||
self.stopped = self.stopped or stop
|
||||
if self.stopped or self.issued >= self.total:
|
||||
return False
|
||||
self.issued += 1
|
||||
return True
|
||||
|
||||
@property
|
||||
def drained(self) -> bool:
|
||||
return (self.stopped or self.issued == self.total) and self.completed == self.issued
|
||||
|
||||
|
||||
_CORPUS_ENDPOINT: Final[Mapping[Endpoint, CorpusEndpoint]] = MappingProxyType(
|
||||
|
|
|
|||
102
tests/e2e/perf/test_database_upgrade.py
Normal file
102
tests/e2e/perf/test_database_upgrade.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from idp import Keycloak
|
||||
from lifecycle import ResourceManager
|
||||
from memory_readings import read_rss_everywhere
|
||||
from proxy_client import ProxyClient
|
||||
from setup.datastore import read_db_counters, read_redis_counters
|
||||
from setup.observe import (
|
||||
Observation,
|
||||
evaluate,
|
||||
junit_properties,
|
||||
read_mock_requests,
|
||||
render,
|
||||
reset_mock,
|
||||
set_mock_latency,
|
||||
)
|
||||
from setup.plan import build_plan
|
||||
from setup.podcpu import gateway_cpu_sampler
|
||||
from setup.provision import MOCK_PROVIDERS, build_setup, mock_url
|
||||
from setup.run import run_plan
|
||||
from setup.upgrade import assert_finished, monitor_for, upgrade_case, upgrade_properties
|
||||
from setup.upgrade_worker import connect, read_state
|
||||
|
||||
pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack]
|
||||
|
||||
|
||||
@pytest.mark.skipif(os.environ.get("E2E_DATABASE_UPGRADE") != "1", reason="Dedicated disposable snapshot lane only")
|
||||
def test_database_upgrade(
|
||||
proxy: ProxyClient, idp: Keycloak, resources: ResourceManager, request: pytest.FixtureRequest
|
||||
) -> None:
|
||||
case: Final = upgrade_case()
|
||||
urls: Final = tuple(mock_url(provider) for provider in MOCK_PROVIDERS)
|
||||
for url in urls:
|
||||
_ = reset_mock(url)
|
||||
_ = set_mock_latency(url, case.mock_latency)
|
||||
setup: Final = build_setup(case, proxy=proxy, idp=idp, resources=resources)
|
||||
metrics: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip())
|
||||
with connect() as connection:
|
||||
assert read_state(connection).phase == "schema_ready", "Expected the target schema ready, index work waiting"
|
||||
rss_before: Final = read_rss_everywhere(proxy)
|
||||
db_before: Final = read_db_counters()
|
||||
redis_before: Final = read_redis_counters()
|
||||
monitor: Final = monitor_for(connection, metrics)
|
||||
with gateway_cpu_sampler() as cpu:
|
||||
outcome: Final = run_plan(case, build_plan(case), setup, stop_when=monitor.poll, request_timeout=30)
|
||||
assert outcome.locust_exit == 0, outcome.locust_output
|
||||
remaining: Final = outcome.records[monitor.processed :]
|
||||
_ = monitor.poll(remaining)
|
||||
deadline: Final = time.monotonic() + 120
|
||||
while monitor.pending and time.monotonic() < deadline:
|
||||
time.sleep(1)
|
||||
_ = monitor.poll(())
|
||||
state: Final = read_state(connection)
|
||||
_ = sys.stdout.write(
|
||||
f"successful_requests={monitor.successes} persisted_requests={monitor.persisted_count} "
|
||||
f"unmatched_requests={len(monitor.pending)}\n"
|
||||
)
|
||||
rss_after: Final = read_rss_everywhere(proxy)
|
||||
observed: Final = Observation(
|
||||
records=outcome.records,
|
||||
mock_requests=tuple(seen for url in urls for seen in read_mock_requests(url)),
|
||||
guardrail_calls=(),
|
||||
rss_before=rss_before,
|
||||
rss_after=rss_after,
|
||||
wall_seconds=outcome.wall_seconds,
|
||||
spend_logged_share=1,
|
||||
db_before=db_before,
|
||||
db_after=read_db_counters(),
|
||||
redis_before=redis_before,
|
||||
redis_after=read_redis_counters(),
|
||||
gateway_cpu_seconds=cpu.seconds,
|
||||
)
|
||||
rows, exceeded = evaluate(case, observed)
|
||||
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
|
||||
assert isinstance(item, pytest.Item)
|
||||
item.user_properties[:] = [
|
||||
entry for entry in item.user_properties if not entry[0].startswith("perf_")
|
||||
] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set
|
||||
item.user_properties.extend(junit_properties(rows))
|
||||
item.user_properties.extend(upgrade_properties(state, monitor))
|
||||
_ = sys.stdout.write(render(case, observed, rows) + "\n")
|
||||
_ = sys.stdout.write(
|
||||
f"schema_seconds={state.schema_seconds:.3f} index_seconds={state.index_seconds:.3f} "
|
||||
f"overlap_writes={monitor.overlap_writes} max_lock_waiters={monitor.max_lock_waiters} "
|
||||
f"pending_indexes={state.pending_indexes}\n"
|
||||
)
|
||||
artifact: Final = os.environ.get("E2E_ARTIFACT_DIR")
|
||||
if artifact:
|
||||
_ = (Path(artifact) / "database-upgrade.json").write_text(state.model_dump_json(), encoding="utf-8")
|
||||
monitor.assert_complete(state)
|
||||
assert_finished(connection, state)
|
||||
assert {(r.replica, r.hostname) for r in rss_before.readings} == {
|
||||
(r.replica, r.hostname) for r in rss_after.readings
|
||||
}, "Gateway pods replaced during the upgrade"
|
||||
assert not exceeded, "\n".join(exceeded)
|
||||
|
|
@ -5,7 +5,10 @@ clients saw, what the mocks received and how the workers' memory moved."""
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from datetime import UTC, datetime
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -15,7 +18,18 @@ from lifecycle import ResourceManager
|
|||
from memory_readings import read_rss_everywhere
|
||||
from models import SpendLogsParams
|
||||
from proxy_client import ProxyClient
|
||||
from setup.datastore import read_db_counters, read_redis_counters, redis_monitor
|
||||
from setup.datastore import connections_sampler, read_db_counters, read_redis_counters, redis_monitor
|
||||
from setup.event_loop import (
|
||||
LOG_FILE_ENV,
|
||||
OPT_IN_ENV,
|
||||
StallRow,
|
||||
offenders,
|
||||
parse_stalls,
|
||||
read_gateway_log,
|
||||
render_stalls,
|
||||
summarize,
|
||||
)
|
||||
from setup.event_loop import junit_properties as stall_properties
|
||||
from setup.observe import (
|
||||
Observation,
|
||||
evaluate,
|
||||
|
|
@ -26,14 +40,17 @@ from setup.observe import (
|
|||
reset_mock,
|
||||
set_mock_latency,
|
||||
)
|
||||
from setup.performance_test_case import PerformanceTestCase
|
||||
from setup.plan import build_plan
|
||||
from setup.performance_test_case import MockLatency, PerformanceTestCase
|
||||
from setup.plan import PlannedRequest, build_plan
|
||||
from setup.podcpu import gateway_cpu_sampler
|
||||
from setup.provision import MOCK_PROVIDERS, build_setup, guardrail_url, mock_url
|
||||
from setup.ramp import step_properties
|
||||
from setup.run import run_plan
|
||||
from setup.telemetry import read_gateway_telemetry, telemetry_sampler
|
||||
|
||||
pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack]
|
||||
_SPEND_SAMPLE: Final = 40
|
||||
_WARMUP_SECONDS: Final = 60
|
||||
|
||||
|
||||
def _spend_logged_share(proxy: ProxyClient, call_ids: tuple[str, ...]) -> float | None:
|
||||
|
|
@ -48,6 +65,31 @@ def _spend_logged_share(proxy: ProxyClient, call_ids: tuple[str, ...]) -> float
|
|||
return found / len(call_ids)
|
||||
|
||||
|
||||
def _warmup_plan(case: PerformanceTestCase, plan: tuple[PlannedRequest, ...]) -> tuple[PlannedRequest, ...]:
|
||||
"""The plan's first minute of traffic. It is sent before the memory baseline is read so a worker
|
||||
has shed its startup allocations and `memory_growth_megabytes` is what the measured traffic added,
|
||||
not startup churn (a lightly loaded case read negative growth)."""
|
||||
return plan[: max(1, round(case.traffic.rps * _WARMUP_SECONDS))]
|
||||
|
||||
|
||||
def _arm_mocks(mock_urls: Mapping[str, str], latency: MockLatency) -> None:
|
||||
for url in (*mock_urls.values(), guardrail_url()):
|
||||
_ = reset_mock(url)
|
||||
for url in mock_urls.values():
|
||||
_ = set_mock_latency(url, latency)
|
||||
|
||||
|
||||
def _event_loop_rows(case: PerformanceTestCase, since: datetime) -> tuple[StallRow, ...]:
|
||||
"""The gateway's asyncio debug holds for the traffic window; a case that asks for them cannot pass without a log."""
|
||||
assert case.event_loop is not None
|
||||
lines: Final = read_gateway_log(since)
|
||||
assert lines is not None, (
|
||||
f"{case.name} needs the gateway log: run in the perf lane (pods/log with the runner service account) or set "
|
||||
f"{LOG_FILE_ENV} to the gateway's log file"
|
||||
)
|
||||
return summarize(case.event_loop, parse_stalls(lines))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES, ids=[case.name for case in CASES])
|
||||
def test_perf_case(
|
||||
case: PerformanceTestCase,
|
||||
|
|
@ -56,24 +98,36 @@ def test_perf_case(
|
|||
resources: ResourceManager,
|
||||
request: pytest.FixtureRequest,
|
||||
) -> None:
|
||||
if case.event_loop is not None and not os.environ.get(OPT_IN_ENV):
|
||||
pytest.skip(
|
||||
f"{case.name} reads asyncio debug output: deploy the gateway with PYTHONASYNCIODEBUG=1 and "
|
||||
f"LITELLM_EVENT_LOOP=asyncio, then set {OPT_IN_ENV}=1"
|
||||
)
|
||||
plan: Final = build_plan(case)
|
||||
mock_urls: Final = {provider: mock_url(provider) for provider in MOCK_PROVIDERS}
|
||||
for url in (*mock_urls.values(), guardrail_url()):
|
||||
_ = reset_mock(url)
|
||||
for url in mock_urls.values():
|
||||
_ = set_mock_latency(url, case.mock_latency)
|
||||
_arm_mocks(mock_urls, case.mock_latency)
|
||||
setup_started: Final = time.monotonic()
|
||||
setup: Final = build_setup(case, proxy=proxy, idp=idp, resources=resources)
|
||||
_ = run_plan(case, _warmup_plan(case, plan), setup)
|
||||
_arm_mocks(mock_urls, case.mock_latency)
|
||||
rss_before: Final = read_rss_everywhere(proxy)
|
||||
telemetry_before: Final = read_gateway_telemetry(proxy)
|
||||
db_before: Final = read_db_counters()
|
||||
redis_before: Final = read_redis_counters()
|
||||
run_started: Final = time.monotonic()
|
||||
with gateway_cpu_sampler() as cpu, redis_monitor() as monitor:
|
||||
traffic_started_at: Final = datetime.now(tz=UTC)
|
||||
with (
|
||||
gateway_cpu_sampler() as cpu,
|
||||
redis_monitor() as monitor,
|
||||
connections_sampler() as connections,
|
||||
telemetry_sampler(proxy) as telemetry,
|
||||
):
|
||||
outcome: Final = run_plan(case, plan, setup)
|
||||
run_finished: Final = time.monotonic()
|
||||
db_after: Final = read_db_counters()
|
||||
redis_after: Final = read_redis_counters()
|
||||
rss_after: Final = read_rss_everywhere(proxy)
|
||||
telemetry_after: Final = read_gateway_telemetry(proxy)
|
||||
rss_read: Final = time.monotonic()
|
||||
assert outcome.records, f"locust wrote no records (exit {outcome.locust_exit}):\n{outcome.locust_output}"
|
||||
served_call_ids: Final = tuple(
|
||||
|
|
@ -100,15 +154,28 @@ def test_perf_case(
|
|||
redis_after=redis_after,
|
||||
redis_commands=monitor.commands,
|
||||
gateway_cpu_seconds=cpu.seconds,
|
||||
telemetry_before=telemetry_before,
|
||||
telemetry_after=telemetry_after,
|
||||
peak_connections=connections.peak,
|
||||
ramp=case.traffic.ramp,
|
||||
cpu_series=cpu.series,
|
||||
connection_series=connections.series,
|
||||
telemetry_series=telemetry.series,
|
||||
)
|
||||
rows, exceeded_limits = evaluate(case, observation)
|
||||
stall_rows: Final = _event_loop_rows(case, traffic_started_at) if case.event_loop is not None else ()
|
||||
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
|
||||
assert isinstance(item, pytest.Item)
|
||||
item.user_properties[:] = [
|
||||
entry for entry in item.user_properties if not entry[0].startswith("perf_")
|
||||
] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set
|
||||
item.user_properties.extend(junit_properties(rows))
|
||||
item.user_properties.extend(
|
||||
stall_properties(case.event_loop, stall_rows) if case.event_loop is not None else junit_properties(rows)
|
||||
)
|
||||
item.user_properties.extend(step_properties(observation.steps()))
|
||||
print(render(case, observation, rows))
|
||||
if case.event_loop is not None:
|
||||
print(render_stalls(case.event_loop, stall_rows))
|
||||
print(
|
||||
f" phases: setup {run_started - setup_started:.0f}s, locust {run_finished - run_started:.0f}s, "
|
||||
f"observe {time.monotonic() - run_finished:.0f}s (rss {rss_read - run_finished:.0f}s, spend sample "
|
||||
|
|
@ -118,3 +185,4 @@ def test_perf_case(
|
|||
f"{len(plan)} planned, {len(outcome.records)} recorded; locust said:\n{outcome.locust_output}"
|
||||
)
|
||||
assert not exceeded_limits, "limits exceeded (observed > max):\n" + "\n".join(exceeded_limits)
|
||||
assert not offenders(stall_rows), "event-loop holds over the threshold:\n" + "\n".join(offenders(stall_rows))
|
||||
|
|
|
|||
137
tests/e2e/perf/test_ui_journeys.py
Normal file
137
tests/e2e/perf/test_ui_journeys.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import FrameType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME
|
||||
from e2e_http import send_text_external, without_retries
|
||||
from playwright.sync_api import Page
|
||||
from setup.ui_journeys import (
|
||||
Dashboard,
|
||||
Journey,
|
||||
Sample,
|
||||
assert_within_ceiling,
|
||||
dashboard_page,
|
||||
database_failures,
|
||||
error_code,
|
||||
journeys,
|
||||
measure,
|
||||
summarize,
|
||||
ui_junit_properties,
|
||||
)
|
||||
from setup.ui_query_fixture import UiFixture, prepare_fixture
|
||||
from setup.ui_query_isolation import isolated_database, settled_counter
|
||||
from transport import HttpTransport
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.perf,
|
||||
pytest.mark.quiet_stack,
|
||||
pytest.mark.skipif(os.environ.get("E2E_UI_PERF_DISPOSABLE") != "1", reason="Dedicated disposable database only"),
|
||||
]
|
||||
|
||||
|
||||
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
||||
if "journey_name" in metafunc.fixturenames:
|
||||
metafunc.parametrize("journey_name", [journey.name for journey in journeys("")])
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ui_fixture() -> UiFixture:
|
||||
return prepare_fixture()
|
||||
|
||||
|
||||
def _metrics(transport: HttpTransport) -> float:
|
||||
urls: Final = tuple(url.strip() for url in os.environ["E2E_DATABASE_METRICS_URLS"].split(",") if url.strip())
|
||||
try:
|
||||
assert urls, "Provide a metrics URL for every proxy worker/pod"
|
||||
readings: Final = tuple(send_text_external("GET", url, headers=transport.master, timeout=10) for url in urls)
|
||||
assert all(reading.status_code == 200 for reading in readings), "Cannot read database failure metrics"
|
||||
return sum(database_failures(reading.body) for reading in readings)
|
||||
except Exception as error:
|
||||
pytest.exit(f"Database metrics unavailable ({type(error).__name__}). Stopping UI measurements", returncode=2)
|
||||
|
||||
|
||||
def _samples(dashboard: Dashboard, journey: Journey, timeout: float, repeats: int) -> tuple[Sample, ...]:
|
||||
with dashboard_page(dashboard, timeout) as page:
|
||||
return tuple(_within_budget(page, journey, timeout, repeats))
|
||||
|
||||
|
||||
def _within_budget(page: Page, journey: Journey, timeout: float, repeats: int) -> Generator[Sample]:
|
||||
"""One hanging browser call must not eat the lane: SIGALRM ends the journey at timeout x (repeats + 2),
|
||||
the budget every sample needs (start plus act plus settle) with one spare sample of slack."""
|
||||
budget: Final = timeout * (repeats + 2)
|
||||
|
||||
def expire(signum: int, frame: FrameType | None) -> None:
|
||||
raise TimeoutError(f"{journey.name} exceeded its {budget:.0f}s wall clock")
|
||||
|
||||
previous: Final = signal.signal(signal.SIGALRM, expire)
|
||||
started: Final = time.monotonic()
|
||||
try:
|
||||
_ = signal.setitimer(signal.ITIMER_REAL, budget)
|
||||
for _ in range(repeats + 1):
|
||||
yield measure(page, journey, timeout)
|
||||
except Exception as error:
|
||||
if time.monotonic() - started < budget:
|
||||
raise
|
||||
yield Sample(seconds=time.monotonic() - started, error=f"wall_clock {type(error).__name__}: {error}"[:300])
|
||||
finally:
|
||||
_ = signal.setitimer(signal.ITIMER_REAL, 0)
|
||||
_ = signal.signal(signal.SIGALRM, previous)
|
||||
|
||||
|
||||
def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: pytest.FixtureRequest) -> None:
|
||||
journey: Final = next(journey for journey in journeys(ui_fixture.api_key) if journey.name == journey_name)
|
||||
transport: Final = HttpTransport(base_url=UI_BASE_URL, master_key=MASTER_KEY)
|
||||
timeout: Final = float(os.environ.get("E2E_UI_JOURNEY_TIMEOUT_SECONDS", "120"))
|
||||
repeats: Final = int(os.environ.get("E2E_UI_JOURNEY_REPEATS", "9"))
|
||||
assert 0 < timeout <= 600 and 1 <= repeats <= 1000, "Invalid UI performance sample settings"
|
||||
target: Final = Path(os.environ["E2E_ARTIFACT_DIR"])
|
||||
assert target.is_dir(), "E2E_ARTIFACT_DIR must exist"
|
||||
evidence: Final = target / f"ui-isolation-{journey.name}.jsonl"
|
||||
dashboard: Final = Dashboard(
|
||||
base_url=UI_BASE_URL, username=UI_USERNAME, password=UI_PASSWORD, anchor=ui_fixture.anchor
|
||||
)
|
||||
with without_retries():
|
||||
with isolated_database(journey.name):
|
||||
before: Final = settled_counter(lambda: _metrics(transport), evidence)
|
||||
samples: Final = _samples(dashboard, journey, timeout, repeats)
|
||||
with evidence.open("a", encoding="utf-8") as sample_output:
|
||||
_ = sample_output.write(
|
||||
json.dumps({"samples": tuple(sample.model_dump() for sample in samples)}) + "\n"
|
||||
)
|
||||
after_requests: Final = _metrics(transport)
|
||||
item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped
|
||||
assert isinstance(item, pytest.Item)
|
||||
item.user_properties.extend(ui_junit_properties(samples))
|
||||
item.user_properties.extend(ui_fixture.volume)
|
||||
after: Final = settled_counter(lambda: _metrics(transport), evidence)
|
||||
item.user_properties.extend(
|
||||
(
|
||||
("perf_ui_database_failures", after_requests - before),
|
||||
("perf_ui_cleanup_database_failures", after - after_requests),
|
||||
)
|
||||
)
|
||||
artifact: Final = {
|
||||
"name": journey.name,
|
||||
"surface": journey.surface,
|
||||
"measured_at": datetime.now(UTC).isoformat(),
|
||||
"samples": [sample.model_dump() for sample in samples],
|
||||
"error_codes": [error_code(sample) for sample in samples],
|
||||
"timings": summarize(samples).model_dump(),
|
||||
"timeouts": sum(sample.error == "timeout" for sample in samples),
|
||||
"failures": sum(bool(sample.error) for sample in samples),
|
||||
"database_failures_before": before,
|
||||
"database_failures_after_requests": after_requests,
|
||||
"database_failures_after_cleanup": after,
|
||||
}
|
||||
output: Final = target / f"ui-journey-{journey.name}.json"
|
||||
_ = output.write_text(json.dumps(artifact, indent=2), encoding="utf-8")
|
||||
assert_within_ceiling(journey.name, samples)
|
||||
|
|
@ -14,7 +14,6 @@ markers =
|
|||
cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set
|
||||
redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_PERF_TESTS is set
|
||||
quiet_stack: measures the proxy itself, so it runs while no other test on this host is hitting the stack; every other test waits for it to finish
|
||||
mcp_oauth_live: real Linear OAuth consent via a captured browser session; deselected unless E2E_MCP_OAUTH_LIVE is set
|
||||
provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set
|
||||
owned_gateway: boots its own proxy from source against the stack's Postgres, so it needs DATABASE_URL on the pytest host; deselected unless E2E_OWNED_GATEWAY is set
|
||||
otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ export const menuLabelToPage: Record<string, Page> = {
|
|||
Budgets: Page.Budgets,
|
||||
"API Playground": Page.TransformRequest,
|
||||
"Tag Management": Page.TagManagement,
|
||||
"Old Usage": Page.Usage,
|
||||
// Tools submenu items
|
||||
"MCP Servers": Page.McpServers,
|
||||
"Vector Stores": Page.VectorStores,
|
||||
|
|
|
|||
|
|
@ -118,12 +118,6 @@ export const MIGRATED_E2E_PAGES: Readonly<Record<string, MigratedPage>> = {
|
|||
content: { role: "heading", name: "AI Hub" },
|
||||
},
|
||||
new_usage: { segment: "usage", linkName: "Usage", content: { role: "heading", name: "Usage" } },
|
||||
usage: {
|
||||
segment: "old-usage",
|
||||
linkName: "Old Usage",
|
||||
group: "Experimental",
|
||||
content: { role: "tab", name: "All Up" },
|
||||
},
|
||||
agents: { segment: "agents", linkName: "Agents", group: "Agentic", content: { role: "heading", name: "Agents" } },
|
||||
"router-settings": {
|
||||
segment: "router-settings",
|
||||
|
|
|
|||
|
|
@ -30,5 +30,4 @@ export enum Page {
|
|||
TagManagement = "tag-management",
|
||||
VectorStores = "vector-stores",
|
||||
NewUsage = "new_usage",
|
||||
Usage = "usage",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { chromium, expect, request } from "@playwright/test";
|
|||
import { users, Role, STORAGE_PATHS } from "./fixtures/users";
|
||||
import { ARTIFACT_DIR, UI_BASE_URL } from "./constants";
|
||||
import { expectUnrestrictedDashboard, setInvitedUserPassword } from "./helpers/userOnboarding";
|
||||
import { hideLiteAdmin } from "./helpers/navigation";
|
||||
import { setAriaHidden } from "./helpers/navigation";
|
||||
import { masterKey as getMasterKey } from "./helpers/traffic";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
|
@ -78,7 +78,7 @@ async function globalSetup() {
|
|||
await dismiss.click();
|
||||
}
|
||||
if (role === Role.ProxyAdmin) {
|
||||
await hideLiteAdmin(page);
|
||||
await setAriaHidden(page, true);
|
||||
}
|
||||
// The login flow stores a post-login return URL in the litellm_return_url
|
||||
// cookie. If the snapshot captures it before the app consumes it, every
|
||||
|
|
|
|||
20
tests/e2e/ui/helpers/guardrails.ts
Normal file
20
tests/e2e/ui/helpers/guardrails.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { Page as PlaywrightPage, expect } from "@playwright/test";
|
||||
import { Page } from "../fixtures/pages";
|
||||
import { dismissFeedbackPopup, navigateToPage } from "./navigation";
|
||||
|
||||
export async function deleteGuardrailFromTable(page: PlaywrightPage, guardrailName: string): Promise<void> {
|
||||
await navigateToPage(page, Page.Guardrails);
|
||||
await dismissFeedbackPopup(page);
|
||||
const row = page.getByRole("row").filter({ hasText: guardrailName });
|
||||
await expect(row).toHaveCount(1, { timeout: 15_000 });
|
||||
await row.getByRole("button", { name: "Open guardrail actions" }).click();
|
||||
await page.getByRole("menuitem", { name: "Delete" }).click();
|
||||
|
||||
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
|
||||
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
|
||||
await deleteModal.getByPlaceholder(guardrailName, { exact: true }).fill(guardrailName);
|
||||
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
|
||||
|
||||
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 });
|
||||
await expect(row).toHaveCount(0, { timeout: 15_000 });
|
||||
}
|
||||
|
|
@ -62,18 +62,29 @@ export async function dismissFeedbackPopup(page: PlaywrightPage): Promise<void>
|
|||
}
|
||||
}
|
||||
|
||||
export async function hideLiteAdmin(page: PlaywrightPage): Promise<void> {
|
||||
export async function setAriaHidden(page: PlaywrightPage, hidden: boolean): Promise<void> {
|
||||
await page.getByRole("button", { name: /Account menu/i }).click();
|
||||
const panel = page.getByTestId("sidebar-account-menu-panel");
|
||||
await expect(panel).toBeVisible({ timeout: 5_000 });
|
||||
const toggle = panel.getByRole("switch", { name: "Toggle hide LiteAdmin" });
|
||||
if ((await toggle.getAttribute("aria-checked")) !== "true") {
|
||||
await toggle.click();
|
||||
const premium = await panel.getByText("Premium", { exact: true }).isVisible();
|
||||
const toggle = panel.getByRole("switch", { name: "Toggle hide Aria Agent", exact: true });
|
||||
if (premium) {
|
||||
await expect(toggle).toBeVisible();
|
||||
if ((await toggle.getAttribute("aria-checked")) !== String(hidden)) {
|
||||
await toggle.click();
|
||||
}
|
||||
await expect(toggle).toHaveAttribute("aria-checked", String(hidden));
|
||||
} else {
|
||||
await expect(toggle).toHaveCount(0);
|
||||
}
|
||||
await expect(toggle).toHaveAttribute("aria-checked", "true");
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(panel).toBeHidden();
|
||||
await expect(page.getByRole("button", { name: "LiteAdmin", exact: true })).toBeHidden();
|
||||
const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true });
|
||||
if (premium && hidden) {
|
||||
await expect(aria).toBeHidden();
|
||||
} else {
|
||||
await expect(aria).toBeVisible();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -2,42 +2,57 @@ import { expect, type Locator, type Page as PlaywrightPage } from "@playwright/t
|
|||
import { navigateToPage, dismissFeedbackPopup } from "./navigation";
|
||||
import { Page } from "../fixtures/pages";
|
||||
|
||||
/** Controls for the Test Key / Playground page, shared with the router-fallback specs. */
|
||||
|
||||
/**
|
||||
* The configuration panel is rendered twice, docked and overlay, with one visible at a time.
|
||||
* Every control is narrowed to the visible copy or it trips strict mode against its hidden twin.
|
||||
*/
|
||||
export const onlyVisible = (locator: Locator): Locator => locator.filter({ visible: true }).first();
|
||||
|
||||
/** The model combobox, addressed by the placeholder its search input shows before selection. */
|
||||
export const modelSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByPlaceholder("Select a Model"));
|
||||
export const composer = (page: PlaywrightPage): Locator =>
|
||||
onlyVisible(page.getByRole("group").filter({ has: page.getByRole("textbox", { name: "Message" }) }));
|
||||
|
||||
export const modelSelect = (page: PlaywrightPage): Locator =>
|
||||
onlyVisible(page.getByRole("combobox", { name: "Model", exact: true }));
|
||||
|
||||
export const sendButton = (page: PlaywrightPage): Locator =>
|
||||
onlyVisible(page.getByRole("button", { name: "Send message" }));
|
||||
composer(page).getByRole("button", { name: "Send message" });
|
||||
|
||||
/** The Virtual Key Source dropdown, addressed by the accessible name on its trigger. */
|
||||
export const keySourceSelect = (page: PlaywrightPage): Locator =>
|
||||
onlyVisible(page.getByLabel("Virtual Key Source"));
|
||||
export const keySourceSelect = (page: PlaywrightPage): Locator => onlyVisible(page.getByLabel("Send as"));
|
||||
|
||||
export async function openPlayground(page: PlaywrightPage): Promise<void> {
|
||||
await navigateToPage(page, Page.LlmPlayground);
|
||||
await dismissFeedbackPopup(page);
|
||||
await expect(onlyVisible(page.getByText("Virtual Key Source"))).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
await expect(modelSelect(page)).toBeVisible({ timeout: 20_000 });
|
||||
const searchHint = page.getByRole("button", { name: "Dismiss search hint" });
|
||||
if (await searchHint.isVisible()) {
|
||||
await searchHint.click();
|
||||
}
|
||||
}
|
||||
|
||||
export async function openRequestSettings(page: PlaywrightPage): Promise<void> {
|
||||
await composer(page).getByRole("button", { name: "Settings", exact: true }).click();
|
||||
await expect(keySourceSelect(page)).toBeVisible({ timeout: 10_000 });
|
||||
}
|
||||
|
||||
export async function enterVirtualKey(page: PlaywrightPage, key: string): Promise<void> {
|
||||
await openRequestSettings(page);
|
||||
await keySourceSelect(page).click();
|
||||
await onlyVisible(page.getByRole("option", { name: "Virtual key", exact: true })).click({ timeout: 15_000 });
|
||||
const keyInput = onlyVisible(page.getByLabel("Virtual key", { exact: true }));
|
||||
await expect(keyInput).toBeVisible({ timeout: 10_000 });
|
||||
await keyInput.fill(key);
|
||||
await page.keyboard.press("Escape");
|
||||
}
|
||||
|
||||
export async function searchModels(page: PlaywrightPage, query: string): Promise<void> {
|
||||
await modelSelect(page).click();
|
||||
await onlyVisible(page.getByPlaceholder("Search models")).fill(query);
|
||||
}
|
||||
|
||||
export async function selectModel(page: PlaywrightPage, model: string): Promise<void> {
|
||||
const select = modelSelect(page);
|
||||
await select.click();
|
||||
// Virtualized: options outside the rendered window are absent from the DOM, so search first.
|
||||
await select.fill(model);
|
||||
await searchModels(page, model);
|
||||
await onlyVisible(page.getByRole("option", { name: model, exact: true })).click({ timeout: 15_000 });
|
||||
await expect(modelSelect(page)).toContainText(model);
|
||||
}
|
||||
|
||||
export async function sendMessage(page: PlaywrightPage, message: string): Promise<void> {
|
||||
const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
|
||||
const input = composer(page).getByRole("textbox", { name: "Message" });
|
||||
await expect(input).toBeVisible({ timeout: 15_000 });
|
||||
await input.fill(message);
|
||||
await sendButton(page).click();
|
||||
|
|
|
|||
|
|
@ -60,13 +60,11 @@ test.describe("Budgets", () => {
|
|||
await expect(modal).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await modal.getByRole("textbox", { name: "Budget ID" }).fill(budgetId);
|
||||
await modal.getByRole("spinbutton", { name: "Max Tokens per minute" }).fill("5000");
|
||||
await modal.getByRole("spinbutton", { name: "Max Requests per minute" }).fill("60");
|
||||
|
||||
await modal.getByRole("button", { name: "Optional Settings" }).click();
|
||||
await modal.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("25.5");
|
||||
await modal.getByRole("combobox", { name: "Reset Budget" }).click();
|
||||
await page.getByRole("option", { name: "weekly" }).click();
|
||||
await modal.getByRole("spinbutton", { name: "Tokens per minute" }).fill("5000");
|
||||
await modal.getByRole("spinbutton", { name: "Requests per minute" }).fill("60");
|
||||
await modal.getByRole("spinbutton", { name: "Spend limit" }).fill("25.5");
|
||||
await modal.getByRole("combobox", { name: "Reset period" }).click();
|
||||
await page.getByRole("option", { name: "per week" }).click();
|
||||
|
||||
await modal.getByRole("button", { name: "Create Budget" }).click();
|
||||
await expect(modal).not.toBeVisible({ timeout: 10_000 });
|
||||
|
|
@ -100,15 +98,12 @@ test.describe("Budgets", () => {
|
|||
const modal = page.getByRole("dialog", { name: "Edit Budget" });
|
||||
await expect(modal).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await modal.getByRole("button", { name: "Optional Settings" }).click();
|
||||
await modal.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("99");
|
||||
await modal.getByRole("spinbutton", { name: "Spend limit" }).fill("99");
|
||||
await modal.getByRole("button", { name: "Save", exact: true }).click();
|
||||
await expect(modal).not.toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await expect(page.getByRole("row").filter({ hasText: budgetId })).toContainText("$99.00", { timeout: 10_000 });
|
||||
|
||||
// Not hypothetical: the edit form posts the whole budget, so a field it fails to
|
||||
// seed from the existing row goes to the server as null and silently clears.
|
||||
const stored = await findBudget(page, budgetId);
|
||||
expect(stored?.max_budget, "spend cap raised").toBe(99);
|
||||
expect(stored?.tpm_limit, "TPM limit untouched by a spend-cap edit").toBe(1000);
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
|
|||
import { ADMIN_STORAGE_PATH, E2E_TEAM_NO_ADMIN_ID } from "../../constants";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
|
||||
import { deleteGuardrailFromTable } from "../../helpers/guardrails";
|
||||
import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic";
|
||||
|
||||
interface StoredGuardrail {
|
||||
|
|
@ -183,21 +184,9 @@ test.describe("Guardrails", () => {
|
|||
test("Delete a guardrail", async ({ page }) => {
|
||||
const stamp = Date.now();
|
||||
const guardrailName = `e2e-guardrail-delete-${stamp}`;
|
||||
const guardrailId = await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
|
||||
await createKeywordGuardrailViaApi(page, guardrailName, `e2edelete${stamp}`);
|
||||
|
||||
await navigateToPage(page, Page.Guardrails);
|
||||
await dismissFeedbackPopup(page);
|
||||
|
||||
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await page.getByTestId(`guardrail-actions-${guardrailId}`).click();
|
||||
await page.getByTestId("guardrail-action-delete").click();
|
||||
|
||||
const modal = page.getByRole("dialog");
|
||||
await expect(modal).toBeVisible({ timeout: 5_000 });
|
||||
await modal.getByRole("button", { name: "Delete", exact: true }).click();
|
||||
|
||||
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(0, { timeout: 15_000 });
|
||||
await deleteGuardrailFromTable(page, guardrailName);
|
||||
|
||||
// The RC checklist deletes then reloads, because a row vanishing from the table has
|
||||
// fooled us before; assert against the route the reload would read.
|
||||
|
|
@ -274,19 +263,7 @@ test.describe("Guardrails", () => {
|
|||
await expect(page.getByRole("option", { name: guardrailName })).toBeVisible({ timeout: 10_000 });
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
await navigateToPage(page, Page.Guardrails);
|
||||
await expect(row).toHaveCount(1, { timeout: 15_000 });
|
||||
await row.getByRole("button", { name: "Open guardrail actions" }).click();
|
||||
await page.getByRole("menuitem", { name: "Delete" }).click();
|
||||
|
||||
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
|
||||
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
|
||||
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
|
||||
|
||||
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({
|
||||
timeout: 10_000,
|
||||
});
|
||||
await expect(row).toHaveCount(0, { timeout: 15_000 });
|
||||
await deleteGuardrailFromTable(page, guardrailName);
|
||||
|
||||
await page.reload();
|
||||
await expect(page.getByRole("button", { name: /Add New Guardrail/i })).toBeVisible({ timeout: 20_000 });
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { test as base, expect, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH, MOCK_PRESIDIO_URL, PROPAGATION_TIMEOUT_MS } from "../../constants";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
import { deleteGuardrailFromTable } from "../../helpers/guardrails";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { CHAT_MODEL_A, masterKey, rootPath, uniqueSuffix, waitForSpendLog } from "../../helpers/traffic";
|
||||
import { openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground";
|
||||
import { composer, openPlayground, selectModel, sendButton, onlyVisible } from "../../helpers/playground";
|
||||
|
||||
const RAW_EMAIL = "jane.doe@example.com";
|
||||
const RAW_PHONE = "555-867-5309";
|
||||
|
|
@ -49,19 +50,6 @@ async function createPresidioGuardrail(page: PlaywrightPage, guardrailName: stri
|
|||
await expect(page.getByRole("row").filter({ hasText: guardrailName })).toHaveCount(1, { timeout: 15_000 });
|
||||
}
|
||||
|
||||
async function deleteGuardrail(page: PlaywrightPage, guardrailName: string): Promise<void> {
|
||||
await navigateToPage(page, Page.Guardrails);
|
||||
await dismissFeedbackPopup(page);
|
||||
const row = page.getByRole("row").filter({ hasText: guardrailName });
|
||||
await expect(row).toHaveCount(1, { timeout: 15_000 });
|
||||
await row.getByRole("button", { name: "Open guardrail actions" }).click();
|
||||
await page.getByRole("menuitem", { name: "Delete" }).click();
|
||||
const deleteModal = page.getByRole("dialog", { name: "Delete Guardrail" });
|
||||
await expect(deleteModal).toBeVisible({ timeout: 5_000 });
|
||||
await deleteModal.getByRole("button", { name: "Delete", exact: true }).click();
|
||||
await expect(page.getByText(`Guardrail "${guardrailName}" deleted successfully`)).toBeVisible({ timeout: 10_000 });
|
||||
}
|
||||
|
||||
const test = base.extend<{ guardrailName: string }>({
|
||||
guardrailName: async ({ page }, use) => {
|
||||
const name = `e2e-presidio-story-${uniqueSuffix()}`;
|
||||
|
|
@ -69,7 +57,7 @@ const test = base.extend<{ guardrailName: string }>({
|
|||
try {
|
||||
await use(name);
|
||||
} finally {
|
||||
await deleteGuardrail(page, name);
|
||||
await deleteGuardrailFromTable(page, name);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
|
@ -123,14 +111,12 @@ test.describe("Presidio PII guardrail, end to end from the dashboard", () => {
|
|||
await openPlayground(page);
|
||||
await selectModel(page, CHAT_MODEL_A);
|
||||
|
||||
const guardrailSelect = onlyVisible(page.getByPlaceholder("Select guardrails"));
|
||||
await expect(guardrailSelect).toBeVisible({ timeout: 20_000 });
|
||||
await guardrailSelect.click();
|
||||
await guardrailSelect.fill(guardrailName);
|
||||
await composer(page).getByRole("combobox", { name: "Guardrails" }).click();
|
||||
await onlyVisible(page.getByPlaceholder("Search guardrails")).fill(guardrailName);
|
||||
await onlyVisible(page.getByRole("option", { name: guardrailName, exact: true })).click({ timeout: 20_000 });
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
const input = onlyVisible(page.getByPlaceholder("Type your message", { exact: false }));
|
||||
const input = composer(page).getByRole("textbox", { name: "Message" });
|
||||
await expect(input).toBeVisible({ timeout: 15_000 });
|
||||
|
||||
await input.fill(prompt);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage, clickTeamId } from "../../helpers/navigation";
|
||||
import { CHAT_MODEL_A, MOCK_RESPONSE_TEXT, masterKey } from "../../helpers/traffic";
|
||||
import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground";
|
||||
import { openPlayground, selectModel, sendMessage, enterVirtualKey } from "../../helpers/playground";
|
||||
|
||||
test.describe("Internal User", () => {
|
||||
test.use({ storageState: INTERNAL_USER_STORAGE_PATH });
|
||||
|
|
@ -67,18 +67,13 @@ test.describe("Internal User", () => {
|
|||
await page.getByRole("button", { name: "Create Key", exact: true }).click();
|
||||
|
||||
await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
|
||||
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim();
|
||||
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
|
||||
expect(apiKey).toMatch(/^sk-/);
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
try {
|
||||
await openPlayground(page);
|
||||
await keySourceSelect(page).click();
|
||||
await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 });
|
||||
|
||||
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
|
||||
await expect(keyInput).toBeVisible({ timeout: 10_000 });
|
||||
await keyInput.fill(apiKey);
|
||||
await enterVirtualKey(page, apiKey);
|
||||
|
||||
await selectModel(page, CHAT_MODEL_A);
|
||||
await sendMessage(page, `internal user team key ping ${keyName}`);
|
||||
|
|
|
|||
|
|
@ -194,17 +194,6 @@ test.describe("Logs page", () => {
|
|||
await expect(row, `no logs row for call id ${callId}`).toHaveCount(1, { timeout: 30_000 });
|
||||
await expect(row, "the row itself shows only the request id").not.toContainText(callId);
|
||||
|
||||
await row.getByText(requestId).hover();
|
||||
const tooltip = page.locator("[data-slot='tooltip-content']");
|
||||
await expect(tooltip, "hovering the Request ID cell does not list the x-litellm-call-id").toContainText(
|
||||
`x-litellm-call-id: ${callId}`,
|
||||
{ timeout: 10_000 },
|
||||
);
|
||||
await tooltip.getByRole("button", { name: "Copy x-litellm-call-id" }).click();
|
||||
if (await page.evaluate(() => window.isSecureContext)) {
|
||||
await expect.poll(() => page.evaluate(() => navigator.clipboard.readText())).toBe(callId);
|
||||
}
|
||||
|
||||
await row.click();
|
||||
const drawer = page.getByRole("dialog").first();
|
||||
await expect(drawer.getByText("Request & Response")).toBeVisible({ timeout: 20_000 });
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import { test, expect, type APIRequestContext, type Locator, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { navigateToPage, dismissFeedbackPopup } from "../../helpers/navigation";
|
||||
import { navigateToPage, dismissFeedbackPopup, setAriaHidden } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { CHAT_MODEL_A, createVirtualKey, sendChatCompletion, waitForSpendLog } from "../../helpers/traffic";
|
||||
import {
|
||||
CHAT_MODEL_A,
|
||||
createVirtualKey,
|
||||
deleteVirtualKey,
|
||||
sendChatCompletion,
|
||||
waitForSpendLog,
|
||||
} from "../../helpers/traffic";
|
||||
|
||||
/**
|
||||
* Session-grouped pagination (#38060): a page of N rows must render exactly N session rows, a
|
||||
|
|
@ -45,6 +51,52 @@ async function setRowsPerPage(page: PlaywrightPage, size: "25" | "50" | "100"):
|
|||
await page.getByRole("option", { name: size, exact: true }).click();
|
||||
}
|
||||
|
||||
test.describe("Logs pagination with Aria in the header", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test("keeps pagination reachable and Aria available at 1440px", async ({ page, request }) => {
|
||||
await page.setViewportSize({ width: 1440, height: 900 });
|
||||
const alias = `e2e-aria-header-${uniqueSuffix()}`;
|
||||
const key = await createVirtualKey(request, { key_alias: alias });
|
||||
try {
|
||||
const ids = await Promise.all(
|
||||
Array.from({ length: 26 }, (_, index) =>
|
||||
sendChatCompletion(request, {
|
||||
model: CHAT_MODEL_A,
|
||||
prompt: `${alias}-${index}`,
|
||||
apiKey: key.key,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await Promise.all(ids.map((id) => waitForSpendLog(request, id)));
|
||||
await openLogs(page);
|
||||
await setAriaHidden(page, false);
|
||||
await applyKeyAliasFilter(page, await openFilterDrawer(page), alias);
|
||||
await setRowsPerPage(page, "25");
|
||||
await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26");
|
||||
await visibleTestId(page, "pagination-next").click();
|
||||
await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 26-26 of 26");
|
||||
|
||||
const aria = page.getByRole("banner").getByRole("button", { name: "Aria Agent", exact: true });
|
||||
await aria.click();
|
||||
await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(aria).toBeFocused();
|
||||
await visibleTestId(page, "pagination-first").click();
|
||||
await expect(visibleTestId(page, "pagination-range")).toHaveText("Showing 1-25 of 26");
|
||||
for (const destination of [Page.NewUsage, Page.LlmPlayground, Page.ApiKeys]) {
|
||||
await navigateToPage(page, destination);
|
||||
await aria.click();
|
||||
await expect(page.getByRole("dialog", { name: "Aria Agent", exact: true })).toBeVisible();
|
||||
await page.keyboard.press("Escape");
|
||||
await expect(aria).toBeFocused();
|
||||
}
|
||||
} finally {
|
||||
await deleteVirtualKey(request, key.token);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test.describe("Logs page session-grouped pagination", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
|
|
|
|||
|
|
@ -1,52 +0,0 @@
|
|||
import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
|
||||
/**
|
||||
* Opens Add Auto Router and returns the Template select's trigger, which is the
|
||||
* shallowest real page that renders SelectContent with tall multi-line options.
|
||||
*/
|
||||
async function openTemplateSelect(page: PlaywrightPage) {
|
||||
await navigateToPage(page, Page.Models);
|
||||
await page.getByRole("tab", { name: "Auto-Routers" }).click();
|
||||
await page.getByRole("button", { name: "Add Auto Router" }).click();
|
||||
|
||||
const trigger = page.getByTestId("template-selector");
|
||||
await expect(trigger).toBeVisible();
|
||||
return trigger;
|
||||
}
|
||||
|
||||
async function boxes(trigger: Locator, options: Locator) {
|
||||
const triggerBox = await trigger.boundingBox();
|
||||
const optionsBox = await options.boundingBox();
|
||||
return triggerBox && optionsBox ? { triggerBox, optionsBox } : null;
|
||||
}
|
||||
|
||||
const clippedPopup = (page: PlaywrightPage) => page.locator('[data-slot="select-content"]');
|
||||
|
||||
function pollOptionsCoverTrigger(trigger: Locator, options: Locator) {
|
||||
return expect.poll(async () => {
|
||||
const box = await boxes(trigger, options);
|
||||
return (
|
||||
box &&
|
||||
box.optionsBox.y < box.triggerBox.y + box.triggerBox.height &&
|
||||
box.optionsBox.y + box.optionsBox.height > box.triggerBox.y
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("Auto Router template select anchoring", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test("keeps the trigger uncovered when the options open with no room below it", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 560 });
|
||||
const trigger = await openTemplateSelect(page);
|
||||
await trigger.scrollIntoViewIfNeeded();
|
||||
|
||||
await trigger.click();
|
||||
await expect(page.getByRole("listbox")).toBeVisible();
|
||||
|
||||
await pollOptionsCoverTrigger(trigger, clippedPopup(page)).toBe(false);
|
||||
});
|
||||
});
|
||||
146
tests/e2e/ui/tests/modelsPage/cacheWriteTiers.spec.ts
Normal file
146
tests/e2e/ui/tests/modelsPage/cacheWriteTiers.spec.ts
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
import { test, expect, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { Role, users } from "../../fixtures/users";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { captureRequestBody, readBack } from "../../helpers/roundTrip";
|
||||
|
||||
const FIVE_MINUTE_PER_MILLION = 3.75;
|
||||
const ONE_HOUR_PER_MILLION = 7.5;
|
||||
const perToken = (perMillion: number): number => perMillion / 1_000_000;
|
||||
const ONE_HOUR_LABEL = /^Cache Write Cost \(1 hour, per 1M tokens\)/;
|
||||
|
||||
type Deployment = {
|
||||
model_name?: string;
|
||||
model_info?: { id?: string };
|
||||
litellm_params: {
|
||||
cache_creation_input_token_cost?: number;
|
||||
cache_creation_input_token_cost_above_1hr?: number;
|
||||
};
|
||||
};
|
||||
|
||||
const masterKey = users[Role.ProxyAdmin].password;
|
||||
const auth = { Authorization: `Bearer ${masterKey}` };
|
||||
|
||||
async function deleteModel(page: PlaywrightPage, id: string) {
|
||||
await page.request.post("/model/delete", { headers: auth, data: { id } });
|
||||
}
|
||||
|
||||
async function readDeployment(page: PlaywrightPage, modelId: string): Promise<Deployment | undefined> {
|
||||
const body = await readBack<{ data: Deployment[] }>(page, `/model/info?litellm_model_id=${modelId}`);
|
||||
return body.data[0];
|
||||
}
|
||||
|
||||
async function selectProvider(page: PlaywrightPage, providerName: string) {
|
||||
const providerDropdown = page.getByRole("combobox", { name: "Provider", exact: true });
|
||||
await providerDropdown.click();
|
||||
await providerDropdown.fill(providerName);
|
||||
await page.getByRole("option").filter({ hasText: providerName }).first().click();
|
||||
await expect(providerDropdown).toHaveValue(providerName);
|
||||
}
|
||||
|
||||
test.describe("Cache write cost tiers", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
let createdModelId: string | null = null;
|
||||
|
||||
test.afterEach(async ({ page }) => {
|
||||
if (createdModelId) {
|
||||
await deleteModel(page, createdModelId);
|
||||
createdModelId = null;
|
||||
}
|
||||
});
|
||||
|
||||
test("the edit form saves the 1 hour rate next to the 5 minute rate and the detail view shows both", async ({
|
||||
page,
|
||||
}) => {
|
||||
const modelName = `e2e-cache-tiers-edit-${Date.now()}`;
|
||||
const created = await page.request.post("/model/new", {
|
||||
headers: auth,
|
||||
data: {
|
||||
model_name: modelName,
|
||||
litellm_params: {
|
||||
model: "anthropic/claude-sonnet-4-6",
|
||||
api_key: "sk-e2e-not-used",
|
||||
cache_creation_input_token_cost: perToken(FIVE_MINUTE_PER_MILLION),
|
||||
},
|
||||
model_info: {},
|
||||
},
|
||||
});
|
||||
expect(created.ok(), `POST /model/new for ${modelName}`).toBe(true);
|
||||
createdModelId = (await created.json()).model_info?.id;
|
||||
expect(createdModelId, "model id from /model/new").toBeTruthy();
|
||||
|
||||
await page.goto("/ui");
|
||||
await page.getByText("Models + Endpoints").click();
|
||||
const modelIdCell = page.getByTestId(`model-id-${createdModelId}`);
|
||||
await expect(modelIdCell).toBeVisible({ timeout: 15_000 });
|
||||
await modelIdCell.click();
|
||||
await expect(page.getByText("Back to Models").first()).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
await page.getByRole("button", { name: "Edit Settings" }).click();
|
||||
const oneHour = page.getByLabel(ONE_HOUR_LABEL);
|
||||
await oneHour.waitFor({ timeout: 15_000 });
|
||||
await oneHour.fill(String(ONE_HOUR_PER_MILLION));
|
||||
|
||||
const patchBody = await captureRequestBody(
|
||||
page,
|
||||
{ method: "PATCH", urlIncludes: `/model/${createdModelId}/update` },
|
||||
async () => {
|
||||
await page.getByRole("button", { name: "Save Changes" }).click();
|
||||
},
|
||||
);
|
||||
expect(patchBody.litellm_params.cache_creation_input_token_cost_above_1hr, "1 hour rate sent per token").toBe(
|
||||
perToken(ONE_HOUR_PER_MILLION),
|
||||
);
|
||||
await expect(page.getByText("Model settings updated successfully")).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
const stored = await readDeployment(page, createdModelId!);
|
||||
expect(stored?.litellm_params?.cache_creation_input_token_cost_above_1hr, "1 hour rate persisted").toBe(
|
||||
perToken(ONE_HOUR_PER_MILLION),
|
||||
);
|
||||
expect(stored?.litellm_params?.cache_creation_input_token_cost, "5 minute rate untouched").toBe(
|
||||
perToken(FIVE_MINUTE_PER_MILLION),
|
||||
);
|
||||
|
||||
await expect(page.getByText(ONE_HOUR_LABEL)).toBeVisible();
|
||||
await expect(page.getByText(ONE_HOUR_PER_MILLION.toFixed(4))).toBeVisible({ timeout: 10_000 });
|
||||
await expect(page.getByText(FIVE_MINUTE_PER_MILLION.toFixed(4))).toBeVisible();
|
||||
});
|
||||
|
||||
test("Add Model sends both cache write tiers per token", async ({ page }) => {
|
||||
const publicName = `e2e-cache-tiers-add-${Date.now()}`;
|
||||
await navigateToPage(page, Page.Models);
|
||||
await page.getByRole("tab", { name: "Add Model" }).click();
|
||||
await selectProvider(page, "Anthropic");
|
||||
|
||||
await page.getByRole("combobox", { name: "Select models" }).click();
|
||||
await page.getByRole("option", { name: "Custom Model Name (Enter below)" }).click();
|
||||
await page.keyboard.press("Escape");
|
||||
await page.getByPlaceholder("Enter custom model name").fill(publicName);
|
||||
await page.locator("#api_key").fill("sk-e2e-not-used");
|
||||
|
||||
await page.getByText("Advanced Settings").click();
|
||||
await page.getByRole("switch", { name: "Custom Pricing" }).click();
|
||||
await page.getByLabel(/^Input Cost \(per 1M tokens\)/).fill("3");
|
||||
await page.getByLabel(/^Output Cost \(per 1M tokens\)/).fill("15");
|
||||
await page.getByLabel(/^Cache Write Cost \(per 1M tokens\)/).fill(String(FIVE_MINUTE_PER_MILLION));
|
||||
await page.getByLabel(ONE_HOUR_LABEL).fill(String(ONE_HOUR_PER_MILLION));
|
||||
|
||||
const body = await captureRequestBody(page, { method: "POST", urlIncludes: "/model/new" }, async () => {
|
||||
await page.getByTestId("add-model-btn").click();
|
||||
});
|
||||
await expect(page.getByText("created successfully")).toBeVisible({ timeout: 15_000 });
|
||||
const listed = await readBack<{ data: Deployment[] }>(page, "/v2/model/info");
|
||||
createdModelId = listed.data.find((row) => row.model_name === publicName)?.model_info?.id ?? null;
|
||||
|
||||
expect(body.model_name).toBe(publicName);
|
||||
expect(body.litellm_params.cache_creation_input_token_cost, "5 minute rate sent per token").toBe(
|
||||
perToken(FIVE_MINUTE_PER_MILLION),
|
||||
);
|
||||
expect(body.litellm_params.cache_creation_input_token_cost_above_1hr, "1 hour rate sent per token").toBe(
|
||||
perToken(ONE_HOUR_PER_MILLION),
|
||||
);
|
||||
expect(createdModelId, "the model landed in the database").toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
|
@ -87,7 +87,7 @@ test.describe("Clear custom pricing on a deployment", () => {
|
|||
// Both cache fields share the same placeholder ("Defaults to Input Cost if blank"),
|
||||
// so disambiguate via their labels.
|
||||
const cacheReadCost = page.getByLabel(/Cache Read Cost/);
|
||||
const cacheWriteCost = page.getByLabel(/Cache Write Cost/);
|
||||
const cacheWriteCost = page.getByLabel("Cache Write Cost (per 1M tokens)", { exact: true });
|
||||
await inputCost.waitFor({ timeout: 15_000 });
|
||||
for (const field of [inputCost, outputCost, cacheReadCost, cacheWriteCost]) {
|
||||
await field.click({ clickCount: 3 });
|
||||
|
|
|
|||
|
|
@ -1,7 +1,14 @@
|
|||
import { test, expect } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { CHAT_MODEL_A, CHAT_MODEL_B, MOCK_RESPONSE_TEXT, createVirtualKey } from "../../helpers/traffic";
|
||||
import { keySourceSelect, onlyVisible, openPlayground, selectModel, sendMessage } from "../../helpers/playground";
|
||||
import {
|
||||
keySourceSelect,
|
||||
openPlayground,
|
||||
openRequestSettings,
|
||||
selectModel,
|
||||
sendMessage,
|
||||
enterVirtualKey,
|
||||
} from "../../helpers/playground";
|
||||
|
||||
/**
|
||||
* The one flow that exercises the dashboard's own LLM call path rather than an admin CRUD endpoint,
|
||||
|
|
@ -14,8 +21,9 @@ test.describe("Playground", () => {
|
|||
test(`chats with ${model} using the current UI session`, async ({ page }) => {
|
||||
await openPlayground(page);
|
||||
|
||||
// "Current UI Session" is the default: the logged-in admin's key, nothing pasted.
|
||||
await expect(keySourceSelect(page)).toContainText("Current UI Session");
|
||||
await openRequestSettings(page);
|
||||
await expect(keySourceSelect(page)).toContainText("Current UI session");
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
await selectModel(page, model);
|
||||
const prompt = `playground ping for ${model}`;
|
||||
|
|
@ -34,14 +42,7 @@ test.describe("Playground", () => {
|
|||
|
||||
await openPlayground(page);
|
||||
|
||||
// Switch the source to "Virtual Key" and paste the key we just minted.
|
||||
await keySourceSelect(page).click();
|
||||
await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 });
|
||||
|
||||
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
|
||||
await expect(keyInput).toBeVisible({ timeout: 10_000 });
|
||||
await keyInput.fill(key);
|
||||
|
||||
await enterVirtualKey(page, key);
|
||||
await selectModel(page, CHAT_MODEL_A);
|
||||
await sendMessage(page, "playground ping via virtual key");
|
||||
|
||||
|
|
|
|||
|
|
@ -254,7 +254,7 @@ test.describe("Proxy Admin - Keys", () => {
|
|||
// verify it can call /chat/completions for the model it was scoped to.
|
||||
// The mock LLM server (fixtures/mock_llm_server/server.py) replies with
|
||||
// a fixed "This is a mock response." body.
|
||||
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim();
|
||||
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
|
||||
expect(apiKey).toMatch(/^sk-/);
|
||||
|
||||
const response = await page.request.post("/chat/completions", {
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ test.describe("Second proxy admin", () => {
|
|||
await page.getByRole("button", { name: "Create Key", exact: true }).click();
|
||||
|
||||
await expect(page.getByText("Save your Key")).toBeVisible({ timeout: 10_000 });
|
||||
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").innerText()).trim();
|
||||
const apiKey = (await page.getByRole("dialog", { name: "Save your Key" }).locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
|
||||
expect(apiKey).toMatch(/^sk-/);
|
||||
await page.keyboard.press("Escape");
|
||||
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ test.describe("Team Admin - Member permissions", () => {
|
|||
await memberPage.getByRole("button", { name: "Create Key", exact: true }).click();
|
||||
const saveDialog = memberPage.getByRole("dialog", { name: "Save your Key" });
|
||||
await expect(saveDialog).toBeVisible({ timeout: 15_000 });
|
||||
const apiKey = (await saveDialog.locator("pre").innerText()).trim();
|
||||
const apiKey = (await saveDialog.locator("pre").filter({ hasText: /^sk-/ }).innerText()).trim();
|
||||
expect(apiKey).toMatch(/^sk-/);
|
||||
createdKeys.push(apiKey);
|
||||
await memberPage.keyboard.press("Escape");
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { Page } from "../../fixtures/pages";
|
|||
import { navigateToPage, dismissFeedbackPopup, clickTeamId } from "../../helpers/navigation";
|
||||
import { captureRequestBody, readBack } from "../../helpers/roundTrip";
|
||||
import { CHAT_MODEL_A, masterKey } from "../../helpers/traffic";
|
||||
import { keySourceSelect, modelSelect, onlyVisible, openPlayground } from "../../helpers/playground";
|
||||
import { onlyVisible, openPlayground, searchModels, enterVirtualKey } from "../../helpers/playground";
|
||||
|
||||
/**
|
||||
* Every identifier a roster is addressable by. Which of user_id / user_email is populated depends on
|
||||
|
|
@ -228,21 +228,14 @@ test.describe("Team Admin", () => {
|
|||
.toBe(true);
|
||||
|
||||
await openPlayground(page);
|
||||
await keySourceSelect(page).click();
|
||||
await onlyVisible(page.getByRole("option", { name: "Virtual Key" })).click({ timeout: 15_000 });
|
||||
await enterVirtualKey(page, teamKey);
|
||||
|
||||
const keyInput = onlyVisible(page.getByPlaceholder("Enter custom Virtual Key"));
|
||||
await expect(keyInput).toBeVisible({ timeout: 10_000 });
|
||||
await keyInput.fill(teamKey);
|
||||
|
||||
const select = modelSelect(page);
|
||||
await select.click();
|
||||
await select.fill(teamModelName);
|
||||
await searchModels(page, teamModelName);
|
||||
await expect(onlyVisible(page.getByRole("option", { name: teamModelName }))).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
||||
await select.fill(CHAT_MODEL_A);
|
||||
await onlyVisible(page.getByPlaceholder("Search models")).fill(CHAT_MODEL_A);
|
||||
await expect(onlyVisible(page.getByRole("option", { name: CHAT_MODEL_A }))).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import {
|
|||
waitForSpendLog,
|
||||
} from "../../helpers/traffic";
|
||||
|
||||
/** Covers /ui/usage. The legacy /ui/old-usage view is deprecated and deliberately not covered. */
|
||||
/** Covers /ui/usage. */
|
||||
|
||||
const topKeysCard = (page: PlaywrightPage): Locator =>
|
||||
page.locator("section").filter({ has: page.getByRole("heading", { name: "Top Virtual Keys", exact: true }) });
|
||||
|
|
|
|||
182
tests/e2e_harness/perf/setup/test_event_loop.py
Normal file
182
tests/e2e_harness/perf/setup/test_event_loop.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from setup.event_loop import EventLoopStalls, KnownStall, junit_properties, offenders, parse_stalls, summarize
|
||||
|
||||
_HANDLER: Final = (
|
||||
"Executing <Task pending name='Task-9' coro=<RequestResponseCycle.run_asgi() running at "
|
||||
"/usr/local/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py:416> wait_for=<Future pending "
|
||||
"cb=[Task.task_wakeup()]> created at /usr/local/lib/python3.12/asyncio/base_events.py:448> took 0.731 seconds"
|
||||
)
|
||||
_STREAM: Final = (
|
||||
"Executing <Task pending name='Task-31' coro=<TaskHandle._run_coro() running at "
|
||||
"/usr/local/lib/python3.12/site-packages/anyio/_core/_tasks.py:275> created at "
|
||||
"/usr/local/lib/python3.12/asyncio/tasks.py:420> took 0.412 seconds"
|
||||
)
|
||||
_STARTUP: Final = (
|
||||
"Executing <Task pending name='Task-2' coro=<LifespanOn.main() running at "
|
||||
"/usr/local/lib/python3.12/site-packages/uvicorn/lifespan/on.py:86> created at "
|
||||
"/usr/local/lib/python3.12/asyncio/base_events.py:448> took 0.302 seconds"
|
||||
)
|
||||
_GC_PASS: Final = (
|
||||
"2026-10-17 04:46:02,101 - LiteLLM Proxy - WARNING - Garbage collection gen=2 collected=98608 took 0.711 seconds"
|
||||
)
|
||||
_SPEC: Final = EventLoopStalls(
|
||||
allowlist=(
|
||||
KnownStall("LifespanOn.main", max_count=3),
|
||||
KnownStall("RequestResponseCycle.run_asgi", max_count=2),
|
||||
KnownStall("garbage collection", max_count=1),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _Slice(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
label: str
|
||||
value: float
|
||||
share: float | None
|
||||
note: str | None
|
||||
over: bool
|
||||
|
||||
|
||||
def _breakdown(properties: dict[str, float | str], name: str) -> list[_Slice]:
|
||||
return TypeAdapter(list[_Slice]).validate_json(str(properties[f"perf_event_loop_{name}_breakdown"]))
|
||||
|
||||
|
||||
def test_parse_keeps_the_coroutine_its_location_and_the_hold() -> None:
|
||||
(stall,) = parse_stalls(("INFO: plain log line", _HANDLER))
|
||||
assert stall.coroutine == "RequestResponseCycle.run_asgi()"
|
||||
assert stall.where == "uvicorn/protocols/http/h11_impl.py:416"
|
||||
assert stall.seconds == 0.731
|
||||
|
||||
|
||||
def test_a_location_off_the_allowlist_offends_once_however_often_it_appears() -> None:
|
||||
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER)))
|
||||
assert [(row.coroutine, row.count, row.offending) for row in rows] == [
|
||||
("TaskHandle._run_coro()", 2, True),
|
||||
("RequestResponseCycle.run_asgi()", 1, False),
|
||||
("LifespanOn.main()", 1, False),
|
||||
]
|
||||
assert offenders(rows) == (
|
||||
"TaskHandle._run_coro() anyio/_core/_tasks.py:275: 2x, max 0.41 s (not on the allowlist)",
|
||||
)
|
||||
|
||||
|
||||
def test_a_known_location_offends_only_past_its_count() -> None:
|
||||
within: Final = summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER)))
|
||||
assert offenders(within) == ()
|
||||
past: Final = summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER, _HANDLER)))
|
||||
assert offenders(past) == (
|
||||
"RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416: 3x, max 0.73 s "
|
||||
"(allowlist RequestResponseCycle.run_asgi permits 2)",
|
||||
)
|
||||
|
||||
|
||||
def test_holds_under_the_threshold_are_not_rows() -> None:
|
||||
short: Final = _HANDLER.replace("took 0.731 seconds", "took 0.099 seconds")
|
||||
assert summarize(_SPEC, parse_stalls((short,))) == ()
|
||||
|
||||
|
||||
def test_a_hold_the_collector_pass_before_it_accounts_for_is_the_collectors_not_the_tasks() -> None:
|
||||
rows: Final = summarize(_SPEC, parse_stalls((_GC_PASS, _STREAM.replace("0.412", "0.731"))))
|
||||
assert [(row.coroutine, row.where, row.max_seconds, row.offending) for row in rows] == [
|
||||
("garbage collection", "gen=2", 0.731, False)
|
||||
]
|
||||
|
||||
|
||||
def test_a_hold_well_past_its_collector_pass_stays_on_the_task() -> None:
|
||||
short_pass: Final = _GC_PASS.replace("0.711", "0.200")
|
||||
rows: Final = summarize(_SPEC, parse_stalls((short_pass, _STREAM.replace("0.412", "0.731"))))
|
||||
assert [(row.coroutine, row.offending) for row in rows] == [("TaskHandle._run_coro()", True)]
|
||||
|
||||
|
||||
def test_a_hold_is_charged_to_the_longest_collector_pass_that_fits_it_not_to_their_sum() -> None:
|
||||
thread_side_pass: Final = _GC_PASS.replace("0.711", "0.627")
|
||||
rows: Final = summarize(
|
||||
_SPEC, parse_stalls((thread_side_pass, _GC_PASS.replace("0.711", "0.605"), _STREAM.replace("0.412", "0.614")))
|
||||
)
|
||||
assert [(row.coroutine, row.where, row.max_seconds, row.offending) for row in rows] == [
|
||||
("garbage collection", "gen=2", 0.614, False)
|
||||
]
|
||||
|
||||
|
||||
def test_a_collector_pass_explains_only_the_next_hold_and_only_one_at_least_as_long() -> None:
|
||||
rows: Final = summarize(_SPEC, parse_stalls((_GC_PASS, _STARTUP, _HANDLER)))
|
||||
assert sorted((row.coroutine, row.offending) for row in rows) == [
|
||||
("LifespanOn.main()", False),
|
||||
("RequestResponseCycle.run_asgi()", False),
|
||||
]
|
||||
assert summarize(_SPEC, parse_stalls((_GC_PASS,))) == ()
|
||||
|
||||
|
||||
def test_junit_properties_count_every_hold_and_only_the_holds_past_their_allowance_as_offending() -> None:
|
||||
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER)))
|
||||
properties: Final = dict(junit_properties(_SPEC, rows))
|
||||
assert properties["perf_event_loop_holds"] == 6
|
||||
assert properties["perf_event_loop_holds_limit"] == 6
|
||||
assert properties["perf_event_loop_offending_holds"] == 3
|
||||
assert properties["perf_event_loop_offending_holds_limit"] == 0
|
||||
assert properties["perf_event_loop_hold_seconds_max"] == 0.731
|
||||
assert math.isclose(float(properties["perf_event_loop_hold_seconds_total"]), 0.302 + 2 * 0.412 + 3 * 0.731)
|
||||
breakdown: Final = _breakdown(properties, "holds")
|
||||
assert [(slice.label, slice.value, slice.note, slice.over) for slice in breakdown] == [
|
||||
(
|
||||
"RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416",
|
||||
3,
|
||||
"3 of 2 allowed, max 0.73 s",
|
||||
True,
|
||||
),
|
||||
("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, "not on the allowlist, max 0.41 s", True),
|
||||
("LifespanOn.main() uvicorn/lifespan/on.py:86", 1, "1 of 3 allowed, max 0.30 s", False),
|
||||
]
|
||||
assert math.isclose(sum(slice.share or 0.0 for slice in breakdown), 1.0)
|
||||
|
||||
|
||||
def test_the_offending_breakdown_lists_only_the_locations_past_their_cap_by_how_far_over_they_are() -> None:
|
||||
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER)))
|
||||
offending: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "offending_holds")
|
||||
assert [(slice.label, slice.value, slice.share, slice.over) for slice in offending] == [
|
||||
("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 1, 1 / 3, True),
|
||||
("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, 2 / 3, True),
|
||||
]
|
||||
clean: Final = dict(junit_properties(_SPEC, summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER)))))
|
||||
assert clean["perf_event_loop_offending_holds"] == 0
|
||||
assert "perf_event_loop_offending_holds_breakdown" not in clean
|
||||
|
||||
|
||||
def test_the_seconds_breakdowns_carry_the_time_per_location_and_the_longest_single_holds() -> None:
|
||||
long_handler: Final = _HANDLER.replace("0.731", "0.950")
|
||||
rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, long_handler, _HANDLER)))
|
||||
properties: Final = dict(junit_properties(_SPEC, rows))
|
||||
per_location: Final = _breakdown(properties, "hold_seconds_total")
|
||||
assert [(slice.label.split()[0], slice.value) for slice in per_location] == [
|
||||
("RequestResponseCycle.run_asgi()", 0.950 + 0.731),
|
||||
("TaskHandle._run_coro()", 0.412),
|
||||
("LifespanOn.main()", 0.302),
|
||||
]
|
||||
assert math.isclose(sum(slice.share or 0.0 for slice in per_location), 1.0)
|
||||
longest: Final = _breakdown(properties, "hold_seconds_max")
|
||||
assert [(slice.label.split()[0], slice.value, slice.share, slice.over) for slice in longest] == [
|
||||
("RequestResponseCycle.run_asgi()", 0.950, None, False),
|
||||
("RequestResponseCycle.run_asgi()", 0.731, None, False),
|
||||
("TaskHandle._run_coro()", 0.412, None, True),
|
||||
("LifespanOn.main()", 0.302, None, False),
|
||||
]
|
||||
|
||||
|
||||
def test_the_longest_holds_breakdown_stops_at_ten() -> None:
|
||||
rows: Final = summarize(_SPEC, parse_stalls(tuple(_STARTUP for _ in range(12))))
|
||||
longest: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "hold_seconds_max")
|
||||
assert len(longest) == 10
|
||||
|
||||
|
||||
def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None:
|
||||
properties: Final = dict(junit_properties(_SPEC, ()))
|
||||
assert properties["perf_event_loop_holds"] == 0
|
||||
assert properties["perf_event_loop_offending_holds"] == 0
|
||||
assert properties["perf_event_loop_hold_seconds_max"] == 0
|
||||
assert not [name for name in properties if name.endswith("_breakdown")]
|
||||
|
|
@ -6,7 +6,7 @@ from typing import Final
|
|||
from cases.small_bodies_5rps import CASE
|
||||
from memory_readings import RssCapture
|
||||
from setup.datastore import DbCounters, StatementCalls
|
||||
from setup.observe import Observation, evaluate, junit_properties
|
||||
from setup.observe import Contributor, Observation, breakdown, evaluate, junit_properties
|
||||
from setup.performance_test_case import Limit
|
||||
from setup.wire import WireRecord
|
||||
|
||||
|
|
@ -36,7 +36,12 @@ def served(seq: int, total_s: float) -> WireRecord:
|
|||
|
||||
|
||||
def _key_lookup(calls: int) -> StatementCalls:
|
||||
return StatementCalls(queryid=1, query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1', calls=calls)
|
||||
return StatementCalls(
|
||||
queryid=1,
|
||||
query='SELECT * FROM "LiteLLM_VerificationToken" WHERE token = $1',
|
||||
calls=calls,
|
||||
total_exec_ms=calls * 2.0,
|
||||
)
|
||||
|
||||
|
||||
def test_junit_properties_carry_each_observed_metric_and_each_limit_including_unobserved_ones() -> None:
|
||||
|
|
@ -62,12 +67,163 @@ def test_junit_properties_carry_each_observed_metric_and_each_limit_including_un
|
|||
properties: Final = junit_properties(rows)
|
||||
|
||||
assert len({name for name, _ in properties}) == len(properties)
|
||||
assert dict(properties) == {
|
||||
assert {name: value for name, value in properties if not name.endswith("_breakdown")} == {
|
||||
"perf_unexpected_status_count": 0.0,
|
||||
"perf_gateway_added_seconds_p50": 0.5,
|
||||
"perf_gateway_added_seconds_p50_limit": 0.625,
|
||||
"perf_gateway_added_seconds_p99": 0.75,
|
||||
"perf_postgres_calls_per_request": 10.0,
|
||||
"perf_postgres_background_calls_per_request": 0.0,
|
||||
"perf_postgres_max_statement_per_request": 10.0,
|
||||
"perf_postgres_max_background_statement_per_request": 0.0,
|
||||
"perf_postgres_active_sessions": 0.02,
|
||||
"perf_postgres_slowest_statement_seconds": 0.002,
|
||||
"perf_redis_calls_per_request_limit": 35.0,
|
||||
}
|
||||
|
||||
|
||||
def _statement(queryid: int, query: str, calls: int) -> StatementCalls:
|
||||
return StatementCalls(queryid=queryid, query=query, calls=calls)
|
||||
|
||||
|
||||
def _observation_with(statements: tuple[StatementCalls, ...]) -> Observation:
|
||||
before: Final = tuple(replace(item, calls=0) for item in statements)
|
||||
return Observation(
|
||||
records=tuple(served(seq, 1.0) for seq in range(1000)),
|
||||
mock_requests=(),
|
||||
guardrail_calls=(),
|
||||
rss_before=NO_RSS,
|
||||
rss_after=NO_RSS,
|
||||
wall_seconds=600.0,
|
||||
db_before=DbCounters(commits=0, statements=before),
|
||||
db_after=DbCounters(commits=0, statements=statements),
|
||||
)
|
||||
|
||||
|
||||
MAX_STATEMENT_CASE: Final = replace(CASE, limits=(Limit("postgres_max_statement_per_request", max=0.5),))
|
||||
BACKGROUND_CAP_CASE: Final = replace(
|
||||
CASE,
|
||||
limits=(Limit("postgres_max_background_statement_per_request", max=0.9),),
|
||||
background_cap_exempt_tables=frozenset({"LiteLLM_DailyGuardrailMetrics"}),
|
||||
)
|
||||
GUARDRAIL_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyGuardrailMetrics" ("guardrail_id","date") VALUES ($1,$2)'
|
||||
MODEL_USAGE_UPSERT: Final = 'INSERT INTO "LiteLLM_DailyModelUsage" ("date","model") VALUES ($1,$2)'
|
||||
TEAM_REFRESH: Final = 'SELECT * FROM "LiteLLM_TeamMembership" WHERE user_id = $1'
|
||||
USER_LOOKUP: Final = 'SELECT * FROM "LiteLLM_UserTable" WHERE user_id = $1'
|
||||
|
||||
|
||||
def test_cache_refreshes_whose_sum_exceeds_the_cap_pass_the_per_statement_limit() -> None:
|
||||
statements: Final = (
|
||||
_statement(1, TEAM_REFRESH, calls=357),
|
||||
_statement(2, USER_LOOKUP, calls=245),
|
||||
_statement(3, 'INSERT INTO "LiteLLM_DailyModelUsage" VALUES ($1)', calls=1000),
|
||||
)
|
||||
|
||||
rows, exceeded = evaluate(MAX_STATEMENT_CASE, _observation_with(statements))
|
||||
|
||||
assert exceeded == []
|
||||
assert dict(junit_properties(rows))["perf_postgres_max_statement_per_request"] == 0.357
|
||||
assert dict(junit_properties(rows))["perf_postgres_calls_per_request"] == 0.602
|
||||
|
||||
|
||||
def test_a_lookup_on_every_request_fails_the_per_statement_limit_and_names_the_statement() -> None:
|
||||
statements: Final = (_statement(1, TEAM_REFRESH, calls=357), _statement(2, USER_LOOKUP, calls=1000))
|
||||
|
||||
_, exceeded = evaluate(MAX_STATEMENT_CASE, _observation_with(statements))
|
||||
|
||||
assert exceeded == [f"postgres_max_statement_per_request: observed 1.0000 > limit 0.5000 ({USER_LOOKUP})"], exceeded
|
||||
|
||||
|
||||
def test_per_statement_limit_without_db_counters_reports_nothing_observed() -> None:
|
||||
observation: Final = replace(_observation_with(()), db_before=None, db_after=None)
|
||||
|
||||
_, exceeded = evaluate(MAX_STATEMENT_CASE, observation)
|
||||
|
||||
assert exceeded == ["postgres_max_statement_per_request: limit set but nothing observed"]
|
||||
|
||||
|
||||
def test_per_flush_rollups_pass_the_background_cap_and_the_exempt_table_is_skipped() -> None:
|
||||
statements: Final = (
|
||||
_statement(1, GUARDRAIL_UPSERT, calls=1150),
|
||||
_statement(2, MODEL_USAGE_UPSERT, calls=810),
|
||||
_statement(3, USER_LOOKUP, calls=1000),
|
||||
)
|
||||
|
||||
rows, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements))
|
||||
|
||||
assert exceeded == []
|
||||
assert dict(junit_properties(rows))["perf_postgres_max_background_statement_per_request"] == 0.81
|
||||
|
||||
|
||||
def test_a_rollup_written_on_every_request_fails_the_background_cap_and_names_the_statement() -> None:
|
||||
statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1050), _statement(2, MODEL_USAGE_UPSERT, calls=1000))
|
||||
|
||||
_, exceeded = evaluate(BACKGROUND_CAP_CASE, _observation_with(statements))
|
||||
|
||||
assert exceeded == [
|
||||
f"postgres_max_background_statement_per_request: observed 1.0000 > limit 0.9000 ({MODEL_USAGE_UPSERT})"
|
||||
], exceeded
|
||||
|
||||
|
||||
def test_background_cap_without_exemptions_counts_the_guardrail_upsert() -> None:
|
||||
case: Final = replace(BACKGROUND_CAP_CASE, background_cap_exempt_tables=frozenset())
|
||||
statements: Final = (_statement(1, GUARDRAIL_UPSERT, calls=1150), _statement(2, MODEL_USAGE_UPSERT, calls=810))
|
||||
|
||||
_, exceeded = evaluate(case, _observation_with(statements))
|
||||
|
||||
assert exceeded == [
|
||||
f"postgres_max_background_statement_per_request: observed 1.1500 > limit 0.9000 ({GUARDRAIL_UPSERT})"
|
||||
], exceeded
|
||||
|
||||
|
||||
def _guardrail_rollup(calls: int) -> StatementCalls:
|
||||
return StatementCalls(
|
||||
queryid=2,
|
||||
query='INSERT INTO "LiteLLM_DailyGuardrailMetrics" ("guardrail_id","date") VALUES ($1,$2) ON CONFLICT DO UPDATE',
|
||||
calls=calls,
|
||||
total_exec_ms=calls * 1.0,
|
||||
)
|
||||
|
||||
|
||||
def _model_rollup(calls: int) -> StatementCalls:
|
||||
return StatementCalls(
|
||||
queryid=3,
|
||||
query='INSERT INTO "LiteLLM_DailyModelUsage" ("model","date") VALUES ($1,$2) ON CONFLICT DO UPDATE',
|
||||
calls=calls,
|
||||
total_exec_ms=calls * 1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_breakdown_names_the_slices_behind_each_metric_and_skips_plain_gauges() -> None:
|
||||
failed: Final = served(3, 1.0).model_copy(update={"status": 500, "ok": False})
|
||||
observation: Final = Observation(
|
||||
records=(served(0, 1.25), served(1, 1.5), served(2, 1.75), failed),
|
||||
mock_requests=(),
|
||||
guardrail_calls=(),
|
||||
rss_before=NO_RSS,
|
||||
rss_after=NO_RSS,
|
||||
wall_seconds=4.0,
|
||||
db_before=DbCounters(
|
||||
commits=100, statements=(_key_lookup(calls=100), _guardrail_rollup(calls=100), _model_rollup(calls=100))
|
||||
),
|
||||
db_after=DbCounters(
|
||||
commits=130, statements=(_key_lookup(calls=130), _guardrail_rollup(calls=124), _model_rollup(calls=112))
|
||||
),
|
||||
gateway_cpu_seconds=2.0,
|
||||
)
|
||||
|
||||
busiest: Final = breakdown("postgres_max_background_statement_per_request", None, observation)
|
||||
assert [(item.value, item.share) for item in busiest] == [(6.0, 24 / 36), (3.0, 12 / 36)]
|
||||
assert "LiteLLM_DailyGuardrailMetrics" in busiest[0].label
|
||||
assert breakdown("gateway_added_seconds", 50, observation) == (Contributor("chat_completions", 0.5, 1.0),)
|
||||
assert breakdown("unexpected_status_count", None, observation) == (
|
||||
Contributor("chat_completions status 500", 1.0, 1.0),
|
||||
)
|
||||
assert breakdown("cpu_seconds_per_request", None, observation) == ()
|
||||
|
||||
rows, _ = evaluate(CASE, observation)
|
||||
properties: Final = dict(junit_properties(rows))
|
||||
assert "perf_cpu_seconds_per_request_breakdown" not in properties
|
||||
assert properties["perf_unexpected_status_count_breakdown"] == (
|
||||
'[{"label": "chat_completions status 500", "value": 1.0, "share": 1.0}]'
|
||||
)
|
||||
|
|
|
|||
168
tests/e2e_harness/perf/setup/test_ramp.py
Normal file
168
tests/e2e_harness/perf/setup/test_ramp.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import replace
|
||||
from typing import Final
|
||||
|
||||
from cases.small_bodies_5rps import CASE
|
||||
from memory_readings import RssCapture
|
||||
from setup.observe import Observation, evaluate, junit_properties
|
||||
from setup.performance_test_case import Limit, Ramp, ramped
|
||||
from setup.plan import build_plan
|
||||
from setup.ramp import StepReading, send_offsets, step_properties, sustained_rps
|
||||
from setup.series import Sample, Series
|
||||
from setup.telemetry import GatewayTelemetry, HistogramTotals
|
||||
from setup.wire import WireRecord
|
||||
|
||||
NO_RSS: Final = RssCapture(readings=(), failures=())
|
||||
|
||||
|
||||
def test_send_offsets_put_each_step_s_requests_inside_its_window_in_plan_order() -> None:
|
||||
ramp: Final = Ramp(first_rps=10.0, last_rps=30.0, step_rps=10.0, step_seconds=10.0, held_p99_seconds=5.0)
|
||||
offsets: Final = send_offsets(ramp, seed=13)
|
||||
assert ramp.counts == (100, 200, 300)
|
||||
assert len(offsets) == ramp.total_requests
|
||||
assert offsets == tuple(sorted(offsets))
|
||||
for seq, offset in enumerate(offsets, start=1):
|
||||
step = ramp.step_of(seq)
|
||||
assert step * 10.0 <= offset < (step + 1) * 10.0, f"request {seq} at {offset:.2f}s is outside step {step}"
|
||||
assert send_offsets(ramp, seed=13) == offsets
|
||||
assert send_offsets(ramp, seed=14) != offsets
|
||||
|
||||
|
||||
def test_build_plan_carries_the_schedule_only_for_a_ramped_case() -> None:
|
||||
ramp: Final = Ramp(first_rps=2.0, last_rps=4.0, step_rps=2.0, step_seconds=5.0, held_p99_seconds=5.0)
|
||||
ramped_case: Final = replace(CASE, traffic=ramped(CASE.traffic, ramp, seed=3, concurrency=20))
|
||||
assert ramped_case.traffic.expected_seconds == 10.0
|
||||
assert [item.send_at_s for item in build_plan(ramped_case)] == list(send_offsets(ramp, seed=3))
|
||||
assert all(item.send_at_s is None for item in build_plan(CASE))
|
||||
|
||||
|
||||
def _step(rps: float, p99: float, completed_rps: float | None = None, unexpected: int = 0) -> StepReading:
|
||||
return StepReading(
|
||||
rps=rps,
|
||||
planned=int(rps * 90),
|
||||
completed_rps=rps if completed_rps is None else completed_rps,
|
||||
gateway_added_p99=p99,
|
||||
unexpected=unexpected,
|
||||
held_p99_seconds=5.0,
|
||||
)
|
||||
|
||||
|
||||
def test_sustained_rps_is_the_last_step_held_before_the_first_that_broke() -> None:
|
||||
assert sustained_rps((_step(10, 0.5), _step(20, 1.0), _step(30, 9.0), _step(40, 0.7))) == 20
|
||||
assert sustained_rps((_step(10, 0.5), _step(20, 1.0, completed_rps=17.0), _step(30, 1.0))) == 10
|
||||
assert sustained_rps((_step(10, 0.5), _step(20, 1.0, unexpected=19), _step(30, 1.0))) == 10
|
||||
assert sustained_rps((_step(10, 0.5), _step(20, 1.0, unexpected=18), _step(30, 1.0))) == 30
|
||||
assert sustained_rps((_step(10, 6.0), _step(20, 1.0))) == 0
|
||||
|
||||
|
||||
def _record(seq: int, started_at: float, total_s: float) -> WireRecord:
|
||||
return WireRecord(
|
||||
seq=seq,
|
||||
marker=f"perfreq-{seq:06d}-n",
|
||||
caller="key-a1",
|
||||
endpoint="chat_completions",
|
||||
stream=False,
|
||||
blocked=False,
|
||||
started_at=started_at,
|
||||
status=200,
|
||||
header_s=total_s,
|
||||
first_byte_s=total_s,
|
||||
first_token_s=None,
|
||||
total_s=total_s,
|
||||
text="",
|
||||
ok=True,
|
||||
mock_delay_s=1.0,
|
||||
call_id=None,
|
||||
error=None,
|
||||
)
|
||||
|
||||
|
||||
def test_a_floor_fails_when_the_second_step_s_latency_breaks_it_and_the_properties_carry_the_floor() -> None:
|
||||
ramp: Final = Ramp(first_rps=10.0, last_rps=20.0, step_rps=10.0, step_seconds=100.0, held_p99_seconds=2.0)
|
||||
case: Final = replace(
|
||||
CASE,
|
||||
traffic=ramped(CASE.traffic, ramp, seed=1, concurrency=50),
|
||||
limits=(Limit("sustained_rps", min=20),),
|
||||
)
|
||||
offsets: Final = send_offsets(ramp, seed=1)
|
||||
records: Final = tuple(
|
||||
_record(seq, 1000.0 + offset, 1.5 if ramp.step_of(seq) == 0 else 4.0)
|
||||
for seq, offset in enumerate(offsets, start=1)
|
||||
)
|
||||
observation: Final = Observation(
|
||||
records=records,
|
||||
mock_requests=(),
|
||||
guardrail_calls=(),
|
||||
rss_before=NO_RSS,
|
||||
rss_after=NO_RSS,
|
||||
wall_seconds=200.0,
|
||||
ramp=ramp,
|
||||
)
|
||||
first, second = observation.steps()
|
||||
assert first.held and first.gateway_added_p99 == 0.5 and first.completed_rps > 9.5
|
||||
assert not second.held and second.gateway_added_p99 == 3.0
|
||||
rows, exceeded = evaluate(case, observation)
|
||||
assert exceeded == ["sustained_rps: observed 10.0000 < floor 20.0000"]
|
||||
assert {("perf_sustained_rps", 10.0), ("perf_sustained_rps_floor", 20.0)} <= set(junit_properties(rows))
|
||||
assert dict(step_properties((first, second))) == {
|
||||
"perf_ramp_hold_p99_seconds": 2.0,
|
||||
"perf_ramp_step_10_gateway_added_p99": 0.5,
|
||||
"perf_ramp_step_10_completed_rps": first.completed_rps,
|
||||
"perf_ramp_step_10_unexpected": 0.0,
|
||||
"perf_ramp_step_10_gateway_added_p50": 0.5,
|
||||
"perf_ramp_step_20_gateway_added_p99": 3.0,
|
||||
"perf_ramp_step_20_completed_rps": second.completed_rps,
|
||||
"perf_ramp_step_20_unexpected": 0.0,
|
||||
"perf_ramp_step_20_gateway_added_p50": 3.0,
|
||||
}
|
||||
assert not evaluate(replace(case, limits=(Limit("sustained_rps", min=10),)), observation)[1]
|
||||
|
||||
|
||||
def test_each_step_reads_the_pod_s_cpu_redis_latency_and_connections_from_its_own_window() -> None:
|
||||
ramp: Final = Ramp(first_rps=10.0, last_rps=20.0, step_rps=10.0, step_seconds=100.0, held_p99_seconds=2.0)
|
||||
records: Final = tuple(
|
||||
_record(seq, 1000.0 + offset, 1.5) for seq, offset in enumerate(send_offsets(ramp, seed=1), start=1)
|
||||
)
|
||||
started: Final = min(record.started_at for record in records)
|
||||
redis: Final = tuple(
|
||||
Sample(at=started + at, value=GatewayTelemetry(redis_latency=HistogramTotals(total=total, count=count)))
|
||||
for at, total, count in ((-1.0, 0.0, 0.0), (50.0, 1.0, 100.0), (99.0, 2.0, 200.0), (199.0, 42.0, 300.0))
|
||||
)
|
||||
observation: Final = Observation(
|
||||
records=records,
|
||||
mock_requests=(),
|
||||
guardrail_calls=(),
|
||||
rss_before=NO_RSS,
|
||||
rss_after=NO_RSS,
|
||||
wall_seconds=200.0,
|
||||
ramp=ramp,
|
||||
cpu_series=Series(samples=tuple(Sample(at=started + at, value=5.0) for at in (10.0, 60.0, 110.0, 160.0))),
|
||||
connection_series=Series(
|
||||
samples=tuple(
|
||||
Sample(at=started + at, value=n) for at, n in ((1.0, 12), (50.0, 18), (150.0, 25), (199.0, 20))
|
||||
)
|
||||
),
|
||||
telemetry_series=Series(samples=redis),
|
||||
)
|
||||
first, second = observation.steps()
|
||||
first_completed: Final = round(first.completed_rps * ramp.step_seconds)
|
||||
assert first.cpu_seconds_per_request is not None and math.isclose(
|
||||
first.cpu_seconds_per_request, 10.0 / first_completed
|
||||
)
|
||||
assert first.redis_latency_mean_seconds is not None and math.isclose(first.redis_latency_mean_seconds, 0.01)
|
||||
assert first.postgres_peak_connections == 18
|
||||
assert second.redis_latency_mean_seconds is not None and math.isclose(second.redis_latency_mean_seconds, 0.4)
|
||||
assert second.postgres_peak_connections == 25
|
||||
assert first.time_to_first_token_p99 is None
|
||||
names: Final = {name for name, _ in step_properties((first, second)) if name.startswith("perf_ramp_step_10_")}
|
||||
assert names == {
|
||||
"perf_ramp_step_10_gateway_added_p99",
|
||||
"perf_ramp_step_10_completed_rps",
|
||||
"perf_ramp_step_10_unexpected",
|
||||
"perf_ramp_step_10_gateway_added_p50",
|
||||
"perf_ramp_step_10_cpu_seconds_per_request",
|
||||
"perf_ramp_step_10_redis_latency_mean_seconds",
|
||||
"perf_ramp_step_10_postgres_peak_connections",
|
||||
}
|
||||
173
tests/e2e_harness/perf/setup/test_ui_journeys.py
Normal file
173
tests/e2e_harness/perf/setup/test_ui_journeys.py
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from setup.ui_journeys import (
|
||||
UI_JOURNEY_LIMITS,
|
||||
Sample,
|
||||
assert_within_ceiling,
|
||||
database_failures,
|
||||
error_code,
|
||||
journeys,
|
||||
summarize,
|
||||
ui_junit_properties,
|
||||
)
|
||||
|
||||
|
||||
def test_first_journey_is_separate_from_repeated_latency() -> None:
|
||||
samples: Final = (Sample(seconds=80), *(Sample(seconds=i) for i in range(1, 101)))
|
||||
result: Final = summarize(samples)
|
||||
assert result.first_seconds == 80
|
||||
assert result.repeated_samples == 100
|
||||
assert result.repeated_p50_seconds == 50.5
|
||||
assert result.repeated_p95_seconds == 95
|
||||
assert result.repeated_p99_seconds == 99
|
||||
assert result.repeated_max_seconds == 100
|
||||
|
||||
|
||||
def test_failed_repeats_do_not_look_like_fast_successes() -> None:
|
||||
result: Final = summarize((Sample(seconds=10), Sample(seconds=0.01, error="AssertionError: no rows")))
|
||||
assert result.repeated_samples == 0
|
||||
assert result.repeated_p50_seconds is None
|
||||
assert result.repeated_p95_seconds is None
|
||||
assert result.repeated_p99_seconds is None
|
||||
assert result.repeated_max_seconds is None
|
||||
|
||||
|
||||
def test_small_sample_does_not_report_tail_percentiles() -> None:
|
||||
result: Final = summarize(tuple(Sample(seconds=1) for _ in range(6)))
|
||||
assert result.repeated_samples == 5
|
||||
assert result.repeated_p50_seconds == 1
|
||||
assert result.repeated_p95_seconds is None
|
||||
assert result.repeated_p99_seconds is None
|
||||
|
||||
|
||||
def test_failed_first_journey_emits_error_code_without_latency() -> None:
|
||||
sample: Final = Sample(seconds=120, error="timeout")
|
||||
assert summarize((sample,)).first_seconds is None
|
||||
assert dict(ui_junit_properties((sample,))) == {
|
||||
"perf_ui_journeys": 1,
|
||||
"perf_ui_failures": 1,
|
||||
"perf_ui_samples": 0,
|
||||
"perf_ui_repeated_samples": 0,
|
||||
"perf_ui_error_timeout": 1,
|
||||
}
|
||||
|
||||
|
||||
def test_junit_counts_failures_but_only_measures_successful_latency() -> None:
|
||||
samples: Final = (
|
||||
Sample(seconds=2, rows=25),
|
||||
Sample(seconds=3, rows=25),
|
||||
Sample(seconds=120, error="timeout"),
|
||||
Sample(seconds=1, error="AssertionError: Logs table rendered empty"),
|
||||
Sample(seconds=4, rows=25),
|
||||
)
|
||||
assert dict(ui_junit_properties(samples)) == {
|
||||
"perf_ui_journeys": 5,
|
||||
"perf_ui_failures": 2,
|
||||
"perf_ui_samples": 3,
|
||||
"perf_ui_repeated_samples": 2,
|
||||
"perf_ui_error_timeout": 1,
|
||||
"perf_ui_error_ui": 1,
|
||||
"perf_ui_seconds_p50": 3,
|
||||
"perf_ui_seconds_p90": 4,
|
||||
"perf_ui_seconds_max": 4,
|
||||
"perf_ui_first_seconds": 2,
|
||||
"perf_ui_repeated_seconds_p50": 3.5,
|
||||
"perf_ui_repeated_seconds_max": 4,
|
||||
}
|
||||
|
||||
|
||||
def test_all_journey_percentiles_include_first_and_use_nearest_rank_p90() -> None:
|
||||
samples: Final = tuple(Sample(seconds=seconds) for seconds in (10, 1, 9, 2, 8, 3, 7, 4, 6, 5))
|
||||
metrics: Final = dict(ui_junit_properties(samples))
|
||||
assert {key: value for key, value in metrics.items() if key.startswith("perf_ui_seconds_")} == {
|
||||
"perf_ui_seconds_p50": 5.5,
|
||||
"perf_ui_seconds_p90": 9,
|
||||
"perf_ui_seconds_max": 10,
|
||||
}
|
||||
assert metrics["perf_ui_samples"] == 10
|
||||
|
||||
|
||||
def test_journeys_have_unique_names_and_cover_every_dashboard_surface() -> None:
|
||||
names: Final = tuple(journey.name for journey in journeys("key"))
|
||||
assert len(names) == len(frozenset(names))
|
||||
assert frozenset(journey.surface for journey in journeys("key")) == {"logs", "usage", "caching", "models", "keys"}
|
||||
|
||||
|
||||
def _metrics(postgres: str, batch: str) -> str:
|
||||
return (
|
||||
"# TYPE litellm_postgres_failed_requests_total counter\n"
|
||||
f'litellm_postgres_failed_requests_total{{call_type="query_raw"}} {postgres}\n'
|
||||
"# TYPE litellm_batch_write_to_db_failed_requests_total counter\n"
|
||||
f'litellm_batch_write_to_db_failed_requests_total{{call_type="batch"}} {batch}\n'
|
||||
)
|
||||
|
||||
|
||||
def test_postgres_failures_include_background_batch_writes() -> None:
|
||||
assert database_failures(_metrics("0", "0")) == 0
|
||||
assert database_failures(_metrics("2", "3")) == 5
|
||||
|
||||
|
||||
def test_lazy_failure_counters_require_evidence_of_database_activity() -> None:
|
||||
assert (
|
||||
database_failures(
|
||||
"# TYPE litellm_postgres_total_requests_total counter\nlitellm_postgres_total_requests_total 3\n"
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text", ("", "# TYPE unrelated counter\nunrelated_total 0\n", _metrics("NaN", "0"), _metrics("-1", "0"))
|
||||
)
|
||||
def test_missing_or_invalid_metrics_cannot_pass_as_zero(text: str) -> None:
|
||||
with pytest.raises(AssertionError):
|
||||
_ = database_failures(text)
|
||||
|
||||
|
||||
def test_every_journey_has_a_latency_ceiling() -> None:
|
||||
assert frozenset(journey.name for journey in journeys("key")) == frozenset(UI_JOURNEY_LIMITS)
|
||||
|
||||
|
||||
def test_failed_sample_fails_the_journey_before_latency_is_judged() -> None:
|
||||
with pytest.raises(AssertionError, match="1 of 2 logs_open_24h samples failed: timeout"):
|
||||
assert_within_ceiling("logs_open_24h", (Sample(seconds=0.2, rows=25), Sample(seconds=120, error="timeout")))
|
||||
|
||||
|
||||
def test_slow_journey_fails_its_ceiling() -> None:
|
||||
fast: Final = tuple(Sample(seconds=1, rows=25) for _ in range(9))
|
||||
assert_within_ceiling("logs_open_24h", fast)
|
||||
with pytest.raises(AssertionError, match=r"logs_open_24h p90 25\.0s over 20s"):
|
||||
assert_within_ceiling("logs_open_24h", (*fast[:8], Sample(seconds=25, rows=25), Sample(seconds=25, rows=25)))
|
||||
with pytest.raises(AssertionError, match=r"logs_open_24h p50 11\.0s over 10s"):
|
||||
assert_within_ceiling("logs_open_24h", tuple(Sample(seconds=11, rows=25) for _ in range(3)))
|
||||
|
||||
|
||||
def test_failed_data_request_status_becomes_the_error_code() -> None:
|
||||
gateway_error: Final = Sample(seconds=0.4, error="AssertionError: Locator expected to be enabled", status=500)
|
||||
slow: Final = Sample(seconds=60.0, error="timeout", status=200, request="GET /spend/logs/ui")
|
||||
hung: Final = Sample(seconds=700.0, error="wall_clock TimeoutError: logs_next_page exceeded its 660s wall clock")
|
||||
assert (error_code(gateway_error), error_code(slow), error_code(hung)) == ("http_500", "timeout", "wall_clock")
|
||||
metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung)))
|
||||
assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1
|
||||
assert "perf_ui_error_ui" not in metrics
|
||||
|
||||
|
||||
def test_volume_properties_name_rows_and_bytes_per_queried_table() -> None:
|
||||
from setup.ui_query_fixture import TableVolume, volume_properties
|
||||
|
||||
properties: Final = volume_properties(
|
||||
(
|
||||
TableVolume(table="LiteLLM_SpendLogs", rows=850_000_000, bytes=1_000_000_000_000),
|
||||
TableVolume(table="LiteLLM_VerificationToken", rows=5_000, bytes=4_194_304),
|
||||
)
|
||||
)
|
||||
|
||||
assert properties == (
|
||||
("perf_ui_db_rows_LiteLLM_SpendLogs", 850_000_000),
|
||||
("perf_ui_db_bytes_LiteLLM_SpendLogs", 1_000_000_000_000),
|
||||
("perf_ui_db_rows_LiteLLM_VerificationToken", 5_000),
|
||||
("perf_ui_db_bytes_LiteLLM_VerificationToken", 4_194_304),
|
||||
)
|
||||
28
tests/e2e_harness/perf/setup/test_ui_query_isolation.py
Normal file
28
tests/e2e_harness/perf/setup/test_ui_query_isolation.py
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from setup.ui_query_isolation import settled_counter, validate_target
|
||||
|
||||
|
||||
@pytest.mark.parametrize("host", ("production.example.com", "db-perf-source.litellm.svc", "localhost"))
|
||||
def test_query_cancellation_rejects_non_lane_databases(host: str, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("E2E_UI_PERF_DISPOSABLE", "1")
|
||||
monkeypatch.setenv("E2E_RESET_SPEND_LOGS", "0")
|
||||
with pytest.raises(AssertionError, match="restricted"):
|
||||
validate_target(f"postgresql://user@{host}/litellm")
|
||||
|
||||
|
||||
def test_query_cancellation_requires_explicit_disposable_opt_in(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("E2E_UI_PERF_DISPOSABLE", raising=False)
|
||||
with pytest.raises(AssertionError, match="confirmation"):
|
||||
validate_target("postgresql://user@db-perf-perf-test-1.litellm.svc/litellm")
|
||||
|
||||
|
||||
def test_counter_stabilization_waits_for_delayed_failures(tmp_path: Path) -> None:
|
||||
readings: Final = iter((0.0, 0.0, 2.0, 2.0, 2.0))
|
||||
evidence: Final = tmp_path / "isolation.jsonl"
|
||||
assert settled_counter(lambda: next(readings), evidence) == 2
|
||||
assert len(evidence.read_text().splitlines()) == 5
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue