diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index c25e958242f..3aff9092530 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -128,6 +128,30 @@ Current limits: Bedrock cannot be mounted in record or replay (SigV4 signs the H The harness is fully typed with no error budget: `make lint-e2e-basedpyright` must report zero basedpyright errors, and CI enforces that on any PR touching `tests/e2e/**/*.py`. When a response field is untyped, model it in `models.py` (just the fields you read) and let pydantic validate it, rather than threading a `dict` or `Any` through the test +## Typed test metadata + +Separate from the coverage registry and additive to it: `@meta(Subject(...))` from `e2e_metadata.py` says what a test DRIVES, as closed enums rather than a string id. `@pytest.mark.covers("cell.id")` is untouched and keeps working exactly as before; the two markers coexist on the same test, and `@meta` always goes BELOW `@covers` so `Item.location` still anchors at the first decorator and every `source` deep link stays put + +```python +@pytest.mark.covers("quota_management.budget.key.blocks_over_limit") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=CHEAP_ANTHROPIC_MODEL, + mode=Mode.NONSTREAM, + ) +) +def test_bare_key_blocks_over_its_own_budget(...) -> None: ... +``` + +Every field is optional today (the backfill of the rest of the suite is a later PR) and every field is a closed enum, so a typo is a basedpyright error at the call site rather than a property that silently never appears. `capabilities` is a tuple even with one member, and it is deduped and sorted at declaration so the committed run artifacts diff cleanly. `Subject` is serialized with `dataclasses.asdict`, so a new scalar field needs no serializer edit; empty fields emit no `` at all. A declared `model` names the constant the test drives (`CHEAP_ANTHROPIC_MODEL`, the file's own `BACKEND`), never a copy of its value, so the property cannot claim one model while an env override runs another. `e2e_metadata` is stdlib-only and so are its call sites: `Provider` mirrors litellm's `LlmProviders` values instead of importing them, because tests/e2e is shipped to the runner image on its own and a `from litellm...` at module scope would make the litellm package a hard dependency of COLLECTING the suite. `TestProviderMirrorsLitellm` in `test_junit_properties.py` fails on drift wherever litellm is importable and skips where it is not, so adding a provider is one line in `e2e_metadata` + +The other half is recorded, not declared. `@step("POST /chat/completions")` goes on HARNESS helpers - client methods, `ResourceManager.key`, poll loops - never on a test, and appends its label to the running test's `user_properties` in call order. The list IS the test's user story, and because the label is recorded BEFORE the wrapped call, a failing test's LAST step is where it died. Consecutive duplicates collapse and the log caps at 50 entries, so a poll loop is one beat of the story rather than fifty. Nothing about steps is hand-written: the call sequence cannot drift from what the test actually did + +Both halves ride out as JUnit `` entries (`junit_properties.py`), repeated rather than delimiter-joined, since a free-text label has no separator that can be reserved + ## Coverage registry The set of tests we want is a registry checked into this repo, one row per behavior; that file is the definition of done and the denominator. Each e2e test declares what it covers with `@pytest.mark.covers("...")`, and a small collector diffs the registry against the tests and ships coverage to the existing Grafana. No Allure, no new dependencies diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 8776d00d502..3f503e1d15c 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -39,10 +39,11 @@ from e2e_config import ( ) from e2e_db import RESET_OPT_IN_ENV, reset_spend_logs, run_spend_log_cleanup from e2e_http import unwrap +from e2e_metadata import STEPS from fixture_mode import fixture_mode_collection_error, fixture_report_lines from fixture_mode import pytest_fixture_setup as pytest_fixture_setup from idp import Identity, Keycloak, keycloak_from_env -from junit_properties import attach_result_properties +from junit_properties import attach_result_properties, attach_step_properties from lifecycle import ProxyClientProvider, ResourceManager from models import TeamNewBody, UserNewBody, UserNewResponse from provider_cache_routing import LIVE_PROVIDER_REQUIRED @@ -109,6 +110,11 @@ def pytest_configure(config: pytest.Config) -> None: "markers", "covers(cell_id, *, exercised_on=()): coverage-registry cell(s) this test covers", ) + config.addinivalue_line( + "markers", + "meta(subject): typed e2e_metadata.Subject describing what this test drives" + " (domain/route/provider/model/capabilities/mode); attach it with @meta(Subject(...))", + ) config.addinivalue_line( "markers", "replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes " @@ -259,7 +265,14 @@ def pytest_runtest_makereport( item: pytest.Item, call: pytest.CallInfo[None] ) -> Generator[None, pytest.TestReport, pytest.TestReport]: """Stash the call-phase outcome so teardown can tell a passed test from a - failed one without re-deriving it.""" + failed one without re-deriving it, and attach the runtime-recorded steps. + + The steps cannot ride along with the other properties in + `pytest_collection_modifyitems`: that hook runs before any test body has, so + the recorder is empty there. They are attached on every call-phase outcome, + failures included -- a failing test's last step is where it died, which is the + whole reason the field exists. + """ report = yield if item.get_closest_marker("mcp_oauth_live") is not None and call.excinfo is not None: # Publish code locations only, never exception messages, source text or locals. @@ -270,6 +283,7 @@ def pytest_runtest_makereport( report.user_properties = list(item.user_properties) if report.when == "call": item.stash[_CALL_PASSED] = report.passed + attach_step_properties(item) return report @@ -306,6 +320,21 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: ) +@pytest.fixture(autouse=True) +def _record_steps() -> Iterator[None]: # pyright: ignore[reportUnusedFunction] # autouse: nothing names it + """Empty the step log before each test, so the story a test tells is its own. + + Autouse and unconditional: the recorder is written by @step-decorated harness + helpers, which a test reaches through fixtures as readily as through its own + body, so setup-phase steps have to be kept too. The log is read after the call + phase by `pytest_runtest_makereport`; a test that dies partway keeps the + partial list, which is the point -- its last element is where it died. + """ + STEPS.reset() + yield + STEPS.reset() + + @pytest.fixture(scope="session") def proxy() -> ProxyClient: """The shared ProxyClient every suite's client is built from. Suite `client` diff --git a/tests/e2e/e2e_metadata.py b/tests/e2e/e2e_metadata.py new file mode 100644 index 00000000000..061d4c70d0f --- /dev/null +++ b/tests/e2e/e2e_metadata.py @@ -0,0 +1,392 @@ +"""Typed per-test metadata for the e2e suite: what a test drives, and what it did. + +Two halves, deliberately separated. + +The DECLARED half is `Subject`: one frozen dataclass passed as the single +positional argument of `@meta(...)`. Every field is a closed enum (or a free +string for `model`), so a typo is a basedpyright error at the call site rather +than a silently dropped property. `dataclasses.asdict()` turns the whole thing +into pairs with no per-field plumbing -- adding a scalar field later +needs zero serializer changes. + +The RECORDED half is `steps`, and it is NOT a field of `Subject`. Steps are +appended at runtime by `@step`-decorated harness helpers, in call order, so the +list IS the test's user story and its last element is where a failing test died. +Putting it on the declarable dataclass would invite hand-writing it, which is +exactly what it replaces. + +Stdlib-only on purpose, and that includes the call sites. tests/e2e is a +black-box HTTP suite that imports litellm in zero files and is shipped to the +runner image as tests/e2e alone; a `from litellm...` at the top of a test module +would make the litellm package a COLLECTION-time dependency of the whole suite, +so an image without it would fail collection rather than run tests. `Provider` +below therefore mirrors litellm's `LlmProviders` values here instead of +importing them, and `TestProviderMirrorsLitellm` in test_junit_properties.py +fails wherever litellm IS importable if the two ever drift. +""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from dataclasses import dataclass +from enum import Enum +from functools import wraps +from typing import Final, ParamSpec, TypeVar, cast + +import pytest + + +class Domain(str, Enum): + """The OSS issue-label taxonomy, verbatim. + + Shared with GitHub issue labels so an issue and a test join on one string. + Exactly one per test; `UNKNOWN` is the honest answer, not an omission. + """ + + LLM_TRANSLATION = "llm-translation" + SPEND_BUDGETS = "spend-budgets" + UI = "ui" + MCP = "mcp" + OBSERVABILITY = "observability" + ROUTING = "routing" + DEPLOY_OPS = "deploy-ops" + COST_MAP = "cost-map" + PROXY_AUTH = "proxy-auth" + GUARDRAILS = "guardrails" + MANAGEMENT = "management" + SDK = "sdk" + PASSTHROUGH = "passthrough" + DB = "db" + CACHING = "caching" + DOCS = "docs" + AGENTS_API = "agents-api" + UNKNOWN = "unknown" + + +class Route(str, Enum): + """The customer-facing HTTP surface the test DRIVES. + + Orthogonal to the suite directory: a reliability test in router/ and a + logging test in logging/ both drive `CHAT_COMPLETIONS`, which is why this is + per-test and not per-module. + + "route" here means endpoint, matching litellm's own `LiteLLMRoutes` + (litellm/proxy/_types.py). The coverage registry's `LlmCell.route` uses the + same word for PROVIDER; that is a different namespace and is left alone. + + Deliberately collapsed against the registry's `LlmEndpoint`: + images_generations + images_edits -> IMAGES, audio_speech + + audio_transcriptions -> AUDIO, bedrock_native + google_native -> + PASSTHROUGH. Those splits are wire detail, not a customer-facing surface, + and `model` + `capabilities` already carry them. + """ + + # Core: each is the subject of many e2e tests. + CHAT_COMPLETIONS = "chat_completions" + MESSAGES = "messages" + RESPONSES = "responses" + EMBEDDINGS = "embeddings" + COMPLETIONS = "completions" + FILES = "files" + BATCHES = "batches" + PASSTHROUGH = "passthrough" + MCP = "mcp" + GUARDRAILS = "guardrails" + KEY_MANAGEMENT = "key_management" + TEAM_MANAGEMENT = "team_management" + SPEND_REPORTING = "spend_reporting" + MODEL_MANAGEMENT = "model_management" + + # Long tail: real, but one or two test files each. + IMAGES = "images" + AUDIO = "audio" + MODERATIONS = "moderations" + RERANK = "rerank" + OCR = "ocr" + VECTOR_STORES = "vector_stores" + REALTIME = "realtime" + A2A = "a2a" + USER_MANAGEMENT = "user_management" + BUDGET_MANAGEMENT = "budget_management" + + # Ops surfaces. logging/, load/, other/ and ui/ have no LLM route of their + # own and would otherwise have to lie. + HEALTH = "health" + METRICS = "metrics" + PROXY_CONFIG = "proxy_config" + ADMIN_UI = "admin_ui" + + +class Provider(str, Enum): + """The upstream LLM provider the test drives, spelled exactly as litellm's + own `LlmProviders` (litellm/types/utils.py) spells it. + + A deliberate mirror, not an import. Importing `LlmProviders` at the top of a + test module pulls `litellm/__init__` (measured: 1.44s, 2474 modules) and, + worse, makes the litellm package a hard dependency of COLLECTING tests/e2e -- + which is shipped to the e2e runner image on its own, so a missing package + would not slow the suite down, it would error every test out at collection. + The suite has zero runtime litellm imports and this keeps it that way. + + The mirror cannot drift silently: `TestProviderMirrorsLitellm` in + test_junit_properties.py asserts every value here is a real `LlmProviders` + value, and runs wherever litellm is importable (dev checkouts, the repo's own + CI) while skipping where it is not. Adding a provider is one line here. + """ + + OPENAI = "openai" + OPENAI_LIKE = "openai_like" + CUSTOM_OPENAI = "custom_openai" + AZURE = "azure" + AZURE_AI = "azure_ai" + ANTHROPIC = "anthropic" + GEMINI = "gemini" + VERTEX_AI = "vertex_ai" + BEDROCK = "bedrock" + SAGEMAKER = "sagemaker" + XAI = "xai" + GROQ = "groq" + DEEPSEEK = "deepseek" + MISTRAL = "mistral" + COHERE = "cohere" + PERPLEXITY = "perplexity" + OPENROUTER = "openrouter" + TOGETHER_AI = "together_ai" + FIREWORKS_AI = "fireworks_ai" + CEREBRAS = "cerebras" + SAMBANOVA = "sambanova" + NVIDIA_NIM = "nvidia_nim" + DATABRICKS = "databricks" + WATSONX = "watsonx" + OLLAMA = "ollama" + VLLM = "vllm" + HOSTED_VLLM = "hosted_vllm" + VOYAGE = "voyage" + JINA_AI = "jina_ai" + DEEPGRAM = "deepgram" + ELEVENLABS = "elevenlabs" + ASSEMBLYAI = "assemblyai" + LITELLM_PROXY = "litellm_proxy" + + +class Capability(str, Enum): + """A MODEL feature the test depends on, anchored 1:1 to a `supports_*` key + in model_prices_and_context_window.json. + + Not to be confused with the coverage registry's `LlmCell.capability`, which + means the endpoint feature under test (`basic`, `multi_turn`, ...) and half + of whose values have no `supports_*` key at all. + + Plural by necessity, never a single enum: `thinking_with_tool_use` and + `tool_search_history` only exist in the registry because a single-enum field + had nowhere to put a conjunction. Here they are + (REASONING, FUNCTION_CALLING) and (TOOL_SEARCH,). + + `audio_output` is deliberately absent: litellm/utils.py reads + `supports_audio_input` for it, so the value would silently alias + AUDIO_INPUT. Add it once that bug is fixed upstream. + """ + + FUNCTION_CALLING = "function_calling" + PARALLEL_FUNCTION_CALLING = "parallel_function_calling" + TOOL_CHOICE = "tool_choice" + TOOL_SEARCH = "tool_search" + VISION = "vision" + PDF_INPUT = "pdf_input" + AUDIO_INPUT = "audio_input" + REASONING = "reasoning" + WEB_SEARCH = "web_search" + PROMPT_CACHING = "prompt_caching" + RESPONSE_SCHEMA = "response_schema" + MID_CONVERSATION_SYSTEM = "mid_conversation_system" + + +class Mode(str, Enum): + """How the route was driven. The registry's cell ids already carry this + axis as a segment (221 `.nonstream.`, 37 `.stream.`).""" + + NONSTREAM = "nonstream" + STREAM = "stream" + BATCH = "batch" + WEBSOCKET = "websocket" + + +@dataclass(frozen=True, slots=True) +class Subject: + """What a test is about. + + Every field is optional in this first phase -- nothing is enforced, and the + backfill of the existing ~908 tests comes later. Named `Subject` rather than + `TestMeta` because pytest tries to collect any imported class named `Test*` + and would warn in every one of the ~570 modules that import it. + """ + + domain: Domain | None = None + route: Route | None = None + provider: Provider | None = None + model: str | None = None + capabilities: tuple[Capability, ...] = () + mode: Mode | None = None + + def __post_init__(self) -> None: + """Canonicalize capabilities at declaration: deduped and sorted by + value, so the committed run files diff cleanly however a test spelled + the tuple, and the serializer stays field-agnostic.""" + object.__setattr__( + self, + "capabilities", + tuple(sorted(dict.fromkeys(self.capabilities), key=lambda c: c.value)), + ) + + +def meta(subject: Subject) -> pytest.MarkDecorator: + """Attach a `Subject` to a test: `@meta(Subject(route=Route.RESPONSES, ...))`. + + A typed wrapper around `pytest.mark.meta` (registered in conftest.py's + `pytest_configure`, like `covers`) so passing the wrong thing is a type + error rather than a property that silently never appears. + + Separate from `@pytest.mark.covers` on purpose: `covers` args are flattened + by `dedupe_covers`, which drops non-strings silently, and by + tests/integration/conftest.py, which has no such filter and would hard-fail + collection. `covers` is untouched by this change. + """ + return pytest.mark.meta(subject) + + +# --- The recorded half: steps ------------------------------------------------- + +_P = ParamSpec("_P") +_R = TypeVar("_R") + +# A retrying helper (poll_cost_row) or a load test calling a decorated helper in +# a loop would otherwise emit thousands of entries per testcase. +MAX_STEPS: Final = 50 +MAX_STEP_CHARS: Final = 200 + + +class _StepRecorder: + """The ordered step log for the running test. + + A plain lock-guarded list rather than a ContextVar: ContextVars do not + propagate into worker threads, and several e2e helpers call out from + threads. Under xdist each worker is its own process, so there is no + cross-test bleed beyond what the per-test reset already handles. + """ + + def __init__(self) -> None: + self._lock = threading.Lock() + self._steps: list[str] = [] + + def reset(self) -> None: + """Called by an autouse fixture at setup, so each test starts empty.""" + with self._lock: + self._steps.clear() + + def record(self, label: str) -> None: + cleaned = " ".join(label.split())[:MAX_STEP_CHARS] + if not cleaned: + return + with self._lock: + # A poll loop is one step in the story, not fifty. + if self._steps and self._steps[-1] == cleaned: + return + if len(self._steps) >= MAX_STEPS: + return + self._steps.append(cleaned) + + def taken(self) -> tuple[str, ...]: + with self._lock: + return tuple(self._steps) + + +STEPS: Final = _StepRecorder() + + +def step(label: str) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: + """Record `label` on the running test whenever this helper is called. + + Goes on HARNESS helpers (client methods, fixtures), never on tests. The + label is recorded BEFORE the wrapped call, so a helper that raises still + leaves its own label as the last element -- which is the whole point: the + last step is where the test died. + """ + + def decorate(fn: Callable[_P, _R]) -> Callable[_P, _R]: + @wraps(fn) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + STEPS.record(label) + return fn(*args, **kwargs) + + return wrapper + + return decorate + + +# --- Serialization: asdict() -> pairs, no per-field plumbing ------- + +# The only field-specific knowledge in the module: which fields are plural, and +# what their repeated is called. A new SCALAR field needs no edit. +_REPEATED: Final[dict[str, str]] = {"capabilities": "capability"} + + +def _scalar(value: object) -> str: + """`str(member)` on a (str, Enum) gives 'Route.RESPONSES', not 'responses' + -- StrEnum would not, but it is 3.11+ and this repo floors at 3.10. So the + value is read explicitly, once, for every enum field.""" + if isinstance(value, Enum): + # `Enum.value` is `Any` by construction, for every enum there has ever + # been; this is the one place in the module that reads it, and `str()` + # lands it back in the type system immediately. + return str(value.value) # pyright: ignore[reportAny] # Enum.value is Any for every enum + return str(value) + + +def _members(value: object) -> tuple[object, ...]: + """The elements of a plural field, whatever `asdict` rebuilt it as. + + `asdict` reconstructs a tuple field as a tuple of the same members, but hands + it back inside an untyped dict, so the elements are re-declared as plain + objects here and converted by `_scalar` like any other value. + """ + return cast("tuple[object, ...]", value) if isinstance(value, tuple) else () + + +def _declared_subject(args: tuple[object, ...]) -> Subject | None: + """The `Subject` a `meta` marker carries, or None for anything else. + + `Mark.args` is `tuple[Any, ...]`; taking it as `tuple[object, ...]` is what + keeps the Any from leaking past this line. A bare `@pytest.mark.meta` (no + args) and a `@pytest.mark.meta("spend-budgets")` (wrong type) both land here, + and neither may produce a property whose value is a repr. + """ + first = args[0] if args else None + return first if isinstance(first, Subject) else None + + +def subject_properties(item: pytest.Item) -> tuple[tuple[str, str], ...]: + """The declared half, in dataclass field order. Empty fields emit nothing; + the emitter is what guarantees every key exists in the JSON.""" + from dataclasses import asdict # local: keeps module import trivially cheap + + marker = item.get_closest_marker("meta") + if marker is None: + return () + subject = _declared_subject(marker.args) + if subject is None: + return () + fields: dict[str, object] = asdict(subject) + pairs: list[tuple[str, str]] = [] + for name, value in fields.items(): + repeated = _REPEATED.get(name) + if repeated is not None: + pairs.extend((repeated, _scalar(member)) for member in _members(value)) + elif value is not None and value != "": + pairs.append((name, _scalar(value))) + return tuple(pairs) + + +def step_properties() -> tuple[tuple[str, str], ...]: + """The recorded half. Appended after the call phase, never at collection.""" + return tuple(("step", label) for label in STEPS.taken()) diff --git a/tests/e2e/junit_properties.py b/tests/e2e/junit_properties.py index b9f5da871ae..17e891922a6 100644 --- a/tests/e2e/junit_properties.py +++ b/tests/e2e/junit_properties.py @@ -20,6 +20,7 @@ from collections.abc import Iterable import pytest from coverage_registry.management_cases import case_properties +from e2e_metadata import step_properties, subject_properties # Hardcoded because the runner image copies tests/e2e/ to /app/e2e, so nothing # at runtime names this suite's place in the repo. test_junit_properties.py @@ -89,13 +90,22 @@ def covers_from_item(item: pytest.Item) -> tuple[str, ...]: def result_properties(item: pytest.Item) -> tuple[tuple[str, str], ...]: """The custom signals a standard reporter cannot derive: the normalized suite - package, the comma-joined coverage-registry cell ids this test covers, and the - repo-relative `path:line` its source sits at.""" - return ( + package, the comma-joined coverage-registry cell ids this test covers, the + repo-relative `path:line` its source sits at, and the typed `@meta(Subject(...))` + fields. + + The fixed three-tuple prefix is load-bearing and stays byte-identical: Loki, + Grafana, the status page and tests/integration/conftest.py all read + `package`/`covers`/`source` today. `subject_properties` only ever appends, and + appends nothing at all for a test with no `meta` marker -- which is every test + in the suite until the backfill lands. + """ + fixed = ( ("package", package_from_nodeid(item.nodeid)), ("covers", ",".join(covers_from_item(item))), ("source", source_from_item(item)), - ) + case_properties(item.nodeid) + ) + return fixed + case_properties(item.nodeid) + subject_properties(item) def attach_result_properties(item: pytest.Item) -> None: @@ -105,3 +115,20 @@ def attach_result_properties(item: pytest.Item) -> None: if any(name == "package" for name, _ in item.user_properties): return item.user_properties.extend(result_properties(item)) + + +def attach_step_properties(item: pytest.Item) -> None: + """Attach the runtime-recorded steps after the call phase. + + Separate from `attach_result_properties` because it cannot share its home: + that one runs in `pytest_collection_modifyitems`, before any test body has + executed, so the recorder is necessarily empty there. + + Any `step` entries already on the item are dropped first. The suite runs with + `--reruns 1`, so a flaky test's second attempt would otherwise append a second + copy of the story behind the first, and the report would read as one very long + test that did everything twice. Last attempt wins, which is the attempt whose + outcome JUnit records. + """ + item.user_properties[:] = [entry for entry in item.user_properties if entry[0] != "step"] + item.user_properties.extend(step_properties()) diff --git a/tests/e2e/lifecycle.py b/tests/e2e/lifecycle.py index eb9704d4dcb..b08579d3038 100644 --- a/tests/e2e/lifecycle.py +++ b/tests/e2e/lifecycle.py @@ -12,6 +12,7 @@ from builtins import ExceptionGroup from dataclasses import dataclass, field from typing import Callable, Final, List, Protocol, runtime_checkable +from e2e_metadata import step from proxy_client import ProxyClient from models import KeyGenerateBody @@ -69,6 +70,7 @@ class ResourceManager: response model can be deferred directly.""" self._cleanups.append(cleanup) + @step("generate virtual key") def key(self, models: list[str] | None = None, user_id: str | None = "e2e-test-user") -> str: """Create a virtual key; delete it on teardown. `models` restricts which models the key may call (None/[] means all). `user_id` is required for @@ -78,6 +80,7 @@ class ResourceManager: self.defer(lambda: self.client.delete_key(key)) return key + @step("register end user") def customer(self, customer_id: str) -> str: """Track an end-user id (from the `user` param); delete it on teardown.""" self.defer(lambda: self.client.delete_customers([customer_id])) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index c6acd449884..1e82937addf 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -5,6 +5,7 @@ addopts = --strict-markers --strict-config --reruns 1 --only-rerun "kind='network'" --only-rerun "status_code=5[0-9][0-9]" markers = e2e: live test that requires a running proxy and real provider keys + meta: typed e2e_metadata.Subject describing what this test drives (domain/route/provider/model/capabilities/mode); attach it with @meta(Subject(...)), never as a bare pytest.mark replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes zero provider calls in replay mode; the record/replay CI lane selects it with -m replayable load: heavy throughput/load test; collected last so it never perturbs latency-sensitive suites weekly: real-provider anomaly load test that spends real money; deselected unless E2E_WEEKLY_ANOMALY is set diff --git a/tests/e2e/quota_management/budgets/budget_client.py b/tests/e2e/quota_management/budgets/budget_client.py index 087dc8ca522..701d67f8c6e 100644 --- a/tests/e2e/quota_management/budgets/budget_client.py +++ b/tests/e2e/quota_management/budgets/budget_client.py @@ -17,6 +17,7 @@ from datetime import datetime from pydantic import AliasPath, BaseModel, Field, RootModel from e2e_http import NoBody, Result, StreamingResponse, Success, unwrap +from e2e_metadata import step from proxy_client import ProxyClient from models import ( AnthropicMessagesBody, @@ -224,6 +225,7 @@ class BudgetClient: # ---- generic key ops (delegate to the shared ProxyClient) --------------- + @step("generate virtual key") def generate_key( self, *, @@ -256,12 +258,14 @@ class BudgetClient: def delete_key(self, key: str) -> None: self.proxy.delete_key(key) + @step("read the key's budget windows") def key_budget_windows(self, key: str) -> list[BudgetWindowState]: """A key's budget_limits windows as /key/info stores them. Each window's reset_at is advanced by the reset job in the same pass that zeroes the window's spend counter, so a strictly-later value proves the wipe ran.""" return self.proxy.key_info(key).budget_limits or [] + @step("read the team's budget windows") def team_budget_windows(self, team_id: str) -> list[BudgetWindowState]: """Team analog of key_budget_windows, read from /team/info.""" match self._team_info(team_id): @@ -275,6 +279,7 @@ class BudgetClient: # ---- chat (raw HTTP outcome: a budget block surfaces as a non-2xx) -- + @step("POST /chat/completions") def chat( self, key: str, @@ -297,6 +302,7 @@ class BudgetClient: ), ) + @step("POST /v1/messages") def messages( self, key: str, @@ -317,6 +323,7 @@ class BudgetClient: # ---- internal user -------------------------------------------------- + @step("create internal user with a budget") def create_user(self, *, max_budget: float, budget_duration: str | None = None) -> str: return unwrap( self.proxy.transport.post( @@ -335,6 +342,7 @@ class BudgetClient: response_type=NoBody, ) + @step("read /user/info for recorded spend") def user_info(self, user_id: str) -> UserInfoRow | None: result = self.proxy.transport.get( "/user/info", @@ -350,6 +358,7 @@ class BudgetClient: # ---- customer / end-user ------------------------------------------- + @step("create end user with a budget") def create_customer( self, customer_id: str, @@ -369,6 +378,7 @@ class BudgetClient: # ---- organization --------------------------------------------------- + @step("create organization with a budget") def create_org(self, *, max_budget: float, alias: str, budget_duration: str | None = None) -> str: return unwrap( self.proxy.transport.post( @@ -383,6 +393,7 @@ class BudgetClient: ) ).organization_id + @step("read the organization's budget id") def org_budget_id(self, org_id: str) -> str | None: """The id of the budget row backing an org; its budget_reset_at is read via budget_info (LIT-4570: /organization/new stores budget_duration without @@ -409,6 +420,7 @@ class BudgetClient: # ---- team ----------------------------------------------------------- + @step("create team with a budget") def create_team( self, *, @@ -463,6 +475,7 @@ class BudgetClient: assert last is not None raise AssertionError(last) + @step("add a member to the team") def add_team_member(self, team_id: str, user_id: str, *, max_budget_in_team: float | None = None) -> None: last_body = "" for attempt in range(_TEAM_READY_ATTEMPTS): @@ -484,6 +497,7 @@ class BudgetClient: break raise AssertionError(last_body) + @step("update the member's in-team budget") def update_team_member( self, team_id: str, @@ -504,6 +518,7 @@ class BudgetClient: ) assert resp.ok, resp.body + @step("read the member budget's reset_at") def member_budget_reset_at(self, team_id: str, user_id: str) -> str | None: """The member's per-team budget_reset_at as /team/info reports it, or None if no reset is scheduled. The reset job advances this each time the window @@ -519,6 +534,7 @@ class BudgetClient: # ---- tag ------------------------------------------------------------ + @step("create tag with a budget") def create_tag(self, name: str, *, max_budget: float) -> str: resp = self.proxy.transport.send( "/tag/new", @@ -538,6 +554,7 @@ class BudgetClient: # ---- model access group --------------------------------------------- + @step("set the model access group's budget") def set_access_group_budget( self, access_group: str, @@ -561,6 +578,7 @@ class BudgetClient: ) ) + @step("read the model access group's budget") def access_group_budget(self, access_group: str) -> AccessGroupBudgetResponse: return unwrap( self.proxy.transport.get( @@ -581,6 +599,7 @@ class BudgetClient: # ---- budget table --------------------------------------------------- + @step("create a shared budget") def create_budget( self, *, @@ -611,6 +630,7 @@ class BudgetClient: response_type=NoBody, ) + @step("read /budget/info") def budget_info(self, budget_id: str) -> tuple[BudgetRow, ...]: result = self.proxy.transport.post( "/budget/info", diff --git a/tests/e2e/quota_management/budgets/test_budget_crud_e2e.py b/tests/e2e/quota_management/budgets/test_budget_crud_e2e.py index 5070ec89704..520de814c85 100644 --- a/tests/e2e/quota_management/budgets/test_budget_crud_e2e.py +++ b/tests/e2e/quota_management/budgets/test_budget_crud_e2e.py @@ -10,12 +10,19 @@ from datetime import datetime, timezone import pytest from budget_client import BudgetClient +from e2e_metadata import Domain, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @pytest.mark.covers("mgmt.budget.new.persists") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BUDGET_MANAGEMENT, + ) +) def test_budget_crud_roundtrip(client: BudgetClient, resources: ResourceManager) -> None: budget_id = client.create_budget(max_budget=12.5, soft_budget=10.0, budget_duration="30d") resources.defer(lambda: client.delete_budget(budget_id)) @@ -38,6 +45,12 @@ def test_budget_crud_roundtrip(client: BudgetClient, resources: ResourceManager) @pytest.mark.covers("mgmt.budget.delete.persists") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BUDGET_MANAGEMENT, + ) +) def test_budget_delete_removes_it(client: BudgetClient, resources: ResourceManager) -> None: budget_id = client.create_budget(max_budget=1.0) resources.defer(lambda: client.delete_budget(budget_id)) @@ -45,6 +58,12 @@ def test_budget_delete_removes_it(client: BudgetClient, resources: ResourceManag assert not client.budget_info(budget_id), "budget still present after delete" +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.KEY_MANAGEMENT, + ) +) def test_budget_duration_schedules_reset_on_key(client: BudgetClient, resources: ResourceManager) -> None: key = client.generate_key(max_budget=10.0, budget_duration="30d") resources.defer(lambda: client.delete_key(key)) diff --git a/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py b/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py index 8a9be1d1385..9988c3d6795 100644 --- a/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py +++ b/tests/e2e/quota_management/budgets/test_budget_enforcement_e2e.py @@ -19,6 +19,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -56,6 +57,15 @@ def _assert_blocked_422(client: BudgetClient, key: str) -> StreamingResponse: class TestBudgetBlocksPerLevel: @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_bare_key_blocks_over_its_own_budget(self, client: BudgetClient, resources: ResourceManager) -> None: key = client.generate_key(max_budget=TINY_CAP) resources.defer(lambda: client.delete_key(key)) @@ -63,6 +73,15 @@ class TestBudgetBlocksPerLevel: _assert_blocked_422(client, key) @pytest.mark.covers("quota_management.budget.team.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_team_budget_blocks_every_team_key(self, client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team(alias=f"e2e-budget-team-{unique_marker()}", max_budget=TINY_CAP) resources.defer(lambda: client.delete_team(team_id)) @@ -79,6 +98,15 @@ class TestBudgetBlocksPerLevel: ) @pytest.mark.covers("quota_management.budget.internal_user.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_user_budget_enforced_across_their_personal_keys( self, client: BudgetClient, resources: ResourceManager ) -> None: @@ -113,6 +141,15 @@ class TestBudgetBlocksPerLevel: require_successful_call(team_result) @pytest.mark.covers("quota_management.budget.end_user.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_end_user_budget_blocks_attributed_calls( self, client: BudgetClient, resources: ResourceManager ) -> None: @@ -125,6 +162,15 @@ class TestBudgetBlocksPerLevel: _assert_budget_blocks(client, key, user=customer) @pytest.mark.covers("quota_management.budget.organization.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_org_budget_blocks_keys_under_it(self, client: BudgetClient, resources: ResourceManager) -> None: org_id = client.create_org(max_budget=TINY_CAP, alias=f"e2e-budget-org-{unique_marker()}") resources.defer(lambda: client.delete_org(org_id)) @@ -139,6 +185,15 @@ class TestBudgetBlocksPerLevel: ) @pytest.mark.covers("quota_management.budget.team_member.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_member_budget_blocks_without_touching_teammates( self, client: BudgetClient, resources: ResourceManager ) -> None: @@ -166,6 +221,15 @@ class TestKeyBudgetBlocksAcrossKeyKinds: the capped key is refused, proving nothing around the key was the blocker.""" @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_personal_key_blocks_over_its_own_budget( self, client: BudgetClient, resources: ResourceManager ) -> None: @@ -180,6 +244,15 @@ class TestKeyBudgetBlocksAcrossKeyKinds: require_successful_call(_chat(client, control_key)) @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_team_key_blocks_over_its_own_budget(self, client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team(alias=f"e2e-key-cap-team-{unique_marker()}", max_budget=ROOMY_CAP) resources.defer(lambda: client.delete_team(team_id)) @@ -192,6 +265,15 @@ class TestKeyBudgetBlocksAcrossKeyKinds: require_successful_call(_chat(client, control_key)) @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_team_member_key_blocks_over_its_own_budget( self, client: BudgetClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_budget_fallback_e2e.py b/tests/e2e/quota_management/budgets/test_budget_fallback_e2e.py index fe6db8f0454..df1ac70944b 100644 --- a/tests/e2e/quota_management/budgets/test_budget_fallback_e2e.py +++ b/tests/e2e/quota_management/budgets/test_budget_fallback_e2e.py @@ -10,6 +10,7 @@ import pytest from budget_client import BudgetClient, model_budget from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import AnthropicMessagesResponse @@ -20,6 +21,15 @@ FALLBACK_MODEL = "gpt-5.5" @pytest.mark.covers("quota_management.budget.fallback.routes_to_fallback") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.MESSAGES, + provider=Provider.ANTHROPIC, + model=PRIMARY_MODEL, + mode=Mode.NONSTREAM, + ) +) def test_budget_fallback_reroutes_anthropic_messages_to_openai( client: BudgetClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_budget_reset_advances_e2e.py b/tests/e2e/quota_management/budgets/test_budget_reset_advances_e2e.py index fdd868b6bac..b5627cfed82 100644 --- a/tests/e2e/quota_management/budgets/test_budget_reset_advances_e2e.py +++ b/tests/e2e/quota_management/budgets/test_budget_reset_advances_e2e.py @@ -22,6 +22,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import BudgetWindow @@ -70,6 +71,12 @@ def _drive_to_block(client: BudgetClient, key: str) -> None: # ---- Rung 1: scheduling exists at creation ----------------------------------- +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.KEY_MANAGEMENT, + ) +) def test_key_with_budget_duration_schedules_reset_at_creation(client: BudgetClient, resources: ResourceManager) -> None: """Baseline: a key created with a budget_duration has budget_reset_at populated immediately. The reset job can only advance a timestamp that was scheduled in @@ -86,6 +93,15 @@ def test_key_with_budget_duration_schedules_reset_at_creation(client: BudgetClie @pytest.mark.covers("quota_management.budget.key.blocks_over_limit") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_key_spend_blocks_at_cap(client: BudgetClient, resources: ResourceManager) -> None: """Sanity that the tiny cap is enforced before we test that it resets: spend accrues across calls and eventually returns budget_exceeded, never a 5xx.""" @@ -103,6 +119,15 @@ def test_key_spend_blocks_at_cap(client: BudgetClient, resources: ResourceManage @pytest.mark.covers("quota_management.budget.key.resets_after_window") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_key_budget_reset_at_advances_after_window(client: BudgetClient, resources: ResourceManager) -> None: """The core #25109 guard: after the window elapses the reset job must move budget_reset_at strictly forward AND zero key.spend. The broken nullable-JSON @@ -139,6 +164,15 @@ def test_key_budget_reset_at_advances_after_window(client: BudgetClient, resourc @pytest.mark.covers("quota_management.budget.key_multi_window.resets_windows_independently") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_multi_window_key_resets_each_window_independently(client: BudgetClient, resources: ResourceManager) -> None: """The JSON-backed path #25109 specifically touched. A tight 30s window and a roomy 1m window: the tight window must reset on its own boundary while the roomy @@ -183,6 +217,15 @@ def test_multi_window_key_resets_each_window_independently(client: BudgetClient, @pytest.mark.covers("quota_management.budget.team_member.resets_after_window") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_team_member_budget_reset_at_advances(client: BudgetClient, resources: ResourceManager) -> None: """Per-team member windows are also JSON-backed. member_budget_reset_at must advance after the window; the explicit before None: """The other #25109 failure mode: a reset job that ERRORS on the nullable-JSON column surfaces to the caller as a non-budget 5xx. Across the whole reset wait diff --git a/tests/e2e/quota_management/budgets/test_budget_reset_e2e.py b/tests/e2e/quota_management/budgets/test_budget_reset_e2e.py index b7b7f269c47..fa45a2a1c43 100644 --- a/tests/e2e/quota_management/budgets/test_budget_reset_e2e.py +++ b/tests/e2e/quota_management/budgets/test_budget_reset_e2e.py @@ -7,6 +7,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -49,6 +50,15 @@ def _poll_until_serves_again(client: BudgetClient, key: str) -> None: class TestBudgetResetPerLevel: @pytest.mark.covers("quota_management.budget.key.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_bare_key_budget_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None: key = client.generate_key(max_budget=TINY_CAP, budget_duration=WINDOW) resources.defer(lambda: client.delete_key(key)) @@ -57,6 +67,15 @@ class TestBudgetResetPerLevel: _poll_until_serves_again(client, key) @pytest.mark.covers("quota_management.budget.team.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_team_budget_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team( alias=f"e2e-team-reset-{unique_marker()}", max_budget=TINY_CAP, budget_duration=WINDOW @@ -69,6 +88,15 @@ class TestBudgetResetPerLevel: _poll_until_serves_again(client, key) @pytest.mark.covers("quota_management.budget.organization.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_org_budget_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None: org_id = client.create_org( max_budget=TINY_CAP, alias=f"e2e-org-reset-{unique_marker()}", budget_duration=WINDOW @@ -91,6 +119,15 @@ class TestBudgetResetPerLevel: _poll_until_serves_again(client, key) @pytest.mark.covers("quota_management.budget.internal_user.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_personal_key_user_budget_resets_after_window( self, client: BudgetClient, resources: ResourceManager ) -> None: @@ -109,6 +146,15 @@ class TestKeyBudgetResetAcrossKeyKinds: the only thing that can block and the only thing that has to reset.""" @pytest.mark.covers("quota_management.budget.key.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_personal_key_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None: user_id = client.create_user(max_budget=ROOMY_CAP) resources.defer(lambda: client.delete_user(user_id)) @@ -119,6 +165,15 @@ class TestKeyBudgetResetAcrossKeyKinds: _poll_until_serves_again(client, key) @pytest.mark.covers("quota_management.budget.key.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_team_key_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team(alias=f"e2e-key-reset-team-{unique_marker()}", max_budget=ROOMY_CAP) resources.defer(lambda: client.delete_team(team_id)) @@ -129,6 +184,15 @@ class TestKeyBudgetResetAcrossKeyKinds: _poll_until_serves_again(client, key) @pytest.mark.covers("quota_management.budget.key.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) + ) def test_team_member_key_resets_after_window(self, client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team(alias=f"e2e-key-reset-team-{unique_marker()}", max_budget=ROOMY_CAP) resources.defer(lambda: client.delete_team(team_id)) diff --git a/tests/e2e/quota_management/budgets/test_model_access_group_budget_e2e.py b/tests/e2e/quota_management/budgets/test_model_access_group_budget_e2e.py index 9c927a31216..3e86b9b5e01 100644 --- a/tests/e2e/quota_management/budgets/test_model_access_group_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_model_access_group_budget_e2e.py @@ -21,6 +21,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody, LiteLLMParamsBody, ModelInfoBody, ModelNewBody @@ -102,6 +103,15 @@ def drained(client: BudgetClient) -> Iterator[DrainedPool]: class TestModelAccessGroupBudget: @pytest.mark.covers("quota_management.budget.model_access_group.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_the_key_that_drained_the_pool_stays_blocked( self, client: BudgetClient, drained: DrainedPool ) -> None: @@ -114,6 +124,15 @@ class TestModelAccessGroupBudget: ) @pytest.mark.covers("quota_management.budget.model_access_group.enforced_across_keys") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_a_key_that_spent_nothing_is_blocked_by_the_shared_pool( self, client: BudgetClient, resources: ResourceManager, drained: DrainedPool ) -> None: @@ -127,6 +146,15 @@ class TestModelAccessGroupBudget: ) @pytest.mark.covers("quota_management.budget.model_access_group.isolates_per_group") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_a_drained_group_does_not_block_a_different_group( self, client: BudgetClient, resources: ResourceManager, drained: DrainedPool ) -> None: @@ -141,6 +169,14 @@ class TestModelAccessGroupBudget: require_successful_call(result) @pytest.mark.covers("quota_management.budget.model_access_group.reports_spend") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BUDGET_MANAGEMENT, + provider=Provider.OPENAI, + model=BACKEND, + ) + ) def test_the_budget_read_reports_the_spend_drawn_against_the_pool( self, client: BudgetClient, drained: DrainedPool ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py b/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py index 87ff9d56ab2..e180b7ec725 100644 --- a/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_model_max_budget_e2e.py @@ -13,6 +13,7 @@ import pytest from budget_client import BudgetClient, is_budget_block, model_budget from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ModelBudgetEntry @@ -30,6 +31,15 @@ def _call(client: BudgetClient, key: str, model: str): @pytest.mark.covers("quota_management.budget.model_max.isolates_per_model") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=CAPPED_MODEL, + mode=Mode.NONSTREAM, + ) +) def test_model_max_budget_isolates_per_model( client: BudgetClient, resources: ResourceManager ) -> None: @@ -61,6 +71,15 @@ def test_model_max_budget_isolates_per_model( @pytest.mark.skip(reason="stage red: product gap, end-user model_max_budget rpm_limit is stored but never enforced") @pytest.mark.covers("quota_management.budget.end_user_model_max.blocks_over_limit") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model=FREE_MODEL, + mode=Mode.NONSTREAM, + ) +) def test_end_user_model_max_budget_enforces_per_model_rpm( client: BudgetClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py b/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py index e04f857545d..dd4b1458d62 100644 --- a/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_multi_window_budget_e2e.py @@ -22,6 +22,7 @@ import pytest from budget_client import BudgetClient, is_budget_block, window_reset_at from e2e_http import StreamingResponse, require_successful_call from e2e_config import CHEAP_OPENAI_MODEL, unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import BudgetWindow @@ -57,6 +58,15 @@ def _drive_to_block(client: BudgetClient, key: str) -> StreamingResponse: @pytest.mark.covers("quota_management.budget.key_multi_window.blocks_then_resets") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=CHEAP_OPENAI_MODEL, + mode=Mode.NONSTREAM, + ) +) def test_short_window_blocks_then_resets(client: BudgetClient, resources: ResourceManager) -> None: key = client.generate_key( models=[MODEL], @@ -90,6 +100,15 @@ def test_short_window_blocks_then_resets(client: BudgetClient, resources: Resour @pytest.mark.covers("quota_management.budget.key_multi_window.blocks_then_resets") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=CHEAP_OPENAI_MODEL, + mode=Mode.NONSTREAM, + ) +) def test_long_window_blocks_after_short_window_resets(client: BudgetClient, resources: ResourceManager) -> None: key = client.generate_key( models=[MODEL], diff --git a/tests/e2e/quota_management/budgets/test_soft_budget_e2e.py b/tests/e2e/quota_management/budgets/test_soft_budget_e2e.py index 2006efb5a57..9fba919c893 100644 --- a/tests/e2e/quota_management/budgets/test_soft_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_soft_budget_e2e.py @@ -12,12 +12,22 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @pytest.mark.covers("quota_management.budget.soft.alerts_without_blocking") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_soft_budget_does_not_block( client: BudgetClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_spend_counter_reseed_e2e.py b/tests/e2e/quota_management/budgets/test_spend_counter_reseed_e2e.py index 4a69135cdd1..a4d6e5b81b9 100644 --- a/tests/e2e/quota_management/budgets/test_spend_counter_reseed_e2e.py +++ b/tests/e2e/quota_management/budgets/test_spend_counter_reseed_e2e.py @@ -28,6 +28,7 @@ from pydantic import TypeAdapter, ValidationError from budget_client import BudgetClient from e2e_config import unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager if TYPE_CHECKING: @@ -144,6 +145,15 @@ def _accumulate(client: BudgetClient, key: str, count: int) -> None: @pytest.mark.covers("quota_management.budget.spend_counter.reseed_matches_db") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=MODEL, + mode=Mode.NONSTREAM, + ) +) def test_cold_counter_reseed_keeps_counter_equal_to_db_spend( client: BudgetClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_tag_budget_e2e.py b/tests/e2e/quota_management/budgets/test_tag_budget_e2e.py index b0068c66630..d7c61a7f4dd 100644 --- a/tests/e2e/quota_management/budgets/test_tag_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_tag_budget_e2e.py @@ -13,6 +13,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -34,6 +35,15 @@ def _tagged_call(client: BudgetClient, key: str, tag: str): @pytest.mark.covers("quota_management.budget.tag.blocks_over_limit") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_tag_budget_blocks_tagged_requests( client: BudgetClient, scoped_key: str, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/budgets/test_team_member_budget_e2e.py b/tests/e2e/quota_management/budgets/test_team_member_budget_e2e.py index 0fd0a545660..f496103096e 100644 --- a/tests/e2e/quota_management/budgets/test_team_member_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_team_member_budget_e2e.py @@ -21,6 +21,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import Success, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage @@ -79,6 +80,15 @@ def _send(client: BudgetClient, key: str) -> str | None: class TestTeamMemberBudget: + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_member_spend_attributed_to_team_and_user(self, client: BudgetClient, member: _Member) -> None: sent = frozenset(rid for rid in (_send(client, member.key) for _ in range(BURST)) if rid) assert sent, "no member call went through; cannot check attribution" @@ -98,6 +108,15 @@ class TestTeamMemberBudget: ) @pytest.mark.covers("quota_management.budget.team_member.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_member_spend_over_budget_is_blocked(self, client: BudgetClient, member: _Member) -> None: for _ in range(40): result = client.chat(member.key, MODEL, f"spend {unique_marker()}", max_tokens=16) diff --git a/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py b/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py index f03518f8a17..915fefefed4 100644 --- a/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py +++ b/tests/e2e/quota_management/budgets/test_team_member_budget_isolation_e2e.py @@ -17,6 +17,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import Success, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage @@ -88,6 +89,15 @@ def _roomy_send(client: BudgetClient, key: str) -> str: class TestTeamMemberBudgetIsolation: @pytest.mark.covers("quota_management.budget.team_member.isolates_per_member") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_blocked_member_does_not_block_peer(self, client: BudgetClient, pair: _Pair) -> None: blocked = False for _ in range(40): diff --git a/tests/e2e/quota_management/budgets/test_team_member_budget_reset_e2e.py b/tests/e2e/quota_management/budgets/test_team_member_budget_reset_e2e.py index 5d097a81f92..472dc2f28de 100644 --- a/tests/e2e/quota_management/budgets/test_team_member_budget_reset_e2e.py +++ b/tests/e2e/quota_management/budgets/test_team_member_budget_reset_e2e.py @@ -6,6 +6,7 @@ import pytest from budget_client import BudgetClient from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -17,6 +18,15 @@ def _as_datetime(value: str) -> datetime: @pytest.mark.covers("quota_management.budget.team_member.resets_after_window") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_team_member_budget_reset_keeps_advancing(client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team(alias=f"e2e-member-reset-{unique_marker()}", max_budget=100.0) resources.defer(lambda: client.delete_team(team_id)) diff --git a/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py b/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py index 7683132776b..6e1565b1af0 100644 --- a/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py +++ b/tests/e2e/quota_management/budgets/test_team_multi_window_budget_e2e.py @@ -24,6 +24,7 @@ import pytest from budget_client import BudgetClient, is_budget_block, window_reset_at from e2e_http import StreamingResponse, require_successful_call from e2e_config import unique_marker +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import BudgetWindow @@ -52,6 +53,15 @@ def _drive_to_block(client: BudgetClient, key: str) -> StreamingResponse: @pytest.mark.covers("quota_management.budget.team_multi_window.blocks_then_resets") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: ResourceManager) -> None: team_id = client.create_team( alias=f"e2e-team-window-{unique_marker()}", @@ -85,6 +95,15 @@ def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: R @pytest.mark.covers("quota_management.budget.team_multi_window.blocks_then_resets") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model="claude-haiku-4-5", + mode=Mode.NONSTREAM, + ) +) def test_team_long_window_blocks_after_short_window_resets(client: BudgetClient, resources: ResourceManager) -> None: # 0. key with a short budget window and a long budget window diff --git a/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py b/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py index 4dc7a2df647..88562bedd0a 100644 --- a/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py +++ b/tests/e2e/quota_management/budgets/test_user_budget_across_keys_e2e.py @@ -15,6 +15,7 @@ import pytest from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager pytestmark = pytest.mark.e2e @@ -58,6 +59,15 @@ def _expect_prompt_block(client: BudgetClient, key: str, subject: str) -> None: class TestUserBudgetAcrossKeys: @pytest.mark.covers("quota_management.budget.internal_user.enforced_across_keys") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_user_budget_blocks_a_second_key(self, client: BudgetClient, resources: ResourceManager) -> None: user_id = client.create_user(max_budget=TINY_CAP) resources.defer(lambda: client.delete_user(user_id)) diff --git a/tests/e2e/quota_management/ratelimit/quota_client.py b/tests/e2e/quota_management/ratelimit/quota_client.py index a3a467a1d71..4d0ccfc52e0 100644 --- a/tests/e2e/quota_management/ratelimit/quota_client.py +++ b/tests/e2e/quota_management/ratelimit/quota_client.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from proxy_client import ProxyClient from e2e_http import StreamingResponse +from e2e_metadata import step from models import ChatBody, ChatMessage @@ -16,6 +17,7 @@ from models import ChatBody, ChatMessage class QuotaClient: proxy: ProxyClient + @step("POST /chat/completions") def chat(self, key: str, model: str, content: str, *, max_tokens: int = 16) -> StreamingResponse: return self.proxy.transport.send( "/chat/completions", diff --git a/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py b/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py index a7d548381c1..3b6dabea178 100644 --- a/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_dynamic_rate_limit_priority_e2e.py @@ -46,6 +46,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody, KeyMetadata, LiteLLMParamsBody from quota_client import QuotaClient @@ -157,6 +158,15 @@ class TestDynamicRateLimitPriority: "quota_management.ratelimit.priority_generous.picks_under_tpm", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_generous_mode_lets_priority_borrow_past_reservation( self, client: QuotaClient, resources: ResourceManager ) -> None: @@ -199,6 +209,15 @@ class TestDynamicRateLimitPriority: "quota_management.ratelimit.priority_strict.picks_under_tpm", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_strict_mode_blocks_saturated_priority_but_serves_the_other( self, client: QuotaClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/ratelimit/test_rate_limit_e2e.py b/tests/e2e/quota_management/ratelimit/test_rate_limit_e2e.py index 7d87686b06c..3e3f08884c8 100644 --- a/tests/e2e/quota_management/ratelimit/test_rate_limit_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_rate_limit_e2e.py @@ -39,6 +39,7 @@ from pydantic import BaseModel, ConfigDict, ValidationError from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker from e2e_http import StreamingResponse, require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody from quota_client import QuotaClient @@ -176,6 +177,15 @@ def _assert_rate_limited(outcome: StreamingResponse, limit_type: str) -> None: class TestKeyRateLimits: @pytest.mark.covers("quota_management.ratelimit.rpm.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=CHEAP_ANTHROPIC_MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_rpm_limit_blocks_over_limit(self, client: QuotaClient, resources: ResourceManager) -> None: key = _limited_key(client, resources, rpm_limit=3) info = client.proxy.key_info(key) @@ -188,6 +198,15 @@ class TestKeyRateLimits: _assert_rate_limited(_chat(client, key), "requests") @pytest.mark.covers("quota_management.ratelimit.tpm.blocks_over_limit") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=CHEAP_ANTHROPIC_MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_tpm_limit_blocks_over_limit(self, client: QuotaClient, resources: ResourceManager) -> None: key = _limited_key(client, resources, tpm_limit=TPM_LIMIT) info = client.proxy.key_info(key) @@ -207,6 +226,15 @@ class TestKeyRateLimits: _assert_rate_limited(_chat(client, key), "tokens") @pytest.mark.covers("quota_management.ratelimit.rpm.resets_after_window") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=CHEAP_ANTHROPIC_MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_rpm_limit_resets_after_window(self, client: QuotaClient, resources: ResourceManager) -> None: key = _limited_key(client, resources, rpm_limit=1) @@ -232,6 +260,15 @@ class TestKeyRateLimits: pytest.fail("a blocked key never recovered after the rate-limit window elapsed") @pytest.mark.covers("quota_management.ratelimit.rpm.headers_report_remaining") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=CHEAP_ANTHROPIC_MODEL, + mode=Mode.NONSTREAM, + ) + ) def test_headers_report_limit_and_remaining(self, client: QuotaClient, resources: ResourceManager) -> None: key = _limited_key(client, resources, rpm_limit=5, tpm_limit=100000) diff --git a/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py b/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py index a88f0ca546a..7f39389fc5f 100644 --- a/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_redis_backed_ratelimit_e2e.py @@ -13,6 +13,7 @@ import pytest from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody, LiteLLMParamsBody from quota_client import QuotaClient @@ -40,6 +41,15 @@ class TestRedisBackedRateLimit: "quota_management.ratelimit.redis_backed.blocks_over_limit", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_rpm_limit_one_blocks_second_call( self, client: QuotaClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py b/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py index 3e1bc662470..49e04ef1c4d 100644 --- a/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_redis_circuit_breaker_e2e.py @@ -15,6 +15,7 @@ import pytest from e2e_config import unique_marker from e2e_http import require_successful_call +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import KeyGenerateBody, LiteLLMParamsBody from quota_client import QuotaClient @@ -45,6 +46,15 @@ class TestRedisCircuitBreakerPath: "reliability.circuit_breaker.redis.trips_then_recovers", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_burst_rate_limit_does_not_freeze_fresh_key( self, client: QuotaClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py b/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py index 33d869ee80e..1abbcc78a1d 100644 --- a/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py +++ b/tests/e2e/quota_management/ratelimit/test_tpm_excludes_cached_tokens_e2e.py @@ -24,6 +24,7 @@ from models import ( TextBlock, Usage, ) +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from quota_client import QuotaClient pytestmark = [pytest.mark.e2e, pytest.mark.provider_live] @@ -101,6 +102,16 @@ class TestTpmExcludesCachedTokens: "quota_management.ratelimit.tpm.excludes_cached_tokens", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.ANTHROPIC, + model=ANTHROPIC_MODEL, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_cache_hit_reduces_tpm_by_non_cached_only( self, client: QuotaClient, resources: ResourceManager ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/cost_rows.py b/tests/e2e/quota_management/spend_tracking/cost_rows.py index 87af54fe83f..3bf3f6a90e7 100644 --- a/tests/e2e/quota_management/spend_tracking/cost_rows.py +++ b/tests/e2e/quota_management/spend_tracking/cost_rows.py @@ -36,6 +36,7 @@ from pydantic import BaseModel, RootModel from e2e_config import unique_marker from e2e_http import Success +from e2e_metadata import step from lifecycle import ResourceManager from models import LiteLLMParamsBody, SpendLogsParams from proxy_client import ProxyClient @@ -98,6 +99,7 @@ def approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) +@step("assert the row's total is the sum of its components") def assert_total_is_sum_of_components(row: CostRow) -> None: """The row's total is input + output + tool usage. The cache components are already inside the gross input cost, so adding them again would double-bill.""" @@ -114,6 +116,7 @@ def assert_total_is_sum_of_components(row: CostRow) -> None: ) +@step("assert fresh input tokens are billed at the model's rate") def assert_fresh_tokens_billed_at(row: CostRow, input_rate: float) -> None: """Strip the cache components out of the gross input cost and what is left must be the freshly-read tokens at the deployment's input rate.""" @@ -133,6 +136,7 @@ def assert_fresh_tokens_billed_at(row: CostRow, input_rate: float) -> None: ) +@step("poll /spend/logs for the request's cost row") def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None: """Poll /spend/logs for the call's row until it lands with a cost breakdown (rows flush ~60s behind the call via proxy_batch_write_at); None on timeout.""" @@ -156,6 +160,7 @@ def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None: return None +@step("poll /spend/logs for a matching cost row") def poll_cost_row_where( proxy: ProxyClient, api_key: str, predicate: Callable[[CostRow], bool] ) -> CostRow | None: @@ -182,6 +187,7 @@ def poll_cost_row_where( return None +@step("register a deployment with custom pricing") def register_priced_model( proxy: ProxyClient, resources: ResourceManager, diff --git a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py index b7f59fe5f89..b8406217a9e 100644 --- a/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py +++ b/tests/e2e/quota_management/spend_tracking/spend_e2e_client.py @@ -29,6 +29,7 @@ from e2e_http import ( is_ok, unwrap, ) +from e2e_metadata import step from models import ( AnthropicMessagesBody, ChatBody, @@ -234,6 +235,7 @@ def _chat_body( class SpendClient: proxy: ProxyClient + @step("POST /chat/completions") def chat( self, key: str, @@ -250,6 +252,7 @@ class SpendClient: _chat_body(model, content, max_tokens=max_tokens, tags=tags, user=user, cache=cache), ) + @step("POST /chat/completions (streaming)") def chat_stream( self, key: str, model: str, content: str, *, max_tokens: int | None = None ) -> StreamingResponse: @@ -257,6 +260,7 @@ class SpendClient: key, _chat_body(model, content, max_tokens=max_tokens, stream=True) ) + @step("POST /v1/messages (streaming)") def messages_stream( self, key: str, model: str, content: str, *, max_tokens: int ) -> StreamingResponse: @@ -270,9 +274,11 @@ class SpendClient: ), ) + @step("POST /embeddings") def embed(self, key: str, model: str, content: str) -> Result[EmbedResponse]: return self.proxy.embed(key, EmbedBody(model=model, input=content)) + @step("poll /spend/logs for the key") def poll_logs_for_key( self, key: str, @@ -284,6 +290,7 @@ class SpendClient: key, min_rows=min_rows, predicate=predicate ) + @step("POST /spend/calculate") def calculate_spend(self, model: str, content: str) -> float: return unwrap( self.proxy.transport.post( @@ -296,6 +303,7 @@ class SpendClient: ) ).cost + @step("GET /spend/tags") def spend_by_tags(self) -> list[TagSpend]: result = self.proxy.transport.get( "/spend/tags", @@ -309,6 +317,7 @@ class SpendClient: case _: return [] + @step("poll /spend/tags for the tag") def poll_tag_spend(self, tag: str, *, minimum: float = 0.0) -> TagSpend | None: """Poll /spend/tags until the tag's aggregate reaches `minimum`; last seen.""" deadline = time.monotonic() + self.proxy.poll_timeout @@ -324,6 +333,7 @@ class SpendClient: time.sleep(self.proxy.poll_interval) return entry + @step("poll /key/info for recorded spend") def poll_key_spend(self, key: str, *, minimum: float = 0.0) -> float: deadline = time.monotonic() + self.proxy.poll_timeout spend = 0.0 @@ -334,6 +344,7 @@ class SpendClient: time.sleep(self.proxy.poll_interval) return spend + @step("GET /spend/logs") def spend_logs_page( self, *, api_key: str | None, page: int, page_size: int ) -> SpendLogsPage: @@ -356,6 +367,7 @@ class SpendClient: ) ) + @step("probe the spend route") def probe(self, path: str, *, params: DateRangeParams) -> ProbeResult: return self.proxy.transport.probe(path, params=params) @@ -370,6 +382,7 @@ class SpendClient: ) return outcome.result if isinstance(outcome, Converged) else outcome.last_result + @step("create internal user") def create_user(self, *, email: str, role: UserRole, user_id: str) -> str: return unwrap( self.proxy.transport.post( @@ -390,6 +403,7 @@ class SpendClient: ) ) + @step("generate virtual key") def generate_key_record(self, body: KeyGenerateBody) -> KeyGenerateResponse: return unwrap( self.proxy.transport.post( @@ -400,6 +414,7 @@ class SpendClient: ) ) + @step("POST /chat/completions") def send_chat(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( "/chat/completions", @@ -407,6 +422,7 @@ class SpendClient: json=_chat_body(model, content, max_tokens=max_tokens), ) + @step("POST /chat/completions (queued)") def send_queued_chat(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( "/queue/chat/completions", @@ -418,6 +434,7 @@ class SpendClient: ), ) + @step("POST /v1/messages") def send_messages(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( "/v1/messages", @@ -429,6 +446,7 @@ class SpendClient: ), ) + @step("POST /v1/responses") def send_responses(self, key: str, model: str, content: str) -> StreamingResponse: return self.proxy.transport.send( "/v1/responses", @@ -436,6 +454,7 @@ class SpendClient: json=ResponsesBody(model=model, input=content), ) + @step("POST /embeddings") def send_embed(self, key: str, model: str, content: str) -> StreamingResponse: return self.proxy.transport.send( "/embeddings", @@ -443,6 +462,7 @@ class SpendClient: json=EmbedBody(model=model, input=content), ) + @step("POST the gemini passthrough generateContent") def send_gemini_generate(self, key: str, model: str, content: str, *, max_tokens: int) -> StreamingResponse: return self.proxy.transport.send( f"/gemini/v1beta/models/{model}:generateContent", @@ -453,6 +473,7 @@ class SpendClient: ), ) + @step("POST /v1/files") def upload_batch_file(self, key: str, model: str, content: bytes) -> FileObject: return unwrap( self.proxy.transport.upload( @@ -466,6 +487,7 @@ class SpendClient: ) ) + @step("POST /v1/batches") def create_batch(self, key: str, body: BatchCreateBody) -> BatchObject: return unwrap( self.proxy.transport.post( @@ -476,6 +498,7 @@ class SpendClient: ) ) + @step("GET /v1/batches/{id}") def retrieve_batch(self, key: str, batch_id: str, *, provider: str) -> BatchObject: return unwrap( self.proxy.transport.get( @@ -486,6 +509,7 @@ class SpendClient: ) ) + @step("replay a provider callback log") def replay_callback_log(self, key: str, payload: CallbackLogPayload) -> CallbackLogsResponse: return unwrap( self.proxy.transport.post( @@ -496,9 +520,11 @@ class SpendClient: ) ) + @step("GET /health for the deployment") def health(self, model: str) -> ProbeResult: return self.proxy.transport.probe("/health", params=HealthParams(model=model)) + @step("GET /user/daily/activity") def daily_activity_for_key(self, token: str, *, start: datetime, end: datetime) -> DailyActivityKeyBreakdown | None: response: Final = unwrap( self.proxy.transport.get( @@ -517,6 +543,7 @@ class SpendClient: None, ) + @step("poll /user/daily/activity for the key") def poll_daily_activity_for_key( self, token: str, *, start: datetime, end: datetime, min_requests: int ) -> DailyActivityKeyBreakdown | None: @@ -530,6 +557,7 @@ class SpendClient: ) return outcome.result if isinstance(outcome, Converged) else outcome.last_result + @step("GET the proxy's OpenAPI schema") def openapi(self) -> OpenAPISchema: return unwrap( self.proxy.transport.get( diff --git a/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py index c50ec3d902f..a7348aa3dd3 100644 --- a/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py @@ -52,6 +52,7 @@ from cost_rows import ( ) from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import AnthropicMessagesBody, ChatBody, ChatMessage, LiteLLMParamsBody from pydantic import BaseModel @@ -122,6 +123,16 @@ def _assert_cache_read_billed(row: CostRow) -> None: class TestCacheCostAccounting: @pytest.mark.covers("quota_management.spend_tracking.cache_write.bills_cache_creation_rate") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=CACHE_WRITE_BACKEND, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_cache_write_tokens_billed_at_cache_creation_rate( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -152,6 +163,16 @@ class TestCacheCostAccounting: assert_total_is_sum_of_components(row) @pytest.mark.covers("quota_management.spend_tracking.cost_breakdown.reports_component_costs") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=CACHE_WRITE_BACKEND, + capabilities=(Capability.PROMPT_CACHING, Capability.REASONING), + mode=Mode.NONSTREAM, + ) + ) def test_cost_breakdown_reports_component_costs( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -216,6 +237,16 @@ class TestCacheCostAccounting: _assert_cache_read_billed(row) @pytest.mark.covers("quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=CACHE_READ_BACKEND, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.STREAM, + ) + ) def test_streaming_cache_read_billed_at_cache_read_rate( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -247,6 +278,16 @@ class TestCacheCostAccounting: _assert_cache_read_billed(row) @pytest.mark.covers("quota_management.spend_tracking.messages_bridge.keeps_cache_tokens") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.MESSAGES, + provider=Provider.OPENAI, + model=BRIDGE_BACKEND, + capabilities=(Capability.PROMPT_CACHING,), + mode=Mode.NONSTREAM, + ) + ) def test_messages_bridge_keeps_cache_tokens( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py index abc321ccde8..19f55de7b62 100644 --- a/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py @@ -27,6 +27,7 @@ import pytest from cost_rows import approx_equal, cacheable_prefix, register_priced_model from e2e_config import unique_marker from e2e_http import StreamingResponse +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody from spend_e2e_client import SpendClient @@ -60,6 +61,15 @@ def _header_cost(response: StreamingResponse, name: str) -> float: class TestCostHeaders: @pytest.mark.covers("quota_management.spend_tracking.cost_headers.additive_components") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=BACKEND, + mode=Mode.NONSTREAM, + ) + ) def test_component_cost_headers_sum_to_total( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py index 4a2c23927c6..3afa3d281ef 100644 --- a/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_key_attribution_e2e.py @@ -36,6 +36,7 @@ from datetime import datetime, timedelta, timezone from typing import Final import pytest +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from models import KeyGenerateBody from proxy_client import Converged, await_converged from pydantic import BaseModel @@ -281,6 +282,12 @@ class TestKeyAttribution: "rust_control_plane", ], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) + ) def test_every_write_path_row_joins_the_key(self, client: SpendClient, driven: DrivenKey) -> None: assert tuple(path.name for path in driven.paths) == WRITE_PATHS found: Final = tuple((path, client.proxy.poll_logs_for_request_id(path.request_id)) for path in driven.paths) @@ -317,6 +324,12 @@ class TestKeyAttribution: "rust_control_plane", ], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) + ) def test_spend_logs_by_key_return_every_row_with_the_alias(self, client: SpendClient, driven: DrivenKey) -> None: expected_ids: Final = frozenset(path.request_id for path in driven.paths) rows: Final = client.poll_logs_for_key( @@ -345,6 +358,12 @@ class TestKeyAttribution: "rust_control_plane", ], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) + ) def test_user_daily_activity_reports_alias_and_email(self, client: SpendClient, driven: DrivenKey) -> None: breakdown: Final[DailyActivityKeyBreakdown | None] = client.poll_daily_activity_for_key( driven.identity.token, @@ -367,6 +386,12 @@ class TestKeyAttribution: "quota_management.spend_tracking.key_attribution.health_rows_keep_service_account", exercised_on=["chat_completions"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.HEALTH, + ) + ) def test_health_check_rows_keep_the_service_account_key(self, client: SpendClient) -> None: started_at: Final = datetime.now(timezone.utc) probe: Final = client.health(CHAT_MODEL) @@ -380,6 +405,15 @@ class TestKeyAttribution: "quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key", exercised_on=["batches"], ) + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.BATCHES, + provider=Provider.OPENAI, + model=BATCH_MODEL, + mode=Mode.BATCH, + ) + ) def test_terminal_batch_cost_row_joins_the_retrieving_key(self, client: SpendClient, driven: DrivenKey) -> None: provider_batch_id: Final = _provider_batch_id(_driven_batch_id(driven)) fetched: Final = _await_terminal_batch(client, driven.identity.key, provider_batch_id) diff --git a/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py b/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py index 4931af4222d..9ae9cf01515 100644 --- a/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py @@ -14,6 +14,7 @@ write path are all still under test with zero provider calls. import pytest from e2e_config import CHEAP_OPENAI_MODEL, provider_edge_base +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody from spend_e2e_client import SpendClient, unique_marker, unwrap @@ -22,6 +23,15 @@ pytestmark = [pytest.mark.e2e, pytest.mark.replayable] @pytest.mark.covers("quota_management.spend_tracking.chat_completions.logs_cost") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=f"openai/{CHEAP_OPENAI_MODEL}", + mode=Mode.NONSTREAM, + ) +) def test_edge_wired_chat_writes_nonzero_spend_row( client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py index 770c5699b4e..9967dbccba6 100644 --- a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py @@ -26,6 +26,7 @@ from cost_rows import ( ) from e2e_config import unique_marker from e2e_http import unwrap +from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import ChatBody, ChatMessage, LiteLLMParamsBody from spend_e2e_client import SpendClient @@ -45,6 +46,16 @@ REASONING_EFFORT = "high" class TestServiceTierPricing: @pytest.mark.covers("quota_management.spend_tracking.service_tier.bills_tier_rates") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model=BACKEND, + capabilities=(Capability.REASONING,), + mode=Mode.NONSTREAM, + ) + ) def test_priority_tier_bills_priority_rates( self, client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_routes.py b/tests/e2e/quota_management/spend_tracking/test_spend_routes.py index 67fd88bc84d..5b57ea83b86 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_routes.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_routes.py @@ -22,6 +22,7 @@ from typing import Final import pytest from e2e_http import ProbeResult +from e2e_metadata import Domain, Route, Subject, meta from models import DateRangeParams from spend_e2e_client import SpendClient @@ -102,12 +103,24 @@ def _probe(client: SpendClient, route: str) -> ProbeResult: @pytest.mark.parametrize("route", SPEND_ROUTES) +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) +) def test_spend_route_responsive(client: SpendClient, route: str) -> None: result = _probe(client, route) print(f"{route} -> {result.status_code}\n{result.body[:600]}") assert result.healthy, f"{route} -> {result.status_code}\n{result.body[:600]}" +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) +) def test_schema_listed_spend_routes_are_responsive(client: SpendClient) -> None: """Probe any spend GET route the schema lists that isn't in SPEND_ROUTES.""" schema = client.openapi() diff --git a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py index 8a91e53e7d7..5c6e3b26212 100644 --- a/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_spend_tracking_e2e.py @@ -22,6 +22,7 @@ from typing import Final import pytest from e2e_http import Success +from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody, SpendLogs, SpendLogsParams from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap @@ -62,6 +63,15 @@ def _require_row( @pytest.mark.covers("quota_management.spend_tracking.chat_completions.logs_cost") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_chat_completion_writes_nonzero_spend_row( client: SpendClient, scoped_key: str ) -> None: @@ -97,6 +107,15 @@ def test_chat_completion_writes_nonzero_spend_row( @pytest.mark.covers("quota_management.spend_tracking.stream.logs_cost") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.STREAM, + ) +) def test_streaming_chat_completion_tracks_spend( client: SpendClient, scoped_key: str ) -> None: @@ -125,6 +144,15 @@ def test_streaming_chat_completion_tracks_spend( @pytest.mark.covers("quota_management.spend_tracking.messages_bridge.logs_cost") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.MESSAGES, + provider=Provider.OPENAI, + model="openai-responses-codex", + mode=Mode.STREAM, + ) +) def test_streaming_messages_via_responses_bridge_tracks_spend( client: SpendClient, scoped_key: str ) -> None: @@ -195,6 +223,15 @@ def test_streaming_messages_via_responses_bridge_tracks_spend( @pytest.mark.covers("quota_management.spend_tracking.embeddings.logs_cost") @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.cost_logged") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.EMBEDDINGS, + provider=Provider.OPENAI, + model="openai-text-embedding-3-small", + mode=Mode.NONSTREAM, + ) +) def test_embedding_writes_nonzero_spend_row( client: SpendClient, scoped_key: str ) -> None: @@ -218,6 +255,15 @@ def test_embedding_writes_nonzero_spend_row( @pytest.mark.covers("quota_management.spend_tracking.cache_hit.zero_cost") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_cache_hit_is_zero_cost_and_suffixed( client: SpendClient, scoped_key: str ) -> None: @@ -254,6 +300,15 @@ def test_cache_hit_is_zero_cost_and_suffixed( @pytest.mark.covers("quota_management.spend_tracking.key_rollup.matches_sum_of_logs") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_key_spend_equals_sum_of_logs(client: SpendClient, scoped_key: str) -> None: for _ in range(2): _ = unwrap( @@ -282,6 +337,15 @@ def test_key_spend_equals_sum_of_logs(client: SpendClient, scoped_key: str) -> N @pytest.mark.replayable @pytest.mark.covers("quota_management.spend_tracking.concurrent_burst.loses_no_spend") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model="openai/gpt-5.6-luna", + mode=Mode.NONSTREAM, + ) +) def test_burst_of_concurrent_calls_loses_no_spend( client: SpendClient, resources: ResourceManager ) -> None: @@ -299,6 +363,15 @@ def test_burst_of_concurrent_calls_loses_no_spend( @pytest.mark.covers("quota_management.spend_tracking.pagination.keeps_total") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_spend_logs_v2_pagination_caps_pages_and_keeps_total( client: SpendClient, scoped_key: str ) -> None: @@ -352,6 +425,15 @@ def test_spend_logs_v2_pagination_caps_pages_and_keeps_total( @pytest.mark.covers("quota_management.spend_tracking.tags.attributes_spend") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_request_tags_round_trip(client: SpendClient, scoped_key: str) -> None: tag = f"e2e-spend-{unique_marker()}" _ = unwrap( @@ -369,6 +451,15 @@ def test_request_tags_round_trip(client: SpendClient, scoped_key: str) -> None: @pytest.mark.covers("quota_management.spend_tracking.tags.attributes_spend") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_tag_spend_matches_sum_of_tagged_logs( client: SpendClient, scoped_key: str ) -> None: @@ -407,6 +498,15 @@ def test_tag_spend_matches_sum_of_tagged_logs( @pytest.mark.covers("quota_management.spend_tracking.end_user.attributes_spend") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_end_user_spend_attributed_on_row( client: SpendClient, scoped_key: str, resources: ResourceManager ) -> None: @@ -425,6 +525,15 @@ def test_end_user_spend_attributed_on_row( @pytest.mark.covers("quota_management.spend_tracking.per_model.writes_own_rows") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_each_model_on_a_shared_key_gets_its_own_row( client: SpendClient, scoped_key: str ) -> None: @@ -474,6 +583,15 @@ def test_each_model_on_a_shared_key_gets_its_own_row( @pytest.mark.covers("quota_management.spend_tracking.failure.writes_failure_row") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.OPENAI, + model="openai/gpt-5.5", + mode=Mode.NONSTREAM, + ) +) def test_failure_call_writes_failure_status_row( client: SpendClient, resources: ResourceManager, scoped_key: str ) -> None: @@ -499,6 +617,14 @@ def test_failure_call_writes_failure_status_row( @pytest.mark.covers("quota_management.spend_tracking.spend_calculate.returns_cost") +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + ) +) def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: cost = client.calculate_spend( "gemini-2.5-flash", "estimate the cost of this request" @@ -509,6 +635,15 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: ) +@meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + provider=Provider.GEMINI, + model="gemini-2.5-flash", + mode=Mode.NONSTREAM, + ) +) def test_spend_logs_endpoint_returns_spend( client: SpendClient, scoped_key: str ) -> None: diff --git a/tests/e2e/quota_management/spend_tracking/test_team_daily_activity_e2e.py b/tests/e2e/quota_management/spend_tracking/test_team_daily_activity_e2e.py index ef635e59743..c92e68d04de 100644 --- a/tests/e2e/quota_management/spend_tracking/test_team_daily_activity_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_team_daily_activity_e2e.py @@ -14,6 +14,7 @@ from typing import Final import pytest from e2e_http import ProbeResult +from e2e_metadata import Domain, Provider, Route, Subject, meta from lifecycle import ResourceManager from proxy_client import Converged, await_converged from pydantic import BaseModel @@ -82,6 +83,14 @@ def _probe(client: SpendClient, params: BaseModel) -> ProbeResult: class TestTeamDailyActivity: @pytest.mark.replayable @pytest.mark.covers("mgmt.team.daily_activity.happy_path") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + provider=Provider.OPENAI, + model="openai/gpt-5.6-luna", + ) + ) def test_valid_date_range_returns_results_and_metadata( self, client: SpendClient, resources: ResourceManager ) -> None: @@ -199,6 +208,12 @@ class TestTeamDailyActivity: assert empty.metadata.total_failed_requests == 0 @pytest.mark.covers("mgmt.team.daily_activity.missing_start_date_rejected") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) + ) def test_missing_start_date_is_rejected(self, client: SpendClient) -> None: end = datetime.now(timezone.utc).date().isoformat() result = _probe(client, TeamDailyActivityParams(end_date=end, page=1)) @@ -207,6 +222,12 @@ class TestTeamDailyActivity: ) @pytest.mark.covers("mgmt.team.daily_activity.missing_end_date_rejected") + @meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.SPEND_REPORTING, + ) + ) def test_missing_end_date_is_rejected(self, client: SpendClient) -> None: start = (datetime.now(timezone.utc).date() - timedelta(days=1)).isoformat() result = _probe(client, TeamDailyActivityParams(start_date=start, page=1)) diff --git a/tests/e2e/test_junit_properties.py b/tests/e2e/test_junit_properties.py index 02c1413c840..afcf4e76c41 100644 --- a/tests/e2e/test_junit_properties.py +++ b/tests/e2e/test_junit_properties.py @@ -10,12 +10,28 @@ rollups and, for ``source``, the status page's per-test links to GitHub. from __future__ import annotations +from dataclasses import fields from pathlib import Path import pytest +from e2e_metadata import ( + MAX_STEPS, + STEPS, + Capability, + Domain, + Mode, + Provider, + Route, + Subject, + meta, + step, + step_properties, + subject_properties, +) from junit_properties import ( SUITE_ROOT, attach_result_properties, + attach_step_properties, dedupe_covers, package_from_nodeid, result_properties, @@ -129,3 +145,263 @@ class TestSuiteRoot: class TestDedupeCovers: def test_ids_are_unique_order_preserving_and_non_empty_strings(self) -> None: assert dedupe_covers([("A", "B"), ("B", ""), ("C", 7)]) == ("A", "B", "C") + + +def fixed_prefix(covers: str, lineno: int) -> tuple[tuple[str, str], ...]: + """The three-tuple every testcase in this suite has carried since before the + typed marker existed. Spelled out rather than derived, so a change to its + shape or order fails a test instead of agreeing with itself.""" + return ( + ("package", "root"), + ("covers", covers), + ("source", f"tests/e2e/test_junit_properties.py:{lineno}"), + ) + + +class TestSubjectProperties: + """The declared half: `@meta(Subject(...))` -> `` pairs. + + Markers are applied at run time via `request.applymarker`, the idiom the + `covers` tests above already use, so the coverage registry's collect-only pass + never sees a marker that exists only to be serialized. + """ + + def test_every_declared_field_becomes_a_property_in_field_order(self, request: pytest.FixtureRequest) -> None: + """One pass over `dataclasses.asdict`: declaration order is emission order, + a (str, Enum) member is written as its `.value` and never `str(member)`, + the plural field emits a repeated SINGULAR `capability`, and an unset field + (`provider` here) emits nothing at all.""" + test = type(self).test_every_declared_field_becomes_a_property_in_field_order + request.applymarker( + meta( + Subject( + domain=Domain.SPEND_BUDGETS, + route=Route.CHAT_COMPLETIONS, + model="gpt-5.5", + capabilities=(Capability.VISION, Capability.FUNCTION_CALLING, Capability.VISION), + mode=Mode.NONSTREAM, + ) + ) + ) + assert subject_properties(collected_item(request, test.__name__)) == ( + ("domain", "spend-budgets"), + ("route", "chat_completions"), + ("model", "gpt-5.5"), + ("capability", "function_calling"), + ("capability", "vision"), + ("mode", "nonstream"), + ) + + def test_scalar_property_names_are_the_dataclass_field_names(self, request: pytest.FixtureRequest) -> None: + """The mapping is `asdict`, not a hand-written table: a scalar field added + to `Subject` later serializes under its own name with no edit to the + serializer. Proven by reading the field list back off the dataclass.""" + test = type(self).test_scalar_property_names_are_the_dataclass_field_names + request.applymarker(meta(Subject(domain=Domain.UNKNOWN, route=Route.HEALTH, model="m", mode=Mode.STREAM))) + declared = tuple(field.name for field in fields(Subject)) + emitted = tuple(name for name, _ in subject_properties(collected_item(request, test.__name__))) + assert emitted == tuple(name for name in declared if name in {"domain", "route", "model", "mode"}) + + def test_capabilities_are_deduped_and_sorted_at_declaration(self) -> None: + """Canonicalized in `__post_init__`, so two tests that spelled the same set + in different orders produce byte-identical properties and the committed run + files diff cleanly.""" + assert Subject(capabilities=(Capability.VISION, Capability.REASONING, Capability.VISION)).capabilities == ( + Capability.REASONING, + Capability.VISION, + ) + + def test_the_typed_marker_only_ever_appends_to_the_fixed_prefix(self, request: pytest.FixtureRequest) -> None: + """Loki, Grafana and the status page read `package`/`covers`/`source`; the + typed fields ride behind them and must not disturb them.""" + test = type(self).test_the_typed_marker_only_ever_appends_to_the_fixed_prefix + request.applymarker(pytest.mark.covers("quota_management.budget.key.blocks_over_limit")) + request.applymarker(meta(Subject(route=Route.SPEND_REPORTING))) + assert result_properties(collected_item(request, test.__name__)) == fixed_prefix( + "quota_management.budget.key.blocks_over_limit", test.__code__.co_firstlineno + ) + (("route", "spend_reporting"),) + + def test_a_test_with_only_the_old_string_covers_is_unchanged(self, request: pytest.FixtureRequest) -> None: + """The existing `@pytest.mark.covers("cell.id")` call sites keep emitting + exactly what they emitted before the typed marker existed.""" + test = type(self).test_a_test_with_only_the_old_string_covers_is_unchanged + request.applymarker(pytest.mark.covers("llm.responses.openai.tool_use.nonstream.works")) + assert result_properties(collected_item(request, test.__name__)) == fixed_prefix( + "llm.responses.openai.tool_use.nonstream.works", test.__code__.co_firstlineno + ) + + def test_a_test_with_neither_marker_carries_only_the_prefix(self, request: pytest.FixtureRequest) -> None: + """Which is every test in the suite until the backfill lands: an empty + `covers` and no typed properties at all, never five empty ones.""" + test = type(self).test_a_test_with_neither_marker_carries_only_the_prefix + item = collected_item(request, test.__name__) + assert subject_properties(item) == () + assert result_properties(item) == fixed_prefix("", test.__code__.co_firstlineno) + + def test_a_marker_carrying_something_other_than_a_subject_emits_nothing( + self, request: pytest.FixtureRequest + ) -> None: + """`@meta` is typed, but `pytest.mark.meta` is not, and a bare + `@pytest.mark.meta` carries no args at all. Neither may produce a property + whose value is a repr.""" + test = type(self).test_a_marker_carrying_something_other_than_a_subject_emits_nothing + request.applymarker(pytest.mark.meta("spend-budgets")) + assert subject_properties(collected_item(request, test.__name__)) == () + + +class TestProviderMirrorsLitellm: + """`Provider` copies litellm's `LlmProviders` values rather than importing + them, so collecting tests/e2e never needs the litellm package -- the suite is + shipped to the e2e runner image on its own, and a `from litellm...` at module + scope in a test file would turn a missing package into a collection error for + every test rather than a slow import. + + A copy can drift, so it is checked here, wherever litellm IS importable (a dev + checkout, this repo's own CI). Where it is not, the whole point is that + nothing fails, so the check skips. + """ + + def test_every_provider_value_is_a_real_litellm_provider(self) -> None: + """One direction only. litellm ships 155 providers and the e2e suite names + a handful; a value missing from `Provider` is a line to add when a test + needs it, but a value that is not a provider at all would ship a property + no consumer can join on.""" + try: + # Local, and the only litellm import under tests/e2e: at module scope + # it would be exactly the collection-time dependency `Provider` exists + # to avoid. + from litellm.types.utils import LlmProviders + except ImportError: # pragma: no cover - the runner image's shape + pytest.skip("litellm is not importable here, which is the property under test") + known = {str(member.value) for member in LlmProviders} + unknown = sorted(member.value for member in Provider if member.value not in known) + assert not unknown, f"not LlmProviders values: {unknown}" + + +class TestStepRecording: + """The recorded half: `@step`-decorated harness helpers append to the running + test's story as they execute. + + Each test here starts from an empty log because conftest's autouse + `_record_steps` fixture resets the recorder at setup -- the same reset the + live suite relies on for per-test isolation. + """ + + def test_steps_land_in_call_order(self) -> None: + @step("register deployment") + def register() -> str: + return "model-id" + + @step("generate virtual key") + def generate() -> str: + return "sk-x" + + _ = register() + _ = generate() + assert STEPS.taken() == ("register deployment", "generate virtual key") + + def test_a_decorated_helper_still_returns_exactly_what_it_did(self) -> None: + """`@step` records, it does not intercept: arguments, return value and + `__name__` all survive it, so decorating a live harness method cannot + change what the test observes.""" + + @step("POST /chat/completions") + def chat(key: str, *, model: str) -> str: + return f"{key}:{model}" + + assert chat("sk-x", model="gpt-5.5") == "sk-x:gpt-5.5" + assert chat.__name__ == "chat" + + def test_a_helper_that_raises_leaves_its_own_label_last(self) -> None: + """The whole point of the field. The label is recorded BEFORE the call, so + a test that dies inside a helper keeps a partial story whose last element + names the helper it died in.""" + + @step("generate virtual key") + def generate() -> str: + return "sk-x" + + @step("POST /chat/completions") + def chat() -> None: + raise RuntimeError("502 from upstream") + + _ = generate() + with pytest.raises(RuntimeError, match="502 from upstream"): + chat() + assert STEPS.taken() == ("generate virtual key", "POST /chat/completions") + + def test_a_poll_loop_is_one_step_in_the_story_not_fifty(self) -> None: + @step("poll /spend/logs for the request id") + def poll() -> None: + return None + + for _ in range(20): + poll() + assert STEPS.taken() == ("poll /spend/logs for the request id",) + + def test_the_same_label_recorded_again_later_is_a_new_step(self) -> None: + """Only CONSECUTIVE duplicates collapse; a helper called again after + something else happened is a genuine second beat of the story.""" + STEPS.record("POST /chat/completions") + STEPS.record("poll /spend/logs") + STEPS.record("POST /chat/completions") + assert STEPS.taken() == ("POST /chat/completions", "poll /spend/logs", "POST /chat/completions") + + def test_the_log_is_capped_so_a_load_test_cannot_bury_the_story(self) -> None: + for index in range(MAX_STEPS * 2): + STEPS.record(f"call {index}") + taken = STEPS.taken() + assert len(taken) == MAX_STEPS + assert taken[0] == "call 0" + + def test_whitespace_is_normalized_and_an_empty_label_records_nothing(self) -> None: + STEPS.record(" POST /chat/completions\n ") + STEPS.record(" ") + assert STEPS.taken() == ("POST /chat/completions",) + + def test_reset_empties_the_log_so_one_test_never_inherits_another_s(self) -> None: + STEPS.record("register deployment") + STEPS.reset() + assert STEPS.taken() == () + assert step_properties() == () + + def test_steps_serialize_as_repeated_properties_in_order(self) -> None: + """Repeated rather than joined on a delimiter: the labels are free text, so + no separator can be reserved, and a repeated property has none to corrupt.""" + STEPS.record('attach guardrail, comma & "quoted" ') + STEPS.record("POST /chat/completions") + assert step_properties() == ( + ("step", 'attach guardrail, comma & "quoted" '), + ("step", "POST /chat/completions"), + ) + + +class TestAttachStepProperties: + def test_steps_are_appended_after_the_declared_properties(self, request: pytest.FixtureRequest) -> None: + """Order inside `` is list order, so the story reads after the + fixed prefix the collection hook already attached.""" + test = type(self).test_steps_are_appended_after_the_declared_properties + item = collected_item(request, test.__name__) + STEPS.record("register deployment") + STEPS.record("POST /chat/completions") + attach_step_properties(item) + assert [name for name, _ in item.user_properties] == ["package", "covers", "source", "step", "step"] + assert [value for name, value in item.user_properties if name == "step"] == [ + "register deployment", + "POST /chat/completions", + ] + + def test_a_rerun_replaces_the_story_rather_than_appending_a_second_one( + self, request: pytest.FixtureRequest + ) -> None: + """The suite runs with `--reruns 1`. Without this the retry's steps would + queue up behind the first attempt's and the report would read as one test + that did everything twice.""" + test = type(self).test_a_rerun_replaces_the_story_rather_than_appending_a_second_one + item = collected_item(request, test.__name__) + STEPS.record("attempt one died here") + attach_step_properties(item) + STEPS.reset() + STEPS.record("attempt two got further") + attach_step_properties(item) + assert [value for name, value in item.user_properties if name == "step"] == ["attempt two got further"]