mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-26 01:12:21 +00:00
ee5efaed7b
A test's JUnit report says what happened to it, never what it was about.
`@pytest.mark.covers("cell.id")` is a registry key, not a description: it
cannot answer "which tests drive /v1/responses on Anthropic", and nothing
in the report says where a failing test actually died.
Two halves, deliberately separated, both riding out as JUnit <property>
entries the downstream emitter already knows how to read.
DECLARED - `@meta(Subject(...))` from the new tests/e2e/e2e_metadata.py.
One frozen dataclass, every field a closed enum (domain, route, provider,
model, capabilities, mode), so a typo is a basedpyright error at the call
site rather than a property that silently never appears. Serialization is
one pass over `dataclasses.asdict`, so a new scalar field needs no
serializer edit; `capabilities` is deduped and sorted at declaration so
committed run artifacts diff cleanly whatever order a test spelled it in.
Empty fields emit nothing - the suite does not pad every testcase with
five empty entries.
RECORDED - `@step("POST /chat/completions")` on harness helpers, never on
tests. Each call appends its label to the running test's user_properties
in call order, so the list IS the test's user story and cannot drift from
what the test did. The label is recorded BEFORE the wrapped call, so a
helper that raises still leaves its own label last: a failing test's last
step is where it died. Consecutive duplicates collapse and the log caps at
50, so a poll loop is one beat of the story rather than fifty.
Steps cannot be attached where the other properties are -
`pytest_collection_modifyitems` runs before any test body, so the recorder
is empty there. They attach from the existing `pytest_runtest_makereport`
wrapper on the call phase, which is what puts them on failures too, and an
autouse fixture empties the log at setup. The attach drops any prior step
entries first, because the suite runs `--reruns 1` and a retry would
otherwise stack a second copy of the story behind the first.
`covers` is untouched: the marker is separate because a dataclass passed
to `covers` would be dropped silently by `dedupe_covers` and would hard-
fail collection in tests/integration/conftest.py. The fixed
package/covers/source prefix stays byte-identical, and `@meta` goes BELOW
`@covers` so `Item.location` still anchors at the first decorator and
every `source` deep link keeps pointing where it pointed.
`Provider` mirrors litellm's `LlmProviders` values instead of importing
them, so nothing here - the module or its call sites - imports litellm.
tests/e2e is a black-box HTTP suite that is copied to the runner image on
its own, so a `from litellm...` at the top of a test module would make the
package a COLLECTION-time dependency: where it is absent, every test in
the suite errors out before running rather than importing slowly. The
mirror cannot drift silently - `TestProviderMirrorsLitellm` asserts every
value is a real `LlmProviders` value wherever litellm is importable, and
skips where it is not, which is the property it is guarding.
A declared `model` names the constant the test drives, never a copy of its
value: `CHEAP_ANTHROPIC_MODEL` and `CHEAP_OPENAI_MODEL` are env-overridable
(`E2E_CHEAP_ANTHROPIC_MODEL`, `E2E_CHEAP_OPENAI_MODEL`), so a hardcoded
default would have reported a model the run never touched. The same holds
for a file's own `BACKEND`/`MODEL` constant, where the copy was merely
waiting to drift.
Pilot: tests/e2e/quota_management, all 85 tests annotated and its three
clients plus cost_rows @step-decorated, to prove the API against real
tests rather than a toy. The rest of the suite is a later backfill.
Verified: 42 harness unit tests in test_junit_properties.py (19 new), 550
harness unit tests green, basedpyright over tests/e2e at the same 29
pre-existing errors as origin/main (all in the untouched mcp/
oauth_chat_client.py), ruff clean, the coverage-registry collector
byte-identical before and after (473/586), and the test-quality gate OK
against origin/main. Collection was also run with the litellm package
blocked at the import hook: 1339/1350 collected either way, the one error
being the pre-existing missing `httpx2` in mcp/. The live e2e tests need a
deployed proxy and real provider keys and were not run.
103 lines
3.9 KiB
Python
103 lines
3.9 KiB
Python
"""Resource cleanup for stateful e2e tests.
|
|
|
|
Shared by every e2e suite under tests/e2e/. The proxy under test is
|
|
long-lived and never reset between tests, so anything a test creates (keys,
|
|
customers, teams, orgs, users, guardrails, budgets, ...) persists unless
|
|
explicitly deleted. The `resources` fixture (see conftest.py) hands each test a
|
|
ResourceManager; the test registers a cleanup for every resource it creates, and
|
|
the fixture's teardown releases them all even when the test body raises.
|
|
"""
|
|
|
|
from builtins import ExceptionGroup
|
|
from dataclasses import dataclass, field
|
|
from typing import Callable, Final, List, Protocol, runtime_checkable
|
|
|
|
from e2e_metadata import step
|
|
from proxy_client import ProxyClient
|
|
from models import KeyGenerateBody
|
|
|
|
|
|
@runtime_checkable
|
|
class ResourceClient(Protocol):
|
|
"""Proxy operations the convenience creators use. Resource types without a
|
|
creator here are handled generically via ResourceManager.defer(). The ProxyClient
|
|
satisfies this."""
|
|
|
|
def generate_key(self, body: KeyGenerateBody) -> str: ...
|
|
|
|
def delete_key(self, key: str) -> None: ...
|
|
|
|
def delete_customers(self, user_ids: List[str]) -> None: ...
|
|
|
|
|
|
@runtime_checkable
|
|
class ProxyClientProvider(Protocol):
|
|
"""Every suite's client exposes the shared ProxyClient, which the resources fixture
|
|
uses for cleanup. The client adds its own route methods on top."""
|
|
|
|
@property
|
|
def proxy(self) -> ProxyClient: ...
|
|
|
|
|
|
@dataclass
|
|
class ResourceManager:
|
|
"""Registry of teardown actions for resources a test creates on the stateful
|
|
proxy.
|
|
|
|
Not limited to any resource type: register a cleanup with ``defer()`` for a
|
|
key, customer, team, org, user, guardrail, budget, MCP server - anything with
|
|
a delete. The two most common resources have sugar (``key``, ``customer``);
|
|
everything else is ``resources.defer(lambda: client.delete_team(team_id))``.
|
|
|
|
Cleanups run LIFO (so a resource is removed before whatever it depends on) and
|
|
best-effort (one failing cleanup never blocks the rest).
|
|
"""
|
|
|
|
client: ResourceClient
|
|
strict_cleanup: bool = False
|
|
_cleanups: List[Callable[[], object]] = field(
|
|
default_factory=list
|
|
) # mutable-ok: append-only teardown registry
|
|
|
|
def init(self) -> None:
|
|
"""No global setup needed today; present for lifecycle symmetry."""
|
|
return None
|
|
|
|
def defer(self, cleanup: Callable[[], object]) -> None:
|
|
"""Register a teardown action for any resource the test just created.
|
|
|
|
Whatever the action returns is discarded, so a delete that answers with a
|
|
response model can be deferred directly."""
|
|
self._cleanups.append(cleanup)
|
|
|
|
@step("generate virtual key")
|
|
def key(self, models: list[str] | None = None, user_id: str | None = "e2e-test-user") -> str:
|
|
"""Create a virtual key; delete it on teardown. `models` restricts which
|
|
models the key may call (None/[] means all). `user_id` is required for
|
|
managed-batch ACL: the proxy stores created_by=user_id and checks it on
|
|
retrieve/cancel; None here means the 403 guard fires."""
|
|
key = self.client.generate_key(KeyGenerateBody(models=models or [], user_id=user_id))
|
|
self.defer(lambda: self.client.delete_key(key))
|
|
return key
|
|
|
|
@step("register end user")
|
|
def customer(self, customer_id: str) -> str:
|
|
"""Track an end-user id (from the `user` param); delete it on teardown."""
|
|
self.defer(lambda: self.client.delete_customers([customer_id]))
|
|
return customer_id
|
|
|
|
def teardown(self) -> None:
|
|
failures: Final = tuple(
|
|
failure for cleanup in reversed(self._cleanups)
|
|
if (failure := _run_cleanup(cleanup)) is not None
|
|
)
|
|
if failures and self.strict_cleanup:
|
|
raise ExceptionGroup("Resource cleanup failed", failures)
|
|
|
|
|
|
def _run_cleanup(cleanup: Callable[[], object]) -> Exception | None:
|
|
try:
|
|
cleanup()
|
|
except Exception as exc:
|
|
return exc
|
|
return None
|