mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
d0e37d39c4
A test's JUnit report says whether it passed, never what it did or where a failing test died. This records that from the harness, so nothing about it is hand-written and it cannot drift from what the test actually ran
`@step("create team with a budget")` from the new tests/e2e/e2e_metadata.py goes on harness helpers, never on tests, and appends its label to the running test's step log in call order. The label is recorded before the wrapped call, so a helper that raises still leaves its own label last: a failing test's last step is where it died. Every public harness method that performs an action now carries one, 355 across the client modules, lifecycle, idp, the logging readers, migrations and the claude_code driver
Only the outermost step records, tracked per thread. Harness layers call each other (ResourceManager.key goes through ProxyClient.generate_key, a domain client wraps the shared ProxyClient), so every layer carries a label and the story still reads at the level the test called in at, one beat per action. A step above @contextmanager holds the guard through __enter__ and __exit__, so a context's cleanup never lands behind the step a test died on, and a bare generator function is refused at import because its body interleaves with its caller's. Consecutive duplicates collapse and the log caps at 50, so a poll loop is one beat rather than fifty. The wrapper is a frame, so the eight cleanup and retry warnings raised directly inside decorated helpers use stacklevel=2 + STEP_FRAMES to keep reporting at their caller
The log is emptied first thing in pytest_runtest_setup and attached from the existing pytest_runtest_makereport wrapper after setup and again after call, so a test that errors in a fixture keeps the steps recorded before the crash. Teardown does not attach: finalizer steps are cleanup. Each attach drops the item's earlier step entries, so the second attach and a --reruns 1 retry replace the story rather than doubling it
Steps ride out as repeated <property name="step"> entries behind the fixed package/covers/source prefix, which stays byte-identical. The project-releaser emitter already regroups them into the results JSON's steps array. test_junit_report.py runs real pytest with --junitxml against this conftest, in-process and under -n 2, and pins the passing, failing, setup-error, rerun and wide-scope-fixture cases on the parsed XML
61 lines
1.9 KiB
Python
61 lines
1.9 KiB
Python
"""Shared helpers for e2e tests that register the real Datadog remote MCP server."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from collections.abc import Sequence
|
|
|
|
from e2e_config import datadog_mcp_url, unique_marker
|
|
from e2e_metadata import step
|
|
from lifecycle import ResourceManager
|
|
from mcp_client import McpClient
|
|
|
|
SEARCH_LOGS_TOOL = "search_datadog_logs"
|
|
|
|
|
|
def _dd_api_key() -> str:
|
|
return os.environ.get("DD_API_KEY", "").strip()
|
|
|
|
|
|
def _dd_app_key() -> str:
|
|
return os.environ.get("DD_APP_KEY", "").strip()
|
|
|
|
|
|
def assert_dd_mcp_creds() -> None:
|
|
if not _dd_api_key() or not _dd_app_key():
|
|
import pytest
|
|
|
|
pytest.fail(
|
|
"Datadog MCP e2e requires DD_API_KEY and DD_APP_KEY "
|
|
"(header auth to mcp.<site>/v1/mcp; on the cluster the secret manager "
|
|
"injects them, locally tests/e2e/.env)"
|
|
)
|
|
|
|
|
|
@step("register the Datadog MCP server")
|
|
def register_datadog_mcp(
|
|
client: McpClient,
|
|
resources: ResourceManager,
|
|
*,
|
|
mcp_access_groups: list[str] | None = None,
|
|
allowed_tools: Sequence[str] | None = (SEARCH_LOGS_TOOL,),
|
|
) -> str:
|
|
"""Register the core Datadog toolset with its credentials from the env. By default
|
|
the server exposes only `search_datadog_logs`; pass `allowed_tools=None` to expose
|
|
every tool the core toolset serves."""
|
|
assert_dd_mcp_creds()
|
|
name = f"e2e_dd_mcp_{unique_marker()}"
|
|
server_id = client.register_server(
|
|
server_name=name,
|
|
alias=name,
|
|
url=datadog_mcp_url(toolsets="core"),
|
|
transport="http",
|
|
static_headers={
|
|
"DD-API-KEY": _dd_api_key(),
|
|
"DD-APPLICATION-KEY": _dd_app_key(),
|
|
},
|
|
allowed_tools=None if allowed_tools is None else list(allowed_tools),
|
|
mcp_access_groups=mcp_access_groups,
|
|
)
|
|
resources.defer(lambda: client.delete_server(server_id))
|
|
return server_id
|