mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
e26a6450c8
* test(integration): drop the contracts.json manifest and the covers requirement Groups live as a GROUPS literal in run.py, the browser expectations move next to the browser tests, and the runner fails only on pytest failure, collection errors or a selected file that collects zero tests. The covers marker stays registered for the existing tests but is no longer checked. The mcp directory gets its own group Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(integration): run mcp as its own shard with xdist and a peer proxy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(integration): INTEGRATION_COVERAGE=1 runs the proxy under coverage for the MCP modules The mcp shard sets it. The proxy and its peer start under coverage run in parallel mode, get SIGTERM after the tests so coverage flushes, and the combined text and HTML reports land in the suite results that CircleCI already stores as artifacts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(integration): let the test proxy flush coverage when uvicorn re-raises SIGTERM Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): add SSE, stdio, scripted, OpenAPI and OAuth 2.1 MCP peer doubles Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): add MCP transport and access-control matrices Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): add MCP credential and OAuth flow coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): add MCP LLM endpoint, accounting, guardrail, resilience and lifecycle coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): stop the same-URL grant test from counting a late initialize as a leaked call and satisfy the test-tree lint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): assert the REST denied-server listing is refused or empty Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): pin the REST denied-server listing to 403 access_denied Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yuneng <yuneng@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
60 lines
2.1 KiB
Python
60 lines
2.1 KiB
Python
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
from pydantic import TypeAdapter
|
|
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|
|
|
|
|
class BrowserAttempt(TypedDict):
|
|
status: ReadOnly[str]
|
|
retry: ReadOnly[int]
|
|
|
|
|
|
class BrowserTest(TypedDict):
|
|
results: ReadOnly[list[BrowserAttempt]]
|
|
|
|
|
|
class BrowserSpec(TypedDict):
|
|
file: ReadOnly[str]
|
|
title: ReadOnly[str]
|
|
tests: ReadOnly[list[BrowserTest]]
|
|
|
|
|
|
class BrowserSuite(TypedDict):
|
|
specs: NotRequired[ReadOnly[list[BrowserSpec]]]
|
|
suites: NotRequired[ReadOnly[list["BrowserSuite"]]]
|
|
|
|
|
|
def main() -> None:
|
|
result: Final = json.loads(Path(sys.argv[1]).read_text())
|
|
assert not result.get("errors"), result.get("errors")
|
|
expected: Final = json.loads(
|
|
(Path(__file__).resolve().parents[2] / "tests/e2e/ui/tests/integrationCritical/expected.json").read_text()
|
|
)
|
|
assert expected and result["stats"]["expected"] == len(expected)
|
|
assert all(result["stats"][name] == 0 for name in ("unexpected", "flaky", "skipped"))
|
|
|
|
def cases(suite: BrowserSuite) -> tuple[BrowserSpec, ...]:
|
|
return tuple(suite.get("specs", ())) + tuple(spec for child in suite.get("suites", ()) for spec in cases(child))
|
|
|
|
suites: Final = TypeAdapter(list[BrowserSuite]).validate_python(result["suites"], strict=True)
|
|
specs: Final = tuple(spec for suite in suites for spec in cases(suite))
|
|
repository: Final = Path(__file__).resolve().parents[2]
|
|
report_root: Final = Path(result["config"]["rootDir"])
|
|
assert report_root.is_absolute(), "Playwright rootDir must be explicit"
|
|
observed: Final = tuple(
|
|
str((report_root / spec["file"]).resolve().relative_to(repository)) + "::" + spec["title"] for spec in specs
|
|
)
|
|
assert sorted(observed) == sorted(expected)
|
|
for spec in specs:
|
|
tests: Final = spec["tests"]
|
|
assert len(tests) == 1 and len(tests[0]["results"]) == 1
|
|
assert tests[0]["results"][0]["status"] == "passed" and tests[0]["results"][0]["retry"] == 0
|
|
|
|
sys.stdout.write("One canonical browser contract passed once without skips or retries\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|