From c3048dcd306ad40950a1694014a8a4b5f202a551 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 17 Sep 2026 10:37:07 -0700 Subject: [PATCH] test(http): move the outbound HTTP/2 check into a new integration sdk suite The check spins up a hypercorn TLS peer and drives the SDK's own httpx handlers at it, so it needs litellm importable, hypercorn installed and a loopback socket. It lived under tests/e2e, whose Buildkite runner image installs neither litellm nor hypercorn by design (the suite drives a remote proxy over HTTP), so every scheduled e2e build since #230 failed to import the module and pytest reported it as a collection error. The unit tree bans sockets, so it does not belong there either tests/integration is the CircleCI tier built for real TCP against local protocol peers. This adds an sdk shard to it for cases that exercise the SDK's clients with no gateway in the path, registers the two HTTP/2 nodes in the contracts manifest, and adds the shard to the CircleCI matrix. The test now flips the feature through LITELLM_HTTP2 (the user surface) instead of patching module attributes, and asserts the version the peer observed on the wire next to the one the client reports --- .circleci/config.yml | 2 +- tests/integration/README.md | 4 +- tests/integration/_support/manifest.py | 1 + tests/integration/contracts.json | 9 ++ .../sdk/test_http2_wire.py} | 142 +++++++++--------- 5 files changed, 81 insertions(+), 77 deletions(-) rename tests/{e2e/llm_translation/test_outbound_http2_e2e.py => integration/sdk/test_http2_wire.py} (54%) diff --git a/.circleci/config.yml b/.circleci/config.yml index e6aa90233e1..df17a9e4402 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3009,7 +3009,7 @@ workflows: name: integration-<< matrix.suite >> matrix: parameters: - suite: [management, accounting, database, providers, extensions, browser] + suite: [management, accounting, database, providers, extensions, sdk, browser] filters: branches: only: diff --git a/tests/integration/README.md b/tests/integration/README.md index 7e1f39025a1..5ea34fc9180 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -Use `tests/integration/run.py management`, `accounting`, `database`, `providers` or `extensions` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate +Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions` or `sdk` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload @@ -26,6 +26,8 @@ Provider contracts exercise actual TCP requests with synthetic credentials and l Streaming checks send real HTTP transfer chunks, including one-byte partitions, fragmented tools, incomplete transfers and a cancellation barrier. They assert meaningful text, tool arguments, final usage and persisted cost. The Redis recovery case owns a separate database and Redis process, uses the supported one-second circuit-breaker recovery setting, waits for the real subscriber and verifies response data in Redis after restart. CircleCI reuses its existing Redis image for that extra process; it never pulls an image during tests +The sdk shard exercises the SDK's own HTTP clients against local protocol peers with no gateway in the path, so a case here fails only when the client library or its wire behavior changes. The HTTP/2 case runs a hypercorn TLS peer offering h2 and http/1.1 over ALPN, drives the sync and async httpx handlers at it with `LITELLM_HTTP2` off and on, and asserts the version both the client and the peer observed on the wire. Put a test here only when it needs no proxy, database or Redis; a case that reaches the gateway belongs in one of the other shards + The extensions shard reuses the existing MCP arithmetic functions with a real SDK server, and uses the built-in generic callback and guardrail transports. It checks actual tool calls after saved edits, discovery preservation, malformed/error responses, callback correlation and credential exclusion, guardrail rewriting and denial, retained OpenAI consumers, persisted toolsets and A2A wire versions Browser contracts live in `tests/e2e/ui/tests/integrationCritical` and run only through `tests/e2e/ui/integration.config.ts`. The CircleCI browser shard builds the checked-out dashboard, starts the owned proxy with that build, and verifies one exact browser result without retries or skips. The default Playwright selection excludes this directory. The focused project flow asserts the submitted create and clear values, fresh SQL state and actual blocked/restored serving while preserving model restrictions diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index b3a82fa4cdd..3c9a5508ad6 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -19,6 +19,7 @@ OWNED_DIRECTORIES: Final = frozenset( "mcp", "observability", "compatibility", + "sdk", } ) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 5c91a50d572..127970b5506 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -21,6 +21,9 @@ "mcp", "observability", "compatibility" + ], + "sdk": [ + "sdk" ] }, "tests": { @@ -187,6 +190,12 @@ ], "tests/integration/spend/test_filtered_ledger.py::test_rotated_keys_users_and_model_groups_preserve_success_failure_cache_ledger": [ "quota_management.spend_tracking.filtered_ledger_preserves_owner_identity_and_totals" + ], + "tests/integration/sdk/test_http2_wire.py::test_async_handler_negotiates_http2_only_when_enabled": [ + "other.sdk_wire.http2.async_handler_negotiates_h2_only_when_enabled" + ], + "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": [ + "other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled" ] }, "browser": { diff --git a/tests/e2e/llm_translation/test_outbound_http2_e2e.py b/tests/integration/sdk/test_http2_wire.py similarity index 54% rename from tests/e2e/llm_translation/test_outbound_http2_e2e.py rename to tests/integration/sdk/test_http2_wire.py index cb2182ffd62..15bb366c7a2 100644 --- a/tests/e2e/llm_translation/test_outbound_http2_e2e.py +++ b/tests/integration/sdk/test_http2_wire.py @@ -1,21 +1,14 @@ -"""Outbound HTTP/2 negotiation for LiteLLM-built httpx clients. - -Spins up a local hypercorn TLS server that offers h2 and http/1.1 over ALPN and -drives the real AsyncHTTPHandler / HTTPHandler at it, so the negotiated protocol -on the wire is the assertion. No running proxy or provider credentials needed, -which is why these tests carry no `e2e` marker (same shape as the markerless -harness checks under tests/e2e/load/). -""" - from __future__ import annotations import asyncio import datetime import ipaddress +import json import socket import threading import time from collections.abc import Iterator +from dataclasses import dataclass from pathlib import Path from typing import Final, cast @@ -28,16 +21,17 @@ from hypercorn.asyncio import ( serve, # pyright: ignore[reportUnknownVariableType] # hypercorn's serve signature passes through untyped worker hooks ) from hypercorn.config import Config -from hypercorn.typing import ( - ASGIReceiveCallable, - ASGISendCallable, - HTTPResponseBodyEvent, - HTTPResponseStartEvent, - Scope, -) +from hypercorn.typing import ASGIReceiveCallable, ASGISendCallable, HTTPResponseBodyEvent, HTTPResponseStartEvent, Scope -import litellm -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +STREAM_CHUNKS: Final = 3 + + +@dataclass(frozen=True, slots=True) +class Observed: + post_version: str + post_peer_version: str + stream_version: str + stream_body: bytes def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]: @@ -71,7 +65,7 @@ def _write_self_signed_cert(cert_dir: Path) -> tuple[Path, Path]: return cert_file, key_file -async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: +async def _peer(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: if scope["type"] != "http": return while True: @@ -80,16 +74,17 @@ async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCa return if message["type"] == "http.request" and not message["more_body"]: break + version: Final = scope["http_version"] if scope["path"] == "/stream": await send( HTTPResponseStartEvent( type="http.response.start", status=200, headers=[(b"content-type", b"text/event-stream")] ) ) - for index in range(3): + for index in range(STREAM_CHUNKS): await send( HTTPResponseBodyEvent( - type="http.response.body", body=f"data: chunk-{index}\n\n".encode(), more_body=True + type="http.response.body", body=f"data: {version}-{index}\n\n".encode(), more_body=True ) ) await send(HTTPResponseBodyEvent(type="http.response.body", body=b"", more_body=False)) @@ -97,18 +92,19 @@ async def _asgi_app(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCa await send( HTTPResponseStartEvent(type="http.response.start", status=200, headers=[(b"content-type", b"application/json")]) ) - await send(HTTPResponseBodyEvent(type="http.response.body", body=b'{"ok": true}', more_body=False)) + await send( + HTTPResponseBodyEvent( + type="http.response.body", body=json.dumps({"http_version": version}).encode(), more_body=False + ) + ) @pytest.fixture(scope="module") -def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: - cert_dir: Final = tmp_path_factory.mktemp("h2certs") - cert_file, key_file = _write_self_signed_cert(cert_dir) - +def http2_tls_peer(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: + cert_file, key_file = _write_self_signed_cert(tmp_path_factory.mktemp("h2certs")) with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port: Final = cast(int, sock.getsockname()[1]) - shutdown: Final = threading.Event() def _serve() -> None: @@ -118,12 +114,11 @@ def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: config.certfile = str(cert_file) config.keyfile = str(key_file) config.alpn_protocols = ["h2", "http/1.1"] - loop.run_until_complete(serve(_asgi_app, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait))) + loop.run_until_complete(serve(_peer, config, shutdown_trigger=lambda: asyncio.to_thread(shutdown.wait))) loop.close() thread: Final = threading.Thread(target=_serve, daemon=True) thread.start() - for _ in range(100): try: with socket.create_connection(("127.0.0.1", port), timeout=0.2): @@ -131,78 +126,75 @@ def http2_tls_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: except OSError: time.sleep(0.05) else: - pytest.fail("hypercorn test server did not start") - + pytest.fail("hypercorn peer did not start") yield f"https://127.0.0.1:{port}" - shutdown.set() thread.join(timeout=10) -def _async_exchange(base_url: str) -> tuple[str, str, bytes]: - async def _run() -> tuple[str, str, bytes]: +def _async_exchange(base_url: str) -> Observed: + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + async def _run() -> Observed: handler: Final = AsyncHTTPHandler(ssl_verify=False) try: response: Final = await handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) - post_version: Final = response.http_version async with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: - stream_version: Final = stream_response.http_version - body: Final = b"".join([chunk async for chunk in stream_response.aiter_bytes()]) - return post_version, stream_version, body + return Observed( + post_version=response.http_version, + post_peer_version=response.json()["http_version"], + stream_version=stream_response.http_version, + stream_body=b"".join([chunk async for chunk in stream_response.aiter_bytes()]), + ) finally: await handler.close() return asyncio.run(_run()) -def _sync_exchange(base_url: str) -> tuple[str, str, bytes]: +def _sync_exchange(base_url: str) -> Observed: + from litellm.llms.custom_httpx.http_handler import HTTPHandler + handler: Final = HTTPHandler(ssl_verify=False) try: response: Final = handler.client.post(f"{base_url}/echo", json={"ping": "pong"}) - post_version: Final = response.http_version with handler.client.stream("POST", f"{base_url}/stream", json={}) as stream_response: - stream_version: Final = stream_response.http_version - body: Final = b"".join(stream_response.iter_bytes()) - return post_version, stream_version, body + return Observed( + post_version=response.http_version, + post_peer_version=response.json()["http_version"], + stream_version=stream_response.http_version, + stream_body=b"".join(stream_response.iter_bytes()), + ) finally: handler.close() -class TestOutboundHttp2: - @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")]) - def test_async_handler_negotiates_http2_only_when_enabled( - self, - monkeypatch: pytest.MonkeyPatch, - http2_tls_server: str, - use_http2: bool, - expected_version: str, - ) -> None: - monkeypatch.setattr(litellm, "http2", use_http2) +def _set_http2(monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None: + if enabled: + monkeypatch.setenv("LITELLM_HTTP2", "True") + else: monkeypatch.delenv("LITELLM_HTTP2", raising=False) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "force_ipv4", False) - post_version, stream_version, body = _async_exchange(http2_tls_server) - assert post_version == expected_version - assert stream_version == expected_version - assert b"data: chunk-0" in body +def _assert_negotiated(observed: Observed, enabled: bool) -> None: + client_version, peer_version = ("HTTP/2", "2") if enabled else ("HTTP/1.1", "1.1") + assert observed.post_version == client_version + assert observed.post_peer_version == peer_version + assert observed.stream_version == client_version + expected_stream: Final = b"".join(f"data: {peer_version}-{index}\n\n".encode() for index in range(STREAM_CHUNKS)) + assert observed.stream_body == expected_stream - @pytest.mark.parametrize("use_http2, expected_version", [(True, "HTTP/2"), (False, "HTTP/1.1")]) - def test_sync_handler_negotiates_http2_only_when_enabled( - self, - monkeypatch: pytest.MonkeyPatch, - http2_tls_server: str, - use_http2: bool, - expected_version: str, - ) -> None: - monkeypatch.setattr(litellm, "http2", use_http2) - monkeypatch.delenv("LITELLM_HTTP2", raising=False) - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - monkeypatch.setattr(litellm, "force_ipv4", False) - post_version, stream_version, body = _sync_exchange(http2_tls_server) +@pytest.mark.covers("other.sdk_wire.http2.async_handler_negotiates_h2_only_when_enabled") +def test_async_handler_negotiates_http2_only_when_enabled(monkeypatch: pytest.MonkeyPatch, http2_tls_peer: str) -> None: + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + for enabled in (False, True): + _set_http2(monkeypatch, enabled) + _assert_negotiated(_async_exchange(http2_tls_peer), enabled) - assert post_version == expected_version - assert stream_version == expected_version - assert b"data: chunk-0" in body + +@pytest.mark.covers("other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled") +def test_sync_handler_negotiates_http2_only_when_enabled(monkeypatch: pytest.MonkeyPatch, http2_tls_peer: str) -> None: + for enabled in (False, True): + _set_http2(monkeypatch, enabled) + _assert_negotiated(_sync_exchange(http2_tls_peer), enabled)