From 34d2d010c3f0f08ab9c13700c7004d98377309fa Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 19:33:59 +0000 Subject: [PATCH] test(cli): drop lite e2e tests, the e2e runner does not install the package Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 2 - tests/e2e/other/test_cli_cost_map_e2e.py | 104 ----------------------- 2 files changed, 106 deletions(-) delete mode 100644 tests/e2e/other/test_cli_cost_map_e2e.py diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index a7ba0dbb8cb..814ebae2e0b 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -48,5 +48,3 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} -- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} -- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py deleted file mode 100644 index 62cd3d6c3f5..00000000000 --- a/tests/e2e/other/test_cli_cost_map_e2e.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -import os -import shutil -import subprocess -import threading -from collections.abc import Mapping -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Final - -import pytest -from e2e_config import MASTER_KEY, PROXY_BASE_URL -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, threading.Thread]: - class CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - with request_log.open("a", encoding="utf-8") as log_file: - log_file.write(f"{self.path}\n") - body: Final = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - return - - server: Final = ThreadingHTTPServer(("127.0.0.1", 0), CostMapHandler) - thread: Final = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - return server, thread - - -def _lite_env(server: ThreadingHTTPServer, api_key: str | None) -> dict[str, str]: - base_env: Final = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} - return { - **base_env, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - **({"LITELLM_PROXY_API_KEY": api_key} if api_key is not None else {}), - } - - -def _run_lite( - args: tuple[str, ...], - server: ThreadingHTTPServer, - env: Mapping[str, str], -) -> subprocess.CompletedProcess[str]: - lite_path: Final = shutil.which("lite") - assert lite_path is not None, "the installed lite executable is required for e2e coverage" - try: - return subprocess.run( - [lite_path, *args], - capture_output=True, - text=True, - timeout=60, - env=env, - ) - finally: - server.shutdown() - server.server_close() - - -def _request_count(request_log: Path) -> int: - return request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 - - -class TestLiteCliCostMapFetch: - @pytest.mark.covers("other.cli.model_cost_map.version_skips_fetch") - def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: - request_log: Final = tmp_path / "requests.log" - server, thread = _start_cost_map_server(request_log) - env: Final = _lite_env(server, None) - try: - result: Final = _run_lite(("--version",), server, env) - finally: - thread.join(timeout=10) - - assert result.returncode == 0 - assert "LiteLLM Proxy CLI Version" in result.stdout - assert _request_count(request_log) == 0 - - @pytest.mark.covers("other.cli.model_cost_map.models_list_skips_fetch") - def test_lite_models_list_uses_proxy_not_cost_map(self, tmp_path: Path, proxy: ProxyClient) -> None: - model_names: Final = tuple(entry.model_name for entry in proxy.model_info()) - assert model_names - request_log: Final = tmp_path / "requests.log" - server, thread = _start_cost_map_server(request_log) - env: Final = _lite_env(server, MASTER_KEY) - try: - result: Final = _run_lite(("models", "list"), server, env) - finally: - thread.join(timeout=10) - - assert result.returncode == 0 - assert result.stdout.strip() - assert any(model_name in result.stdout for model_name in model_names) - assert _request_count(request_log) == 0