From 2ece8735384e063b88ca91cf9d2f8f6d2e94b30a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 17:05:38 +0000 Subject: [PATCH] test(e2e): lite CLI never fetches the model cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 2 + tests/e2e/other/test_cli_cost_map_e2e.py | 109 +++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 tests/e2e/other/test_cli_cost_map_e2e.py diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 814ebae2e0b..8ac4fc0762c 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -2,6 +2,8 @@ # PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. - {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} - {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} +- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} +- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} - {id: other.auth.llm_chat.missing_header_denied, module: other, tier: P0, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Chat with no Authorization header is 401/403"} - {id: other.auth.llm_chat.invalid_bearer_denied, module: other, tier: P0, area: auth, assertions: [invalid_bearer_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Bearer invalid_token on chat is 401/403"} - {id: other.auth.llm_chat.no_bearer_prefix_denied, module: other, tier: P0, area: auth, assertions: [no_bearer_prefix_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Token without Bearer scheme on chat is 401/403"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py new file mode 100644 index 00000000000..6022d5a0a4c --- /dev/null +++ b/tests/e2e/other/test_cli_cost_map_e2e.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +from collections.abc import Mapping +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import MASTER_KEY, PROXY_BASE_URL +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + + +def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, threading.Thread]: + class CostMapHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + with request_log.open("a", encoding="utf-8") as log_file: + log_file.write(f"{self.path}\n") + body: Final = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), CostMapHandler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread + + +def _run_lite( + args: tuple[str, ...], + server: ThreadingHTTPServer, + env: Mapping[str, str], +) -> subprocess.CompletedProcess[str]: + lite_path: Final = shutil.which("lite") + assert lite_path is not None, "the installed lite executable is required for e2e coverage" + try: + return subprocess.run( + [lite_path, *args], + capture_output=True, + text=True, + timeout=60, + env=env, + ) + finally: + server.shutdown() + server.server_close() + + +def _request_count(request_log: Path) -> int: + return request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 + + +class TestLiteCliCostMapFetch: + @pytest.mark.covers("other.cli.model_cost_map.version_skips_fetch") + def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: + request_log: Final = tmp_path / "requests.log" + server, thread = _start_cost_map_server(request_log) + source_root: Final = str(Path(__file__).resolve().parents[3]) + pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) + env: Final = { + **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, + "PYTHONPATH": pythonpath, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + } + try: + result: Final = _run_lite(("--version",), server, env) + finally: + thread.join(timeout=10) + + assert result.returncode == 0 + assert "LiteLLM Proxy CLI Version" in result.stdout + assert _request_count(request_log) == 0 + + @pytest.mark.covers("other.cli.model_cost_map.models_list_skips_fetch") + def test_lite_models_list_uses_proxy_not_cost_map(self, tmp_path: Path, proxy: ProxyClient) -> None: + model_names: Final = tuple(entry.model_name for entry in proxy.model_info()) + assert model_names + request_log: Final = tmp_path / "requests.log" + server, thread = _start_cost_map_server(request_log) + source_root: Final = str(Path(__file__).resolve().parents[3]) + pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) + env: Final = { + **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, + "PYTHONPATH": pythonpath, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + "LITELLM_PROXY_API_KEY": MASTER_KEY, + } + try: + result: Final = _run_lite(("models", "list"), server, env) + finally: + thread.join(timeout=10) + + assert result.returncode == 0 + assert result.stdout.strip() + assert any(model_name in result.stdout for model_name in model_names) + assert _request_count(request_log) == 0