From 25a6c2a2749ba802514fb3a5736b540c21a1e630 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 06:17:59 +0000 Subject: [PATCH 1/7] fix(cli): skip remote model cost map fetch in lite CLI processes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 14 ++++- .../test_get_model_cost_map.py | 25 +++++++++ .../proxy/client/cli/test_global_options.py | 56 ++++++++++++++++++- 3 files changed, 91 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index cdc4810ff04..7d3b67e58c8 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -2,6 +2,7 @@ Pulls the cost + context window + provider route for known models from https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json This can be disabled by setting the LITELLM_LOCAL_MODEL_COST_MAP environment variable to True. +The ``lite`` and ``litellm-proxy`` CLI entry points also use the bundled map without fetching. ``` export LITELLM_LOCAL_MODEL_COST_MAP=True @@ -13,11 +14,13 @@ import hashlib import json import os import random +import sys import time from collections.abc import Awaitable, Callable from dataclasses import dataclass, replace from datetime import datetime, timezone from importlib.resources import files +from pathlib import Path from typing import Final, Protocol import httpx @@ -33,6 +36,12 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" +_CLI_ENTRYPOINT_NAMES: Final = frozenset({"lite", "litellm-proxy"}) + + +def _is_cli_process() -> bool: + return Path(sys.argv[0]).stem in _CLI_ENTRYPOINT_NAMES + # Reserved top-level keys that are not model entries. They must be excluded # from the model-count integrity check so a real upstream shrink can't be masked. @@ -531,7 +540,8 @@ def get_model_cost_map( """ Public entry point — returns the model cost map dict. - 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only. + 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set or this is a ``lite`` / + ``litellm-proxy`` CLI process, uses the local backup only. 2. Otherwise fetches from ``url``, retrying transient HTTP errors (429/5xx/transport) with Retry-After-aware backoff, validates integrity, and falls back to the local backup on any failure. @@ -543,7 +553,7 @@ def get_model_cost_map( _cost_map_source_info.loaded_at = datetime.now(timezone.utc) # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. - if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index 0495440c51c..d092b387259 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -6,6 +6,7 @@ count actual model entries, not reserved meta keys) and the extraction of the import json import os +import sys import pytest @@ -711,3 +712,27 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej assert source["etag"] is None assert source["source_revision"] == _bundled_blob_id() assert source["source_revision"] != git_blob_id(shrunk_body) + + +@pytest.mark.parametrize( + ("argv0", "request_count"), + [ + ("/some/venv/bin/lite", 0), + ("/some/venv/bin/lite.exe", 0), + ("/some/venv/bin/python", 1), + ], +) +def test_boot_load_skips_remote_fetch_for_cli_processes(monkeypatch, argv0, request_count): + monkeypatch.setattr(sys, "argv", [argv0, "--version"]) + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) + + cost_map = get_model_cost_map(url=_URL, client=client) + + assert calls["count"] == request_count + assert cost_map + source = get_model_cost_map_source_info() + if request_count == 0: + assert source["source"] == "local" + else: + assert source["source"] == "remote" diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 0dd388919a5..02b1cdf4296 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,14 +1,16 @@ # stdlib imports import json import os +import shutil +import subprocess +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from unittest.mock import Mock, patch import pytest from click.testing import CliRunner - - import litellm.proxy.client.cli from litellm._version import version as litellm_version from litellm.proxy.client.cli import cli @@ -35,6 +37,56 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output +def test_lite_version_does_not_fetch_model_cost_map(): + lite_path = shutil.which("lite") + if lite_path is None: + pytest.skip("lite executable is unavailable") + + requests = [] + + class _CostMapHandler(BaseHTTPRequestHandler): + def do_GET(self): + requests.append(self.path) + body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), _CostMapHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + env = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} + source_root = str(Path(__file__).resolve().parents[5]) + env["PYTHONPATH"] = os.pathsep.join(filter(None, (source_root, env.get("PYTHONPATH")))) + env.update( + { + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": "http://127.0.0.1:9", + } + ) + result = subprocess.run( + [lite_path, "--version"], + capture_output=True, + text=True, + timeout=120, + env=env, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=10) + + assert result.returncode == 0 + assert "LiteLLM Proxy CLI Version" in result.stdout + assert requests == [] + + def test_cli_source_is_ascii_only(): """Non-ASCII output (emoji, box-drawing chars) raises UnicodeEncodeError on legacy Windows consoles (cp1252), so the whole CLI package must stay ASCII-only.""" From ffae447649e173214a2081b4890410a11dfdf085 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 06:30:50 +0000 Subject: [PATCH 2/7] test(cli): type cost map bypass regression handlers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/test_get_model_cost_map.py | 4 +++- tests/test_litellm/proxy/client/cli/test_global_options.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py index d092b387259..6ab3f9a21af 100644 --- a/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/test_litellm/litellm_core_utils/test_get_model_cost_map.py @@ -722,7 +722,9 @@ def test_boot_load_that_fails_the_integrity_check_reports_the_backup_not_the_rej ("/some/venv/bin/python", 1), ], ) -def test_boot_load_skips_remote_fetch_for_cli_processes(monkeypatch, argv0, request_count): +def test_boot_load_skips_remote_fetch_for_cli_processes( + monkeypatch: pytest.MonkeyPatch, argv0: str, request_count: int +) -> None: monkeypatch.setattr(sys, "argv", [argv0, "--version"]) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) client, calls = _mock_client([httpx.Response(200, content=_real_map_bytes())], client_cls=httpx.Client) diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 02b1cdf4296..6de56573ffb 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -42,10 +42,10 @@ def test_lite_version_does_not_fetch_model_cost_map(): if lite_path is None: pytest.skip("lite executable is unavailable") - requests = [] + requests: list[str] = [] class _CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self): + def do_GET(self) -> None: requests.append(self.path) body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' self.send_response(200) @@ -54,7 +54,7 @@ def test_lite_version_does_not_fetch_model_cost_map(): self.end_headers() self.wfile.write(body) - def log_message(self, format, *args): + def log_message(self, format: str, *args: object) -> None: return server = ThreadingHTTPServer(("127.0.0.1", 0), _CostMapHandler) From 0c0a1dd76f31b56059508dd674fee1764c83828f Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 06:34:36 +0000 Subject: [PATCH 3/7] test(cli): avoid mutable request tracking Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/client/cli/test_global_options.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 6de56573ffb..9ed9f91ab9f 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -37,16 +37,17 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output -def test_lite_version_does_not_fetch_model_cost_map(): +def test_lite_version_does_not_fetch_model_cost_map(tmp_path: Path) -> None: lite_path = shutil.which("lite") if lite_path is None: pytest.skip("lite executable is unavailable") - requests: list[str] = [] + request_log: Path = tmp_path / "requests.log" class _CostMapHandler(BaseHTTPRequestHandler): def do_GET(self) -> None: - requests.append(self.path) + with request_log.open("a", encoding="utf-8") as log_file: + log_file.write(f"{self.path}\n") body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' self.send_response(200) self.send_header("Content-Type", "application/json") @@ -84,7 +85,8 @@ def test_lite_version_does_not_fetch_model_cost_map(): assert result.returncode == 0 assert "LiteLLM Proxy CLI Version" in result.stdout - assert requests == [] + request_count: int = request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 + assert request_count == 0 def test_cli_source_is_ascii_only(): From 1fc1aaabdaa3542cd2d410103ee01ebc94025961 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 16:00:35 +0000 Subject: [PATCH 4/7] test(cli): drop lite --version subprocess regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/client/cli/test_global_options.py | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/tests/test_litellm/proxy/client/cli/test_global_options.py b/tests/test_litellm/proxy/client/cli/test_global_options.py index 9ed9f91ab9f..b73d1acc6e3 100644 --- a/tests/test_litellm/proxy/client/cli/test_global_options.py +++ b/tests/test_litellm/proxy/client/cli/test_global_options.py @@ -1,10 +1,6 @@ # stdlib imports import json import os -import shutil -import subprocess -import threading -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from unittest.mock import Mock, patch @@ -37,58 +33,6 @@ def test_cli_version_flag(cli_runner): assert "LiteLLM Proxy Server Version: 1.2.3" in result.output -def test_lite_version_does_not_fetch_model_cost_map(tmp_path: Path) -> None: - lite_path = shutil.which("lite") - if lite_path is None: - pytest.skip("lite executable is unavailable") - - request_log: Path = tmp_path / "requests.log" - - class _CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - with request_log.open("a", encoding="utf-8") as log_file: - log_file.write(f"{self.path}\n") - body = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), _CostMapHandler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - try: - env = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} - source_root = str(Path(__file__).resolve().parents[5]) - env["PYTHONPATH"] = os.pathsep.join(filter(None, (source_root, env.get("PYTHONPATH")))) - env.update( - { - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": "http://127.0.0.1:9", - } - ) - result = subprocess.run( - [lite_path, "--version"], - capture_output=True, - text=True, - timeout=120, - env=env, - ) - finally: - server.shutdown() - server.server_close() - thread.join(timeout=10) - - assert result.returncode == 0 - assert "LiteLLM Proxy CLI Version" in result.stdout - request_count: int = request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 - assert request_count == 0 - - def test_cli_source_is_ascii_only(): """Non-ASCII output (emoji, box-drawing chars) raises UnicodeEncodeError on legacy Windows consoles (cp1252), so the whole CLI package must stay ASCII-only.""" From 2ece8735384e063b88ca91cf9d2f8f6d2e94b30a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 17:05:38 +0000 Subject: [PATCH 5/7] test(e2e): lite CLI never fetches the model cost map Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 2 + tests/e2e/other/test_cli_cost_map_e2e.py | 109 +++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 tests/e2e/other/test_cli_cost_map_e2e.py diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 814ebae2e0b..8ac4fc0762c 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -2,6 +2,8 @@ # PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. - {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} - {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} +- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} +- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} - {id: other.auth.llm_chat.missing_header_denied, module: other, tier: P0, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Chat with no Authorization header is 401/403"} - {id: other.auth.llm_chat.invalid_bearer_denied, module: other, tier: P0, area: auth, assertions: [invalid_bearer_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Bearer invalid_token on chat is 401/403"} - {id: other.auth.llm_chat.no_bearer_prefix_denied, module: other, tier: P0, area: auth, assertions: [no_bearer_prefix_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Token without Bearer scheme on chat is 401/403"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py new file mode 100644 index 00000000000..6022d5a0a4c --- /dev/null +++ b/tests/e2e/other/test_cli_cost_map_e2e.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +from collections.abc import Mapping +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final + +import pytest +from e2e_config import MASTER_KEY, PROXY_BASE_URL +from proxy_client import ProxyClient + +pytestmark = pytest.mark.e2e + + +def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, threading.Thread]: + class CostMapHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + with request_log.open("a", encoding="utf-8") as log_file: + log_file.write(f"{self.path}\n") + body: Final = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), CostMapHandler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread + + +def _run_lite( + args: tuple[str, ...], + server: ThreadingHTTPServer, + env: Mapping[str, str], +) -> subprocess.CompletedProcess[str]: + lite_path: Final = shutil.which("lite") + assert lite_path is not None, "the installed lite executable is required for e2e coverage" + try: + return subprocess.run( + [lite_path, *args], + capture_output=True, + text=True, + timeout=60, + env=env, + ) + finally: + server.shutdown() + server.server_close() + + +def _request_count(request_log: Path) -> int: + return request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 + + +class TestLiteCliCostMapFetch: + @pytest.mark.covers("other.cli.model_cost_map.version_skips_fetch") + def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: + request_log: Final = tmp_path / "requests.log" + server, thread = _start_cost_map_server(request_log) + source_root: Final = str(Path(__file__).resolve().parents[3]) + pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) + env: Final = { + **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, + "PYTHONPATH": pythonpath, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + } + try: + result: Final = _run_lite(("--version",), server, env) + finally: + thread.join(timeout=10) + + assert result.returncode == 0 + assert "LiteLLM Proxy CLI Version" in result.stdout + assert _request_count(request_log) == 0 + + @pytest.mark.covers("other.cli.model_cost_map.models_list_skips_fetch") + def test_lite_models_list_uses_proxy_not_cost_map(self, tmp_path: Path, proxy: ProxyClient) -> None: + model_names: Final = tuple(entry.model_name for entry in proxy.model_info()) + assert model_names + request_log: Final = tmp_path / "requests.log" + server, thread = _start_cost_map_server(request_log) + source_root: Final = str(Path(__file__).resolve().parents[3]) + pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) + env: Final = { + **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, + "PYTHONPATH": pythonpath, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + "LITELLM_PROXY_API_KEY": MASTER_KEY, + } + try: + result: Final = _run_lite(("models", "list"), server, env) + finally: + thread.join(timeout=10) + + assert result.returncode == 0 + assert result.stdout.strip() + assert any(model_name in result.stdout for model_name in model_names) + assert _request_count(request_log) == 0 From 6e71b90a888b91fe3aa281fa14da58f258b9f056 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 17:07:34 +0000 Subject: [PATCH 6/7] test(e2e): dedupe lite env setup Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 4 ++-- tests/e2e/other/test_cli_cost_map_e2e.py | 29 ++++++++++-------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index 8ac4fc0762c..a7ba0dbb8cb 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -2,8 +2,6 @@ # PROMOTION NOTE: the auth cluster (~14 cells) is a candidate to promote to its own module once stable. - {id: other.auth.master_key.valid_allows, module: other, tier: P0, area: auth, assertions: [valid_allows], source: "user_api_key_auth.py:1569-1588", rationale: "Master key authenticates; timing-safe compare"} - {id: other.auth.master_key.invalid_denied, module: other, tier: P0, area: auth, assertions: [invalid_denied], source: "user_api_key_auth.py:1580", rationale: "Invalid master key rejected"} -- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} -- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} - {id: other.auth.llm_chat.missing_header_denied, module: other, tier: P0, area: auth, assertions: [missing_header_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Chat with no Authorization header is 401/403"} - {id: other.auth.llm_chat.invalid_bearer_denied, module: other, tier: P0, area: auth, assertions: [invalid_bearer_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Bearer invalid_token on chat is 401/403"} - {id: other.auth.llm_chat.no_bearer_prefix_denied, module: other, tier: P0, area: auth, assertions: [no_bearer_prefix_denied], source: "vendor testing strategy §11.1 / LIT-4778", rationale: "Token without Bearer scheme on chat is 401/403"} @@ -50,3 +48,5 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} +- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} +- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py index 6022d5a0a4c..62cd3d6c3f5 100644 --- a/tests/e2e/other/test_cli_cost_map_e2e.py +++ b/tests/e2e/other/test_cli_cost_map_e2e.py @@ -37,6 +37,16 @@ def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, thre return server, thread +def _lite_env(server: ThreadingHTTPServer, api_key: str | None) -> dict[str, str]: + base_env: Final = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} + return { + **base_env, + "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", + "LITELLM_PROXY_URL": PROXY_BASE_URL, + **({"LITELLM_PROXY_API_KEY": api_key} if api_key is not None else {}), + } + + def _run_lite( args: tuple[str, ...], server: ThreadingHTTPServer, @@ -66,14 +76,7 @@ class TestLiteCliCostMapFetch: def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: request_log: Final = tmp_path / "requests.log" server, thread = _start_cost_map_server(request_log) - source_root: Final = str(Path(__file__).resolve().parents[3]) - pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) - env: Final = { - **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, - "PYTHONPATH": pythonpath, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - } + env: Final = _lite_env(server, None) try: result: Final = _run_lite(("--version",), server, env) finally: @@ -89,15 +92,7 @@ class TestLiteCliCostMapFetch: assert model_names request_log: Final = tmp_path / "requests.log" server, thread = _start_cost_map_server(request_log) - source_root: Final = str(Path(__file__).resolve().parents[3]) - pythonpath: Final = os.pathsep.join(filter(None, (source_root, os.environ.get("PYTHONPATH")))) - env: Final = { - **{key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"}, - "PYTHONPATH": pythonpath, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - "LITELLM_PROXY_API_KEY": MASTER_KEY, - } + env: Final = _lite_env(server, MASTER_KEY) try: result: Final = _run_lite(("models", "list"), server, env) finally: From 34d2d010c3f0f08ab9c13700c7004d98377309fa Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 9 Sep 2026 19:33:59 +0000 Subject: [PATCH 7/7] test(cli): drop lite e2e tests, the e2e runner does not install the package Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/coverage_registry/other.yaml | 2 - tests/e2e/other/test_cli_cost_map_e2e.py | 104 ----------------------- 2 files changed, 106 deletions(-) delete mode 100644 tests/e2e/other/test_cli_cost_map_e2e.py diff --git a/tests/e2e/coverage_registry/other.yaml b/tests/e2e/coverage_registry/other.yaml index a7ba0dbb8cb..814ebae2e0b 100644 --- a/tests/e2e/coverage_registry/other.yaml +++ b/tests/e2e/coverage_registry/other.yaml @@ -48,5 +48,3 @@ - {id: other.a2a.message_send.bridge_invokes, module: other, tier: P1, area: a2a, assertions: [bridge_invokes], source: "a2a_protocol/litellm_completion_bridge/handler.py", rationale: "A2A message/send routes through the completion bridge to a real provider and logs an asend_message spend row"} - {id: other.a2a.version.serves_pinned_0_3, module: other, tier: P1, area: a2a, assertions: [serves_pinned_0_3], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 0.3 returns the flat 0.3 message shape (parts on the result)"} - {id: other.a2a.version.serves_pinned_1_0, module: other, tier: P1, area: a2a, assertions: [serves_pinned_1_0], source: "agent_endpoints/a2a_endpoints.py _served_version", rationale: "An agent pinning 1.0 returns the nested 1.0 message shape (result.message with ROLE_AGENT)"} -- {id: other.cli.model_cost_map.version_skips_fetch, module: other, tier: P1, area: cli, assertions: [version_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite version command succeeds without requesting the remote model-cost map"} -- {id: other.cli.model_cost_map.models_list_skips_fetch, module: other, tier: P1, area: cli, assertions: [models_list_skips_fetch], source: "get_model_cost_map.py _is_cli_process / LIT-7385", fail_before_fix: proven, rationale: "The lite models list command uses the proxy without requesting the remote model-cost map"} diff --git a/tests/e2e/other/test_cli_cost_map_e2e.py b/tests/e2e/other/test_cli_cost_map_e2e.py deleted file mode 100644 index 62cd3d6c3f5..00000000000 --- a/tests/e2e/other/test_cli_cost_map_e2e.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -import os -import shutil -import subprocess -import threading -from collections.abc import Mapping -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from typing import Final - -import pytest -from e2e_config import MASTER_KEY, PROXY_BASE_URL -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -def _start_cost_map_server(request_log: Path) -> tuple[ThreadingHTTPServer, threading.Thread]: - class CostMapHandler(BaseHTTPRequestHandler): - def do_GET(self) -> None: - with request_log.open("a", encoding="utf-8") as log_file: - log_file.write(f"{self.path}\n") - body: Final = b'{"test-model": {"litellm_provider": "openai", "mode": "chat"}}' - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format: str, *args: object) -> None: - return - - server: Final = ThreadingHTTPServer(("127.0.0.1", 0), CostMapHandler) - thread: Final = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - return server, thread - - -def _lite_env(server: ThreadingHTTPServer, api_key: str | None) -> dict[str, str]: - base_env: Final = {key: value for key, value in os.environ.items() if key != "LITELLM_LOCAL_MODEL_COST_MAP"} - return { - **base_env, - "LITELLM_MODEL_COST_MAP_URL": f"http://127.0.0.1:{server.server_port}/map.json", - "LITELLM_PROXY_URL": PROXY_BASE_URL, - **({"LITELLM_PROXY_API_KEY": api_key} if api_key is not None else {}), - } - - -def _run_lite( - args: tuple[str, ...], - server: ThreadingHTTPServer, - env: Mapping[str, str], -) -> subprocess.CompletedProcess[str]: - lite_path: Final = shutil.which("lite") - assert lite_path is not None, "the installed lite executable is required for e2e coverage" - try: - return subprocess.run( - [lite_path, *args], - capture_output=True, - text=True, - timeout=60, - env=env, - ) - finally: - server.shutdown() - server.server_close() - - -def _request_count(request_log: Path) -> int: - return request_log.read_text(encoding="utf-8").count("\n") if request_log.exists() else 0 - - -class TestLiteCliCostMapFetch: - @pytest.mark.covers("other.cli.model_cost_map.version_skips_fetch") - def test_lite_version_makes_no_cost_map_request(self, tmp_path: Path) -> None: - request_log: Final = tmp_path / "requests.log" - server, thread = _start_cost_map_server(request_log) - env: Final = _lite_env(server, None) - try: - result: Final = _run_lite(("--version",), server, env) - finally: - thread.join(timeout=10) - - assert result.returncode == 0 - assert "LiteLLM Proxy CLI Version" in result.stdout - assert _request_count(request_log) == 0 - - @pytest.mark.covers("other.cli.model_cost_map.models_list_skips_fetch") - def test_lite_models_list_uses_proxy_not_cost_map(self, tmp_path: Path, proxy: ProxyClient) -> None: - model_names: Final = tuple(entry.model_name for entry in proxy.model_info()) - assert model_names - request_log: Final = tmp_path / "requests.log" - server, thread = _start_cost_map_server(request_log) - env: Final = _lite_env(server, MASTER_KEY) - try: - result: Final = _run_lite(("models", "list"), server, env) - finally: - thread.join(timeout=10) - - assert result.returncode == 0 - assert result.stdout.strip() - assert any(model_name in result.stdout for model_name in model_names) - assert _request_count(request_log) == 0