diff --git a/litellm/constants.py b/litellm/constants.py index 8cc51d0ab5e..e458144474b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,6 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION", NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) +PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index edb3779a159..2340c6ba133 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -2,11 +2,12 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter -import litellm -from litellm.constants import NON_LLM_CONNECTION_TIMEOUT +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import get_httpx_client from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base @@ -30,12 +31,20 @@ class VLLMError(BaseLLMException): ) +_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + def _auth_headers(api_key: str | None) -> Mapping[str, str]: - if api_key is None: + if api_key is None or api_key == PLACEHOLDER_API_KEY: return {} return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} +def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) else None + + class VLLMModelInfo(BaseLLMModelInfo): def validate_environment( self, @@ -89,9 +98,33 @@ class HostedVLLMModelInfo(VLLMModelInfo): return api_key or get_secret_str("HOSTED_VLLM_API_KEY") def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None) + + def discover_models( + self, litellm_params: Mapping[str, object] | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + params: Final = litellm_params or {} + ssl_verify: Final = params.get("ssl_verify") + return self._list_models( + api_key=_str_param(params, "api_key"), + api_base=_str_param(params, "api_base"), + extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}), + ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None, + ) + + def _list_models( + self, + *, + api_key: str | None, + api_base: str | None, + extra_headers: Mapping[str, str], + ssl_verify: bool | str | None, + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models") - response: Final = litellm.module_level_client.get( - url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT + response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get( + url=url, + headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))}, + timeout=NON_LLM_CONNECTION_TIMEOUT, ) response.raise_for_status() return [model["id"] for model in response.json()["data"]] diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 5722bf7347e..1ddfc5bd805 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -284,12 +284,14 @@ _KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) def _prefix_wildcard_model(model: str, wildcard_provider_prefix: str, provider: str) -> str: if model.startswith(f"{wildcard_provider_prefix}/"): return model - # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b"). + # `get_provider_models` can return provider-prefixed ids (e.g. "ollama/gemma3:1b"). # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish # multiple instances), replace that existing provider prefix instead of stacking # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". - # Only strip the leading segment when it is a known provider, so ids whose first - # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. + # Only strip the leading segment when it is a known provider that prefixes this + # provider's ids: either the provider's own name or, for a provider with a static + # catalog, any provider. Ids a server reports with an org or another provider's + # name first (e.g. "meta-llama/Llama-3-8B" or "openai/gpt-oss-20b" on vLLM) keep it. leading, sep, model_suffix = model.partition("/") if sep and leading in _KNOWN_PROVIDERS and (_has_static_catalog(provider) or leading == provider): return f"{wildcard_provider_prefix}/{model_suffix}" @@ -548,30 +550,27 @@ def _get_wildcard_models( llm_router: Router | None = None, team_id: str | None = None, ) -> list[str]: - models_to_remove: Final = set() + wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model)) routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) all_wildcard_models: Final = [] - for model in unique_models: - if _check_wildcard_routing(model=model): - if return_wildcard_routes: - all_wildcard_models.append(model) + for model in wildcard_routes: + if return_wildcard_routes: + all_wildcard_models.append(model) - models_to_remove.add(model) - - model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - if model_list: - for router_model in model_list: - all_wildcard_models.extend( - get_known_models_from_wildcard( - wildcard_model=model, - litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - routes_to_model=routes_to_model, - ) + model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None + if model_list: + for router_model in model_list: + all_wildcard_models.extend( + get_known_models_from_wildcard( + wildcard_model=model, + litellm_params=LiteLLM_Params(**router_model["litellm_params"]), + routes_to_model=routes_to_model, ) - else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) + ) + else: + all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) - for model in models_to_remove: + for model in frozenset(wildcard_routes): unique_models.remove(model) return all_wildcard_models diff --git a/litellm/utils.py b/litellm/utils.py index 48c50969058..50c3d17290e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -28,7 +28,7 @@ import textwrap import threading import time import traceback -from contextlib import nullcontext +from contextlib import nullcontext, suppress from dataclasses import dataclass, field from functools import lru_cache, wraps from importlib import resources @@ -79,6 +79,7 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS, PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) @@ -8386,10 +8387,11 @@ class AvailableModelsCache(InMemoryCache): custom_llm_provider: str, litellm_params: LiteLLM_Params | None, available_models: list[str], + ttl: int | None = None, ): """Set cached model info""" cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - self.set_cache(cache_key, copy.deepcopy(available_models)) + self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl) # Global cache instance @@ -8455,6 +8457,10 @@ def _get_valid_models_from_provider_api( return models except Exception as e: verbose_logger.warning("Error getting valid models: %s", e) + with suppress(ValueError): + _model_cache.set_cached_model_info( + custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + ) return [] diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 1607447f7bf..f5a7de1b2ed 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -291,13 +291,12 @@ async def test_ssl_context_with_shared_session(monkeypatch: pytest.MonkeyPatch): await shared_session.close() -def test_get_ssl_configuration(): +def test_get_ssl_configuration(monkeypatch): """Test that get_ssl_configuration() returns a proper SSL context with certifi CA bundle when no environment variables are set.""" - from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + from litellm.llms.custom_httpx import http_handler - # Clear cache to ensure ssl.create_default_context is called - _ssl_context_cache.clear() + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) with patch.dict(os.environ, clear=True): with patch("ssl.create_default_context") as mock_create_context: diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index 9c07729da86..e5369e44068 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -72,3 +72,33 @@ async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(r assert sent_headers["authorization"] == "Bearer deployment-key" assert sent_headers["x-api-key"] == "deployment-key" + + +@pytest.mark.asyncio +async def test_router_passthrough_to_a_hosted_vllm_deployment_without_an_api_key_sends_no_auth_headers( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vllm-without-key", + "litellm_params": {"model": "hosted_vllm/my-deployment", "api_base": _API_BASE}, + } + ] + ) + + response: Final = await router.allm_passthrough_route( + model="vllm-without-key", + method="POST", + endpoint="v1/chat/completions", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + ) + + assert response.status_code == 200 + sent_headers: Final = route.calls[0].request.headers + assert "authorization" not in sent_headers + assert "x-api-key" not in sent_headers diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 623eed1ba7b..8415f593213 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1,11 +1,22 @@ -from collections.abc import Generator, Mapping +import datetime +import ipaddress +import json +import ssl +import threading +from collections.abc import Generator, Iterator, Mapping from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Final from unittest.mock import patch import httpx import pytest import respx +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID def test_get_team_models_for_all_models_and_team_only_models(): @@ -529,6 +540,136 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_ assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT +@respx.mock +def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get( + "http://vllm-behind-gateway.test:8000/v1/models", + headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"}, + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params( + model="hosted_vllm/*", + api_base="http://vllm-behind-gateway.test:8000/v1", + api_key="vllm-test-key", + extra_headers={"X-Tenant": "team-a"}, + ), + ) + + assert listed == ["hosted_vllm/qwen2.5"] + + +def _self_signed_cert(directory: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + now: Final = datetime.datetime.now(datetime.UTC) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False + ) + .sign(key, hashes.SHA256()) + ) + cert_path: Final = directory / "cert.pem" + key_path: Final = directory / "key.pem" + cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + return cert_path, key_path + + +@pytest.fixture +def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]: + cert_path, key_path = _self_signed_cert(tmp_path) + + class Handler(BaseHTTPRequestHandler): + timeout = 5 + + def do_GET(self) -> None: + body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: # matches BaseHTTPRequestHandler + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_path, key_path) + server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"https://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + server.server_close() + + +def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify( + monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str +) -> None: + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx import http_handler + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + + verified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base) + ) + unverified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False), + ) + + assert verified == [] + assert unverified == ["hosted_vllm/qwen2.5"] + + +@respx.mock +def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get("http://upstream-proxy.test:4000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "team-proxy/*", + LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"), + ) + + assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"] + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38287b25169..7511c7d8b7a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10581,6 +10581,35 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): assert _model_cache.get_cached_model_info("openai") is None +@respx.mock +def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None: + import time + + from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + from litellm.types.router import LiteLLM_Params + + models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( + side_effect=httpx.ReadTimeout("vLLM did not answer") + ) + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1") + failed_at: Final = time.time() + + def list_models() -> list[str]: + return get_valid_models( + check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params + ) + + assert list_models() == [] + assert list_models() == [] + assert models_route.call_count == 1 + + models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1) + + assert list_models() == ["qwen2.5"] + assert models_route.call_count == 2 + + def test_delta_tool_calls_sequential_indices(): """ Test that multiple tool calls without explicit indices receive sequential indices.