fix(vllm): skip the placeholder key, honor deployment TLS and headers, and retry failed listings after a minute

This commit is contained in:
mateo-berri 2026-10-28 15:07:35 -07:00
parent 1009886009
commit 4e109a03e9
8 changed files with 272 additions and 34 deletions

View file

@ -610,6 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION",
NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
"NON_LLM_CONNECTION_TIMEOUT", 15
) # timeout for adjacent services (e.g. jwt auth)
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024

View file

@ -2,11 +2,12 @@ from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.utils import add_path_to_api_base
@ -30,12 +31,20 @@ class VLLMError(BaseLLMException):
)
_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str])
def _auth_headers(api_key: str | None) -> Mapping[str, str]:
if api_key is None:
if api_key is None or api_key == PLACEHOLDER_API_KEY:
return {}
return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"}
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
value: Final = litellm_params.get(key)
return value if isinstance(value, str) else None
class VLLMModelInfo(BaseLLMModelInfo):
def validate_environment(
self,
@ -89,9 +98,33 @@ class HostedVLLMModelInfo(VLLMModelInfo):
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None)
def discover_models(
self, litellm_params: Mapping[str, object] | None = None
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
params: Final = litellm_params or {}
ssl_verify: Final = params.get("ssl_verify")
return self._list_models(
api_key=_str_param(params, "api_key"),
api_base=_str_param(params, "api_base"),
extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}),
ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None,
)
def _list_models(
self,
*,
api_key: str | None,
api_base: str | None,
extra_headers: Mapping[str, str],
ssl_verify: bool | str | None,
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models")
response: Final = litellm.module_level_client.get(
url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT
response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get(
url=url,
headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))},
timeout=NON_LLM_CONNECTION_TIMEOUT,
)
response.raise_for_status()
return [model["id"] for model in response.json()["data"]]

View file

@ -284,12 +284,14 @@ _KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders)
def _prefix_wildcard_model(model: str, wildcard_provider_prefix: str, provider: str) -> str:
if model.startswith(f"{wildcard_provider_prefix}/"):
return model
# `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b").
# `get_provider_models` can return provider-prefixed ids (e.g. "ollama/gemma3:1b").
# When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish
# multiple instances), replace that existing provider prefix instead of stacking
# both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b".
# Only strip the leading segment when it is a known provider, so ids whose first
# segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it.
# Only strip the leading segment when it is a known provider that prefixes this
# provider's ids: either the provider's own name or, for a provider with a static
# catalog, any provider. Ids a server reports with an org or another provider's
# name first (e.g. "meta-llama/Llama-3-8B" or "openai/gpt-oss-20b" on vLLM) keep it.
leading, sep, model_suffix = model.partition("/")
if sep and leading in _KNOWN_PROVIDERS and (_has_static_catalog(provider) or leading == provider):
return f"{wildcard_provider_prefix}/{model_suffix}"
@ -548,30 +550,27 @@ def _get_wildcard_models(
llm_router: Router | None = None,
team_id: str | None = None,
) -> list[str]:
models_to_remove: Final = set()
wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model))
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
all_wildcard_models: Final = []
for model in unique_models:
if _check_wildcard_routing(model=model):
if return_wildcard_routes:
all_wildcard_models.append(model)
for model in wildcard_routes:
if return_wildcard_routes:
all_wildcard_models.append(model)
models_to_remove.add(model)
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
if model_list:
for router_model in model_list:
all_wildcard_models.extend(
get_known_models_from_wildcard(
wildcard_model=model,
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
routes_to_model=routes_to_model,
)
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
if model_list:
for router_model in model_list:
all_wildcard_models.extend(
get_known_models_from_wildcard(
wildcard_model=model,
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
routes_to_model=routes_to_model,
)
else:
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
)
else:
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
for model in models_to_remove:
for model in frozenset(wildcard_routes):
unique_models.remove(model)
return all_wildcard_models

View file

@ -28,7 +28,7 @@ import textwrap
import threading
import time
import traceback
from contextlib import nullcontext
from contextlib import nullcontext, suppress
from dataclasses import dataclass, field
from functools import lru_cache, wraps
from importlib import resources
@ -79,6 +79,7 @@ from litellm.constants import (
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
NON_INFERENCE_CALL_TYPES,
OPENAI_EMBEDDING_PARAMS,
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS,
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
)
@ -8386,10 +8387,11 @@ class AvailableModelsCache(InMemoryCache):
custom_llm_provider: str,
litellm_params: LiteLLM_Params | None,
available_models: list[str],
ttl: int | None = None,
):
"""Set cached model info"""
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
self.set_cache(cache_key, copy.deepcopy(available_models))
self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl)
# Global cache instance
@ -8455,6 +8457,10 @@ def _get_valid_models_from_provider_api(
return models
except Exception as e:
verbose_logger.warning("Error getting valid models: %s", e)
with suppress(ValueError):
_model_cache.set_cached_model_info(
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
)
return []

View file

@ -291,13 +291,12 @@ async def test_ssl_context_with_shared_session(monkeypatch: pytest.MonkeyPatch):
await shared_session.close()
def test_get_ssl_configuration():
def test_get_ssl_configuration(monkeypatch):
"""Test that get_ssl_configuration() returns a proper SSL context with certifi CA bundle
when no environment variables are set."""
from litellm.llms.custom_httpx.http_handler import _ssl_context_cache
from litellm.llms.custom_httpx import http_handler
# Clear cache to ensure ssl.create_default_context is called
_ssl_context_cache.clear()
monkeypatch.setattr(http_handler, "_ssl_context_cache", {})
with patch.dict(os.environ, clear=True):
with patch("ssl.create_default_context") as mock_create_context:

View file

@ -72,3 +72,33 @@ async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(r
assert sent_headers["authorization"] == "Bearer deployment-key"
assert sent_headers["x-api-key"] == "deployment-key"
@pytest.mark.asyncio
async def test_router_passthrough_to_a_hosted_vllm_deployment_without_an_api_key_sends_no_auth_headers(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
return_value=httpx.Response(200, json={"ok": True})
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "vllm-without-key",
"litellm_params": {"model": "hosted_vllm/my-deployment", "api_base": _API_BASE},
}
]
)
response: Final = await router.allm_passthrough_route(
model="vllm-without-key",
method="POST",
endpoint="v1/chat/completions",
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
)
assert response.status_code == 200
sent_headers: Final = route.calls[0].request.headers
assert "authorization" not in sent_headers
assert "x-api-key" not in sent_headers

View file

@ -1,11 +1,22 @@
from collections.abc import Generator, Mapping
import datetime
import ipaddress
import json
import ssl
import threading
from collections.abc import Generator, Iterator, Mapping
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Final
from unittest.mock import patch
import httpx
import pytest
import respx
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
def test_get_team_models_for_all_models_and_team_only_models():
@ -529,6 +540,136 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
@respx.mock
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
respx.get(
"http://vllm-behind-gateway.test:8000/v1/models",
headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"},
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
listed: Final = get_known_models_from_wildcard(
"hosted_vllm/*",
LiteLLM_Params(
model="hosted_vllm/*",
api_base="http://vllm-behind-gateway.test:8000/v1",
api_key="vllm-test-key",
extra_headers={"X-Tenant": "team-a"},
),
)
assert listed == ["hosted_vllm/qwen2.5"]
def _self_signed_cert(directory: Path) -> tuple[Path, Path]:
key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
now: Final = datetime.datetime.now(datetime.UTC)
cert: Final = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(now - datetime.timedelta(days=1))
.not_valid_after(now + datetime.timedelta(days=1))
.add_extension(
x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False
)
.sign(key, hashes.SHA256())
)
cert_path: Final = directory / "cert.pem"
key_path: Final = directory / "key.pem"
cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
key_path.write_bytes(
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
)
return cert_path, key_path
@pytest.fixture
def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]:
cert_path, key_path = _self_signed_cert(tmp_path)
class Handler(BaseHTTPRequestHandler):
timeout = 5
def do_GET(self) -> None:
body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: object) -> None: # matches BaseHTTPRequestHandler
pass
server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain(cert_path, key_path)
server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False)
threading.Thread(target=server.serve_forever, daemon=True).start()
try:
yield f"https://127.0.0.1:{server.server_address[1]}/v1"
finally:
server.shutdown()
server.server_close()
def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify(
monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str
) -> None:
import litellm
from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.llms.custom_httpx import http_handler
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.setattr(litellm, "ssl_verify", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
monkeypatch.setattr(http_handler, "_ssl_context_cache", {})
monkeypatch.delenv("SSL_VERIFY", raising=False)
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
verified: Final = get_known_models_from_wildcard(
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base)
)
unverified: Final = get_known_models_from_wildcard(
"hosted_vllm/*",
LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False),
)
assert verified == []
assert unverified == ["hosted_vllm/qwen2.5"]
@respx.mock
def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
respx.get("http://upstream-proxy.test:4000/v1/models").mock(
return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]})
)
listed: Final = get_known_models_from_wildcard(
"team-proxy/*",
LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"),
)
assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"]
def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
"""Regression test for #30358.

View file

@ -10581,6 +10581,35 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
assert _model_cache.get_cached_model_info("openai") is None
@respx.mock
def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None:
import time
from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
from litellm.types.router import LiteLLM_Params
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
side_effect=httpx.ReadTimeout("vLLM did not answer")
)
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1")
failed_at: Final = time.time()
def list_models() -> list[str]:
return get_valid_models(
check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params
)
assert list_models() == []
assert list_models() == []
assert models_route.call_count == 1
models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1)
assert list_models() == ["qwen2.5"]
assert models_route.call_count == 2
def test_delta_tool_calls_sequential_indices():
"""
Test that multiple tool calls without explicit indices receive sequential indices.