mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(vllm): skip the placeholder key, honor deployment TLS and headers, and retry failed listings after a minute
This commit is contained in:
parent
1009886009
commit
4e109a03e9
8 changed files with 272 additions and 34 deletions
|
|
@ -610,6 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION",
|
|||
NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
|
||||
"NON_LLM_CONNECTION_TIMEOUT", 15
|
||||
) # timeout for adjacent services (e.g. jwt auth)
|
||||
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60
|
||||
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
|
||||
|
|
|
|||
|
|
@ -2,11 +2,12 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.utils import add_path_to_api_base
|
||||
|
|
@ -30,12 +31,20 @@ class VLLMError(BaseLLMException):
|
|||
)
|
||||
|
||||
|
||||
_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
|
||||
|
||||
def _auth_headers(api_key: str | None) -> Mapping[str, str]:
|
||||
if api_key is None:
|
||||
if api_key is None or api_key == PLACEHOLDER_API_KEY:
|
||||
return {}
|
||||
return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"}
|
||||
|
||||
|
||||
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
|
||||
value: Final = litellm_params.get(key)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
class VLLMModelInfo(BaseLLMModelInfo):
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
@ -89,9 +98,33 @@ class HostedVLLMModelInfo(VLLMModelInfo):
|
|||
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None)
|
||||
|
||||
def discover_models(
|
||||
self, litellm_params: Mapping[str, object] | None = None
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
params: Final = litellm_params or {}
|
||||
ssl_verify: Final = params.get("ssl_verify")
|
||||
return self._list_models(
|
||||
api_key=_str_param(params, "api_key"),
|
||||
api_base=_str_param(params, "api_base"),
|
||||
extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}),
|
||||
ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None,
|
||||
)
|
||||
|
||||
def _list_models(
|
||||
self,
|
||||
*,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
extra_headers: Mapping[str, str],
|
||||
ssl_verify: bool | str | None,
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models")
|
||||
response: Final = litellm.module_level_client.get(
|
||||
url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT
|
||||
response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get(
|
||||
url=url,
|
||||
headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))},
|
||||
timeout=NON_LLM_CONNECTION_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return [model["id"] for model in response.json()["data"]]
|
||||
|
|
|
|||
|
|
@ -284,12 +284,14 @@ _KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders)
|
|||
def _prefix_wildcard_model(model: str, wildcard_provider_prefix: str, provider: str) -> str:
|
||||
if model.startswith(f"{wildcard_provider_prefix}/"):
|
||||
return model
|
||||
# `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b").
|
||||
# `get_provider_models` can return provider-prefixed ids (e.g. "ollama/gemma3:1b").
|
||||
# When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish
|
||||
# multiple instances), replace that existing provider prefix instead of stacking
|
||||
# both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b".
|
||||
# Only strip the leading segment when it is a known provider, so ids whose first
|
||||
# segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it.
|
||||
# Only strip the leading segment when it is a known provider that prefixes this
|
||||
# provider's ids: either the provider's own name or, for a provider with a static
|
||||
# catalog, any provider. Ids a server reports with an org or another provider's
|
||||
# name first (e.g. "meta-llama/Llama-3-8B" or "openai/gpt-oss-20b" on vLLM) keep it.
|
||||
leading, sep, model_suffix = model.partition("/")
|
||||
if sep and leading in _KNOWN_PROVIDERS and (_has_static_catalog(provider) or leading == provider):
|
||||
return f"{wildcard_provider_prefix}/{model_suffix}"
|
||||
|
|
@ -548,30 +550,27 @@ def _get_wildcard_models(
|
|||
llm_router: Router | None = None,
|
||||
team_id: str | None = None,
|
||||
) -> list[str]:
|
||||
models_to_remove: Final = set()
|
||||
wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model))
|
||||
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
|
||||
all_wildcard_models: Final = []
|
||||
for model in unique_models:
|
||||
if _check_wildcard_routing(model=model):
|
||||
if return_wildcard_routes:
|
||||
all_wildcard_models.append(model)
|
||||
for model in wildcard_routes:
|
||||
if return_wildcard_routes:
|
||||
all_wildcard_models.append(model)
|
||||
|
||||
models_to_remove.add(model)
|
||||
|
||||
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
|
||||
if model_list:
|
||||
for router_model in model_list:
|
||||
all_wildcard_models.extend(
|
||||
get_known_models_from_wildcard(
|
||||
wildcard_model=model,
|
||||
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
|
||||
routes_to_model=routes_to_model,
|
||||
)
|
||||
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
|
||||
if model_list:
|
||||
for router_model in model_list:
|
||||
all_wildcard_models.extend(
|
||||
get_known_models_from_wildcard(
|
||||
wildcard_model=model,
|
||||
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
|
||||
routes_to_model=routes_to_model,
|
||||
)
|
||||
else:
|
||||
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
|
||||
)
|
||||
else:
|
||||
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
|
||||
|
||||
for model in models_to_remove:
|
||||
for model in frozenset(wildcard_routes):
|
||||
unique_models.remove(model)
|
||||
|
||||
return all_wildcard_models
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import textwrap
|
|||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from contextlib import nullcontext
|
||||
from contextlib import nullcontext, suppress
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache, wraps
|
||||
from importlib import resources
|
||||
|
|
@ -79,6 +79,7 @@ from litellm.constants import (
|
|||
MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
OPENAI_EMBEDDING_PARAMS,
|
||||
PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS,
|
||||
PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO,
|
||||
TOOL_CHOICE_OBJECT_TOKEN_COUNT,
|
||||
)
|
||||
|
|
@ -8386,10 +8387,11 @@ class AvailableModelsCache(InMemoryCache):
|
|||
custom_llm_provider: str,
|
||||
litellm_params: LiteLLM_Params | None,
|
||||
available_models: list[str],
|
||||
ttl: int | None = None,
|
||||
):
|
||||
"""Set cached model info"""
|
||||
cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params)
|
||||
self.set_cache(cache_key, copy.deepcopy(available_models))
|
||||
self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl)
|
||||
|
||||
|
||||
# Global cache instance
|
||||
|
|
@ -8455,6 +8457,10 @@ def _get_valid_models_from_provider_api(
|
|||
return models
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
with suppress(ValueError):
|
||||
_model_cache.set_cached_model_info(
|
||||
custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -291,13 +291,12 @@ async def test_ssl_context_with_shared_session(monkeypatch: pytest.MonkeyPatch):
|
|||
await shared_session.close()
|
||||
|
||||
|
||||
def test_get_ssl_configuration():
|
||||
def test_get_ssl_configuration(monkeypatch):
|
||||
"""Test that get_ssl_configuration() returns a proper SSL context with certifi CA bundle
|
||||
when no environment variables are set."""
|
||||
from litellm.llms.custom_httpx.http_handler import _ssl_context_cache
|
||||
from litellm.llms.custom_httpx import http_handler
|
||||
|
||||
# Clear cache to ensure ssl.create_default_context is called
|
||||
_ssl_context_cache.clear()
|
||||
monkeypatch.setattr(http_handler, "_ssl_context_cache", {})
|
||||
|
||||
with patch.dict(os.environ, clear=True):
|
||||
with patch("ssl.create_default_context") as mock_create_context:
|
||||
|
|
|
|||
|
|
@ -72,3 +72,33 @@ async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(r
|
|||
assert sent_headers["authorization"] == "Bearer deployment-key"
|
||||
assert sent_headers["x-api-key"] == "deployment-key"
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_passthrough_to_a_hosted_vllm_deployment_without_an_api_key_sends_no_auth_headers(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
||||
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json={"ok": True})
|
||||
)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vllm-without-key",
|
||||
"litellm_params": {"model": "hosted_vllm/my-deployment", "api_base": _API_BASE},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response: Final = await router.allm_passthrough_route(
|
||||
model="vllm-without-key",
|
||||
method="POST",
|
||||
endpoint="v1/chat/completions",
|
||||
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
sent_headers: Final = route.calls[0].request.headers
|
||||
assert "authorization" not in sent_headers
|
||||
assert "x-api-key" not in sent_headers
|
||||
|
|
|
|||
|
|
@ -1,11 +1,22 @@
|
|||
from collections.abc import Generator, Mapping
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import ssl
|
||||
import threading
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
|
||||
|
||||
def test_get_team_models_for_all_models_and_team_only_models():
|
||||
|
|
@ -529,6 +540,136 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_
|
|||
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
respx.get(
|
||||
"http://vllm-behind-gateway.test:8000/v1/models",
|
||||
headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"},
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(
|
||||
model="hosted_vllm/*",
|
||||
api_base="http://vllm-behind-gateway.test:8000/v1",
|
||||
api_key="vllm-test-key",
|
||||
extra_headers={"X-Tenant": "team-a"},
|
||||
),
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
|
||||
|
||||
def _self_signed_cert(directory: Path) -> tuple[Path, Path]:
|
||||
key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
|
||||
now: Final = datetime.datetime.now(datetime.UTC)
|
||||
cert: Final = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name)
|
||||
.issuer_name(name)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - datetime.timedelta(days=1))
|
||||
.not_valid_after(now + datetime.timedelta(days=1))
|
||||
.add_extension(
|
||||
x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False
|
||||
)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
cert_path: Final = directory / "cert.pem"
|
||||
key_path: Final = directory / "key.pem"
|
||||
cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
key_path.write_bytes(
|
||||
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
|
||||
)
|
||||
return cert_path, key_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]:
|
||||
cert_path, key_path = _self_signed_cert(tmp_path)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
timeout = 5
|
||||
|
||||
def do_GET(self) -> None:
|
||||
body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None: # matches BaseHTTPRequestHandler
|
||||
pass
|
||||
|
||||
server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(cert_path, key_path)
|
||||
server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
yield f"https://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify(
|
||||
monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.llms.custom_httpx import http_handler
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setattr(litellm, "ssl_verify", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
monkeypatch.setattr(http_handler, "_ssl_context_cache", {})
|
||||
monkeypatch.delenv("SSL_VERIFY", raising=False)
|
||||
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
|
||||
|
||||
verified: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base)
|
||||
)
|
||||
unverified: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False),
|
||||
)
|
||||
|
||||
assert verified == []
|
||||
assert unverified == ["hosted_vllm/qwen2.5"]
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
respx.get("http://upstream-proxy.test:4000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]})
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"team-proxy/*",
|
||||
LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"),
|
||||
)
|
||||
|
||||
assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"]
|
||||
|
||||
|
||||
def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
|
||||
"""Regression test for #30358.
|
||||
|
||||
|
|
|
|||
|
|
@ -10581,6 +10581,35 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
|
|||
assert _model_cache.get_cached_model_info("openai") is None
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import time
|
||||
|
||||
from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
|
||||
side_effect=httpx.ReadTimeout("vLLM did not answer")
|
||||
)
|
||||
litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1")
|
||||
failed_at: Final = time.time()
|
||||
|
||||
def list_models() -> list[str]:
|
||||
return get_valid_models(
|
||||
check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params
|
||||
)
|
||||
|
||||
assert list_models() == []
|
||||
assert list_models() == []
|
||||
assert models_route.call_count == 1
|
||||
|
||||
models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1)
|
||||
|
||||
assert list_models() == ["qwen2.5"]
|
||||
assert models_route.call_count == 2
|
||||
|
||||
|
||||
def test_delta_tool_calls_sequential_indices():
|
||||
"""
|
||||
Test that multiple tool calls without explicit indices receive sequential indices.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue