From b7cfd2d02623ff44141df51dadd3ac6e3d8589b8 Mon Sep 17 00:00:00 2001 From: Denis Khachyan Date: Sat, 23 May 2026 11:27:06 +0300 Subject: [PATCH] feat: add hosted_vllm Anthropic passthrough to skip OpenAI translation Introduces `disable_anthropic_translation` (litellm_params or `DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION` env var) for hosted_vllm deployments. When enabled, /v1/messages requests are forwarded directly to the vLLM Anthropic-compatible endpoint instead of being translated through chat/completions. Co-Authored-By: Claude Opus 4.7 --- .../messages/handler.py | 27 +- litellm/llms/hosted_vllm/messages/__init__.py | 0 .../hosted_vllm/messages/transformation.py | 139 +++++++++ litellm/types/router.py | 6 + tests/litellm/llms/hosted_vllm/__init__.py | 0 .../hosted_vllm/test_anthropic_passthrough.py | 266 ++++++++++++++++++ 6 files changed, 437 insertions(+), 1 deletion(-) create mode 100644 litellm/llms/hosted_vllm/messages/__init__.py create mode 100644 litellm/llms/hosted_vllm/messages/transformation.py create mode 100644 tests/litellm/llms/hosted_vllm/__init__.py create mode 100644 tests/litellm/llms/hosted_vllm/test_anthropic_passthrough.py diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 009ba6ef306..3ae30b05c8e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -7,6 +7,7 @@ import asyncio import contextvars +import os from functools import partial from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, cast @@ -20,6 +21,9 @@ from litellm.llms.base_llm.anthropic_messages.transformation import ( ) from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.hosted_vllm.messages.transformation import ( + HostedVLLMAnthropicMessagesConfig, +) from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -50,6 +54,23 @@ def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: return custom_llm_provider in _RESPONSES_API_PROVIDERS +def _should_skip_anthropic_translation(litellm_params: GenericLiteLLMParams) -> bool: + """Return True when Anthropic→OpenAI translation should be bypassed for hosted_vllm. + + Checked in priority order: + 1. Per-deployment ``disable_anthropic_translation`` in litellm_params + 2. Global env var ``DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION`` + """ + param_flag = litellm_params.get("disable_anthropic_translation") + if param_flag is not None: + return bool(param_flag) + return os.environ.get("DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION", "").lower() in ( + "true", + "1", + "yes", + ) + + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -410,7 +431,11 @@ def anthropic_messages_handler( anthropic_messages_provider_config: Optional[BaseAnthropicMessagesConfig] = None - if custom_llm_provider is not None and custom_llm_provider in [ + if custom_llm_provider == "hosted_vllm" and _should_skip_anthropic_translation( + litellm_params + ): + anthropic_messages_provider_config = HostedVLLMAnthropicMessagesConfig() + elif custom_llm_provider is not None and custom_llm_provider in [ provider.value for provider in LlmProviders ]: anthropic_messages_provider_config = ( diff --git a/litellm/llms/hosted_vllm/messages/__init__.py b/litellm/llms/hosted_vllm/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/hosted_vllm/messages/transformation.py b/litellm/llms/hosted_vllm/messages/transformation.py new file mode 100644 index 00000000000..5c382db7159 --- /dev/null +++ b/litellm/llms/hosted_vllm/messages/transformation.py @@ -0,0 +1,139 @@ +""" +Pass Anthropic /v1/messages requests through to vLLM's native Anthropic-compatible endpoint. + +When `disable_anthropic_translation` is set on a `hosted_vllm` deployment (or the +DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION env var is truthy), LiteLLM skips the +Anthropic→OpenAI chat/completions translation and POSTs the original Anthropic payload +directly to `{api_base}/v1/messages`. +""" + +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.router import GenericLiteLLMParams + + +class HostedVLLMAnthropicMessagesConfig(BaseAnthropicMessagesConfig): + """ + Sends Anthropic-format /v1/messages requests directly to a vLLM instance. + + No format translation is performed — the request body is forwarded as-is. + The response is parsed back into AnthropicMessagesResponse. + """ + + def get_supported_anthropic_messages_params(self, model: str) -> list: + return [ + "messages", + "model", + "system", + "max_tokens", + "stop_sequences", + "temperature", + "top_p", + "top_k", + "tools", + "tool_choice", + "thinking", + "stream", + ] + + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + if "content-type" not in headers: + headers["content-type"] = "application/json" + if api_key and "authorization" not in headers: + headers["authorization"] = f"Bearer {api_key}" + return headers, api_base + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if not api_base: + raise ValueError( + "api_base is required for hosted_vllm Anthropic passthrough. " + "Set it via `api_base` in litellm_params or HOSTED_VLLM_API_BASE env var." + ) + base = api_base.rstrip("/") + if base.endswith("/v1/messages"): + return base + # Standard api_base is http://host/v1 — strip the /v1 suffix so we can + # append the canonical /v1/messages path without doubling it. + if base.endswith("/v1"): + base = base[:-3] + return f"{base}/v1/messages" + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + max_tokens = anthropic_messages_optional_request_params.pop("max_tokens", None) + if max_tokens is None: + raise ValueError("max_tokens is required for Anthropic /v1/messages API") + + return { + "model": model, + "messages": messages, + "max_tokens": max_tokens, + **anthropic_messages_optional_request_params, + } + + def transform_anthropic_messages_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> AnthropicMessagesResponse: + try: + raw_response_json = raw_response.json() + except Exception: + raise ValueError( + f"Failed to parse vLLM Anthropic response: {raw_response.text}" + ) + return AnthropicMessagesResponse(**raw_response_json) + + def get_async_streaming_response_iterator( + self, + model: str, + httpx_response: httpx.Response, + request_body: dict, + litellm_logging_obj: LiteLLMLoggingObj, + ) -> AsyncIterator: + from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import ( + BaseAnthropicMessagesStreamingIterator, + ) + + handler = BaseAnthropicMessagesStreamingIterator( + litellm_logging_obj=litellm_logging_obj, + request_body=request_body, + ) + return handler.get_async_streaming_response_iterator( + httpx_response=httpx_response, + request_body=request_body, + litellm_logging_obj=litellm_logging_obj, + ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 6601f552b52..c918e6497fc 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -258,6 +258,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): milvus_db_name: Optional[str] = None milvus_partition_names: Optional[List[str]] = None + # hosted_vllm passthrough: skip Anthropic→OpenAI translation and send raw Anthropic payload + disable_anthropic_translation: Optional[bool] = None + @model_validator(mode="before") @classmethod def preprocess_input_data(cls, data: Any) -> Any: @@ -376,6 +379,9 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): ## MOCK RESPONSES ## mock_response: Optional[Union[str, ModelResponse, Exception]] + ## TRANSLATION CONTROL ## + disable_anthropic_translation: Optional[bool] + # routing params # use this for tag-based routing tags: Optional[List[str]] diff --git a/tests/litellm/llms/hosted_vllm/__init__.py b/tests/litellm/llms/hosted_vllm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/litellm/llms/hosted_vllm/test_anthropic_passthrough.py b/tests/litellm/llms/hosted_vllm/test_anthropic_passthrough.py new file mode 100644 index 00000000000..9e3e14d3031 --- /dev/null +++ b/tests/litellm/llms/hosted_vllm/test_anthropic_passthrough.py @@ -0,0 +1,266 @@ +""" +Unit tests for the hosted_vllm Anthropic passthrough feature. + +Verifies that: +- The payload stays in Anthropic format when the flag is enabled (no translation). +- The payload is translated to OpenAI format when the flag is disabled (default). +- Both the env-var and per-deployment litellm_param paths work correctly. +""" + +import os +from typing import Any, Dict, List +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +import litellm +from litellm.llms.hosted_vllm.messages.transformation import ( + HostedVLLMAnthropicMessagesConfig, +) +from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + _should_skip_anthropic_translation, +) +from litellm.types.router import GenericLiteLLMParams + + +# --------------------------------------------------------------------------- +# _should_skip_anthropic_translation +# --------------------------------------------------------------------------- + + +def _make_params(**kwargs) -> GenericLiteLLMParams: + return GenericLiteLLMParams(**kwargs) + + +class TestShouldSkipTranslation: + def test_returns_false_by_default(self): + params = _make_params() + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION", None) + assert _should_skip_anthropic_translation(params) is False + + def test_env_var_true_enables_skip(self): + params = _make_params() + with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "true"}): + assert _should_skip_anthropic_translation(params) is True + + def test_env_var_1_enables_skip(self): + params = _make_params() + with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "1"}): + assert _should_skip_anthropic_translation(params) is True + + def test_env_var_false_does_not_enable_skip(self): + params = _make_params() + with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "false"}): + assert _should_skip_anthropic_translation(params) is False + + def test_litellm_param_true_overrides_env(self): + # param=True wins even when env var is absent + params = _make_params(disable_anthropic_translation=True) + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION", None) + assert _should_skip_anthropic_translation(params) is True + + def test_litellm_param_false_overrides_env(self): + # param=False wins even when env var would enable skip + params = _make_params(disable_anthropic_translation=False) + with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "true"}): + assert _should_skip_anthropic_translation(params) is False + + +# --------------------------------------------------------------------------- +# HostedVLLMAnthropicMessagesConfig — URL construction +# --------------------------------------------------------------------------- + + +class TestHostedVLLMConfig: + def setup_method(self): + self.config = HostedVLLMAnthropicMessagesConfig() + + def test_get_complete_url_strips_v1_then_appends_messages(self): + # Standard api_base ends with /v1 — must not double up + url = self.config.get_complete_url( + api_base="http://vllm-host/v1", + api_key="key", + model="qwen", + optional_params={}, + litellm_params={}, + ) + assert url == "http://vllm-host/v1/messages" + + def test_get_complete_url_bare_host_appends_v1_messages(self): + url = self.config.get_complete_url( + api_base="http://vllm-host", + api_key="key", + model="qwen", + optional_params={}, + litellm_params={}, + ) + assert url == "http://vllm-host/v1/messages" + + def test_get_complete_url_no_double_suffix(self): + url = self.config.get_complete_url( + api_base="http://vllm-host/v1/messages", + api_key="key", + model="qwen", + optional_params={}, + litellm_params={}, + ) + assert url == "http://vllm-host/v1/messages" + + def test_get_complete_url_raises_without_api_base(self): + with pytest.raises(ValueError, match="api_base is required"): + self.config.get_complete_url( + api_base=None, + api_key="key", + model="qwen", + optional_params={}, + litellm_params={}, + ) + + # ----------------------------------------------------------------------- + # transform_anthropic_messages_request — payload stays in Anthropic format + # ----------------------------------------------------------------------- + + def test_request_payload_is_anthropic_format(self): + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "max_tokens": 100, + "system": "You are helpful.", + "temperature": 0.7, + } + payload = self.config.transform_anthropic_messages_request( + model="qwen36-27b", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + # Key Anthropic fields must be present + assert payload["messages"] == messages + assert payload["max_tokens"] == 100 + assert payload["model"] == "qwen36-27b" + assert payload["system"] == "You are helpful." + assert payload["temperature"] == 0.7 + + # OpenAI-only fields must NOT appear + assert "n" not in payload + assert "frequency_penalty" not in payload + assert "presence_penalty" not in payload + + def test_request_payload_missing_max_tokens_raises(self): + with pytest.raises(ValueError, match="max_tokens is required"): + self.config.transform_anthropic_messages_request( + model="qwen", + messages=[], + anthropic_messages_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + # ----------------------------------------------------------------------- + # validate_anthropic_messages_environment + # ----------------------------------------------------------------------- + + def test_environment_sets_auth_and_content_type(self): + headers, returned_base = self.config.validate_anthropic_messages_environment( + headers={}, + model="qwen", + messages=[], + optional_params={}, + litellm_params={}, + api_key="my-key", + api_base="http://vllm-host/v1", + ) + assert headers["authorization"] == "Bearer my-key" + assert headers["content-type"] == "application/json" + assert returned_base == "http://vllm-host/v1" + + def test_environment_does_not_overwrite_existing_auth(self): + headers, _ = self.config.validate_anthropic_messages_environment( + headers={"authorization": "Bearer already-set"}, + model="qwen", + messages=[], + optional_params={}, + litellm_params={}, + api_key="different-key", + api_base=None, + ) + assert headers["authorization"] == "Bearer already-set" + + +# --------------------------------------------------------------------------- +# Integration: anthropic_messages_handler routes to HostedVLLMAnthropicMessagesConfig +# --------------------------------------------------------------------------- + + +class TestHandlerRouting: + """ + Verify that anthropic_messages_handler uses HostedVLLMAnthropicMessagesConfig + when the flag is set, and falls back to the completions transformer otherwise. + """ + + @pytest.mark.asyncio + async def test_flag_enabled_uses_passthrough_config(self): + """When disable_anthropic_translation=True, the handler must call + base_llm_http_handler.anthropic_messages_handler (native path) instead of + LiteLLMMessagesToCompletionTransformationHandler.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler as h + + mock_response = MagicMock() + mock_response.id = "msg_test" + + with ( + patch.object(h.base_llm_http_handler, "anthropic_messages_handler", return_value=mock_response) as mock_native, + patch.object( + h.LiteLLMMessagesToCompletionTransformationHandler, + "anthropic_messages_handler", + ) as mock_translate, + patch("litellm.get_llm_provider", return_value=("qwen36-27b-fp8", "hosted_vllm", "key", "http://vllm/v1")), + ): + h.anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "hi"}], + model="hosted_vllm/qwen36-27b-fp8", + api_key="key", + api_base="http://vllm/v1", + disable_anthropic_translation=True, + ) + + mock_native.assert_called_once() + mock_translate.assert_not_called() + + # Confirm HostedVLLMAnthropicMessagesConfig was passed in + call_kwargs = mock_native.call_args.kwargs + assert isinstance( + call_kwargs.get("anthropic_messages_provider_config"), + HostedVLLMAnthropicMessagesConfig, + ) + + @pytest.mark.asyncio + async def test_flag_disabled_uses_translation(self): + """When disable_anthropic_translation is absent (default), the handler must + route to LiteLLMMessagesToCompletionTransformationHandler.""" + from litellm.llms.anthropic.experimental_pass_through.messages import handler as h + + mock_response = MagicMock() + + with ( + patch.object( + h.LiteLLMMessagesToCompletionTransformationHandler, + "anthropic_messages_handler", + return_value=mock_response, + ) as mock_translate, + patch("litellm.get_llm_provider", return_value=("qwen36-27b-fp8", "hosted_vllm", "key", "http://vllm/v1")), + ): + h.anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "hi"}], + model="hosted_vllm/qwen36-27b-fp8", + api_key="key", + api_base="http://vllm/v1", + ) + + mock_translate.assert_called_once()