feat: add hosted_vllm Anthropic passthrough to skip OpenAI translation

Introduces `disable_anthropic_translation` (litellm_params or
`DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION` env var) for hosted_vllm
deployments. When enabled, /v1/messages requests are forwarded directly
to the vLLM Anthropic-compatible endpoint instead of being translated
through chat/completions.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -7,6 +7,7 @@
 
 import asyncio
 import contextvars
+import os
 from functools import partial
 from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union, cast
 
@@ -20,6 +21,9 @@ from litellm.llms.base_llm.anthropic_messages.transformation import (
 )
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.llms.hosted_vllm.messages.transformation import (
+    HostedVLLMAnthropicMessagesConfig,
+)
 from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -50,6 +54,23 @@ def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool:
     return custom_llm_provider in _RESPONSES_API_PROVIDERS
 
 
+def _should_skip_anthropic_translation(litellm_params: GenericLiteLLMParams) -> bool:
+    """Return True when Anthropic→OpenAI translation should be bypassed for hosted_vllm.
+
+    Checked in priority order:
+    1. Per-deployment ``disable_anthropic_translation`` in litellm_params
+    2. Global env var ``DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION``
+    """
+    param_flag = litellm_params.get("disable_anthropic_translation")
+    if param_flag is not None:
+        return bool(param_flag)
+    return os.environ.get("DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION", "").lower() in (
+        "true",
+        "1",
+        "yes",
+    )
+
+
 ####### ENVIRONMENT VARIABLES ###################
 # Initialize any necessary instances or variables here
 base_llm_http_handler = BaseLLMHTTPHandler()
@@ -410,7 +431,11 @@ def anthropic_messages_handler(
 
     anthropic_messages_provider_config: Optional[BaseAnthropicMessagesConfig] = None
 
-    if custom_llm_provider is not None and custom_llm_provider in [
+    if custom_llm_provider == "hosted_vllm" and _should_skip_anthropic_translation(
+        litellm_params
+    ):
+        anthropic_messages_provider_config = HostedVLLMAnthropicMessagesConfig()
+    elif custom_llm_provider is not None and custom_llm_provider in [
         provider.value for provider in LlmProviders
     ]:
         anthropic_messages_provider_config = (

litellm/llms/hosted_vllm/messages/transformation.py

@@ -0,0 +1,139 @@
+"""
+Pass Anthropic /v1/messages requests through to vLLM's native Anthropic-compatible endpoint.
+
+When `disable_anthropic_translation` is set on a `hosted_vllm` deployment (or the
+DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION env var is truthy), LiteLLM skips the
+Anthropic→OpenAI chat/completions translation and POSTs the original Anthropic payload
+directly to `{api_base}/v1/messages`.
+"""
+
+from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
+
+import httpx
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.anthropic_messages.transformation import (
+    BaseAnthropicMessagesConfig,
+)
+from litellm.types.llms.anthropic_messages.anthropic_response import (
+    AnthropicMessagesResponse,
+)
+from litellm.types.router import GenericLiteLLMParams
+
+
+class HostedVLLMAnthropicMessagesConfig(BaseAnthropicMessagesConfig):
+    """
+    Sends Anthropic-format /v1/messages requests directly to a vLLM instance.
+
+    No format translation is performed — the request body is forwarded as-is.
+    The response is parsed back into AnthropicMessagesResponse.
+    """
+
+    def get_supported_anthropic_messages_params(self, model: str) -> list:
+        return [
+            "messages",
+            "model",
+            "system",
+            "max_tokens",
+            "stop_sequences",
+            "temperature",
+            "top_p",
+            "top_k",
+            "tools",
+            "tool_choice",
+            "thinking",
+            "stream",
+        ]
+
+    def validate_anthropic_messages_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[Any],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> Tuple[dict, Optional[str]]:
+        if "content-type" not in headers:
+            headers["content-type"] = "application/json"
+        if api_key and "authorization" not in headers:
+            headers["authorization"] = f"Bearer {api_key}"
+        return headers, api_base
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        if not api_base:
+            raise ValueError(
+                "api_base is required for hosted_vllm Anthropic passthrough. "
+                "Set it via `api_base` in litellm_params or HOSTED_VLLM_API_BASE env var."
+            )
+        base = api_base.rstrip("/")
+        if base.endswith("/v1/messages"):
+            return base
+        # Standard api_base is http://host/v1 — strip the /v1 suffix so we can
+        # append the canonical /v1/messages path without doubling it.
+        if base.endswith("/v1"):
+            base = base[:-3]
+        return f"{base}/v1/messages"
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: List[Dict],
+        anthropic_messages_optional_request_params: Dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Dict:
+        max_tokens = anthropic_messages_optional_request_params.pop("max_tokens", None)
+        if max_tokens is None:
+            raise ValueError("max_tokens is required for Anthropic /v1/messages API")
+
+        return {
+            "model": model,
+            "messages": messages,
+            "max_tokens": max_tokens,
+            **anthropic_messages_optional_request_params,
+        }
+
+    def transform_anthropic_messages_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> AnthropicMessagesResponse:
+        try:
+            raw_response_json = raw_response.json()
+        except Exception:
+            raise ValueError(
+                f"Failed to parse vLLM Anthropic response: {raw_response.text}"
+            )
+        return AnthropicMessagesResponse(**raw_response_json)
+
+    def get_async_streaming_response_iterator(
+        self,
+        model: str,
+        httpx_response: httpx.Response,
+        request_body: dict,
+        litellm_logging_obj: LiteLLMLoggingObj,
+    ) -> AsyncIterator:
+        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+            BaseAnthropicMessagesStreamingIterator,
+        )
+
+        handler = BaseAnthropicMessagesStreamingIterator(
+            litellm_logging_obj=litellm_logging_obj,
+            request_body=request_body,
+        )
+        return handler.get_async_streaming_response_iterator(
+            httpx_response=httpx_response,
+            request_body=request_body,
+            litellm_logging_obj=litellm_logging_obj,
+        )

litellm/types/router.py

@@ -258,6 +258,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     milvus_db_name: Optional[str] = None
     milvus_partition_names: Optional[List[str]] = None
 
+    # hosted_vllm passthrough: skip Anthropic→OpenAI translation and send raw Anthropic payload
+    disable_anthropic_translation: Optional[bool] = None
+
     @model_validator(mode="before")
     @classmethod
     def preprocess_input_data(cls, data: Any) -> Any:
@@ -376,6 +379,9 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
     ## MOCK RESPONSES ##
     mock_response: Optional[Union[str, ModelResponse, Exception]]
 
+    ## TRANSLATION CONTROL ##
+    disable_anthropic_translation: Optional[bool]
+
     # routing params
     # use this for tag-based routing
     tags: Optional[List[str]]

tests/litellm/llms/hosted_vllm/test_anthropic_passthrough.py

@@ -0,0 +1,266 @@
+"""
+Unit tests for the hosted_vllm Anthropic passthrough feature.
+
+Verifies that:
+- The payload stays in Anthropic format when the flag is enabled (no translation).
+- The payload is translated to OpenAI format when the flag is disabled (default).
+- Both the env-var and per-deployment litellm_param paths work correctly.
+"""
+
+import os
+from typing import Any, Dict, List
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import httpx
+import pytest
+
+import litellm
+from litellm.llms.hosted_vllm.messages.transformation import (
+    HostedVLLMAnthropicMessagesConfig,
+)
+from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
+    _should_skip_anthropic_translation,
+)
+from litellm.types.router import GenericLiteLLMParams
+
+
+# ---------------------------------------------------------------------------
+# _should_skip_anthropic_translation
+# ---------------------------------------------------------------------------
+
+
+def _make_params(**kwargs) -> GenericLiteLLMParams:
+    return GenericLiteLLMParams(**kwargs)
+
+
+class TestShouldSkipTranslation:
+    def test_returns_false_by_default(self):
+        params = _make_params()
+        with patch.dict(os.environ, {}, clear=False):
+            os.environ.pop("DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION", None)
+            assert _should_skip_anthropic_translation(params) is False
+
+    def test_env_var_true_enables_skip(self):
+        params = _make_params()
+        with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "true"}):
+            assert _should_skip_anthropic_translation(params) is True
+
+    def test_env_var_1_enables_skip(self):
+        params = _make_params()
+        with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "1"}):
+            assert _should_skip_anthropic_translation(params) is True
+
+    def test_env_var_false_does_not_enable_skip(self):
+        params = _make_params()
+        with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "false"}):
+            assert _should_skip_anthropic_translation(params) is False
+
+    def test_litellm_param_true_overrides_env(self):
+        # param=True wins even when env var is absent
+        params = _make_params(disable_anthropic_translation=True)
+        with patch.dict(os.environ, {}, clear=False):
+            os.environ.pop("DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION", None)
+            assert _should_skip_anthropic_translation(params) is True
+
+    def test_litellm_param_false_overrides_env(self):
+        # param=False wins even when env var would enable skip
+        params = _make_params(disable_anthropic_translation=False)
+        with patch.dict(os.environ, {"DISABLE_HOSTED_VLLM_ANTHROPIC_TRANSLATION": "true"}):
+            assert _should_skip_anthropic_translation(params) is False
+
+
+# ---------------------------------------------------------------------------
+# HostedVLLMAnthropicMessagesConfig — URL construction
+# ---------------------------------------------------------------------------
+
+
+class TestHostedVLLMConfig:
+    def setup_method(self):
+        self.config = HostedVLLMAnthropicMessagesConfig()
+
+    def test_get_complete_url_strips_v1_then_appends_messages(self):
+        # Standard api_base ends with /v1 — must not double up
+        url = self.config.get_complete_url(
+            api_base="http://vllm-host/v1",
+            api_key="key",
+            model="qwen",
+            optional_params={},
+            litellm_params={},
+        )
+        assert url == "http://vllm-host/v1/messages"
+
+    def test_get_complete_url_bare_host_appends_v1_messages(self):
+        url = self.config.get_complete_url(
+            api_base="http://vllm-host",
+            api_key="key",
+            model="qwen",
+            optional_params={},
+            litellm_params={},
+        )
+        assert url == "http://vllm-host/v1/messages"
+
+    def test_get_complete_url_no_double_suffix(self):
+        url = self.config.get_complete_url(
+            api_base="http://vllm-host/v1/messages",
+            api_key="key",
+            model="qwen",
+            optional_params={},
+            litellm_params={},
+        )
+        assert url == "http://vllm-host/v1/messages"
+
+    def test_get_complete_url_raises_without_api_base(self):
+        with pytest.raises(ValueError, match="api_base is required"):
+            self.config.get_complete_url(
+                api_base=None,
+                api_key="key",
+                model="qwen",
+                optional_params={},
+                litellm_params={},
+            )
+
+    # -----------------------------------------------------------------------
+    # transform_anthropic_messages_request — payload stays in Anthropic format
+    # -----------------------------------------------------------------------
+
+    def test_request_payload_is_anthropic_format(self):
+        messages = [{"role": "user", "content": "Hello"}]
+        optional_params = {
+            "max_tokens": 100,
+            "system": "You are helpful.",
+            "temperature": 0.7,
+        }
+        payload = self.config.transform_anthropic_messages_request(
+            model="qwen36-27b",
+            messages=messages,
+            anthropic_messages_optional_request_params=optional_params,
+            litellm_params=GenericLiteLLMParams(),
+            headers={},
+        )
+
+        # Key Anthropic fields must be present
+        assert payload["messages"] == messages
+        assert payload["max_tokens"] == 100
+        assert payload["model"] == "qwen36-27b"
+        assert payload["system"] == "You are helpful."
+        assert payload["temperature"] == 0.7
+
+        # OpenAI-only fields must NOT appear
+        assert "n" not in payload
+        assert "frequency_penalty" not in payload
+        assert "presence_penalty" not in payload
+
+    def test_request_payload_missing_max_tokens_raises(self):
+        with pytest.raises(ValueError, match="max_tokens is required"):
+            self.config.transform_anthropic_messages_request(
+                model="qwen",
+                messages=[],
+                anthropic_messages_optional_request_params={},
+                litellm_params=GenericLiteLLMParams(),
+                headers={},
+            )
+
+    # -----------------------------------------------------------------------
+    # validate_anthropic_messages_environment
+    # -----------------------------------------------------------------------
+
+    def test_environment_sets_auth_and_content_type(self):
+        headers, returned_base = self.config.validate_anthropic_messages_environment(
+            headers={},
+            model="qwen",
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            api_key="my-key",
+            api_base="http://vllm-host/v1",
+        )
+        assert headers["authorization"] == "Bearer my-key"
+        assert headers["content-type"] == "application/json"
+        assert returned_base == "http://vllm-host/v1"
+
+    def test_environment_does_not_overwrite_existing_auth(self):
+        headers, _ = self.config.validate_anthropic_messages_environment(
+            headers={"authorization": "Bearer already-set"},
+            model="qwen",
+            messages=[],
+            optional_params={},
+            litellm_params={},
+            api_key="different-key",
+            api_base=None,
+        )
+        assert headers["authorization"] == "Bearer already-set"
+
+
+# ---------------------------------------------------------------------------
+# Integration: anthropic_messages_handler routes to HostedVLLMAnthropicMessagesConfig
+# ---------------------------------------------------------------------------
+
+
+class TestHandlerRouting:
+    """
+    Verify that anthropic_messages_handler uses HostedVLLMAnthropicMessagesConfig
+    when the flag is set, and falls back to the completions transformer otherwise.
+    """
+
+    @pytest.mark.asyncio
+    async def test_flag_enabled_uses_passthrough_config(self):
+        """When disable_anthropic_translation=True, the handler must call
+        base_llm_http_handler.anthropic_messages_handler (native path) instead of
+        LiteLLMMessagesToCompletionTransformationHandler."""
+        from litellm.llms.anthropic.experimental_pass_through.messages import handler as h
+
+        mock_response = MagicMock()
+        mock_response.id = "msg_test"
+
+        with (
+            patch.object(h.base_llm_http_handler, "anthropic_messages_handler", return_value=mock_response) as mock_native,
+            patch.object(
+                h.LiteLLMMessagesToCompletionTransformationHandler,
+                "anthropic_messages_handler",
+            ) as mock_translate,
+            patch("litellm.get_llm_provider", return_value=("qwen36-27b-fp8", "hosted_vllm", "key", "http://vllm/v1")),
+        ):
+            h.anthropic_messages_handler(
+                max_tokens=100,
+                messages=[{"role": "user", "content": "hi"}],
+                model="hosted_vllm/qwen36-27b-fp8",
+                api_key="key",
+                api_base="http://vllm/v1",
+                disable_anthropic_translation=True,
+            )
+
+        mock_native.assert_called_once()
+        mock_translate.assert_not_called()
+
+        # Confirm HostedVLLMAnthropicMessagesConfig was passed in
+        call_kwargs = mock_native.call_args.kwargs
+        assert isinstance(
+            call_kwargs.get("anthropic_messages_provider_config"),
+            HostedVLLMAnthropicMessagesConfig,
+        )
+
+    @pytest.mark.asyncio
+    async def test_flag_disabled_uses_translation(self):
+        """When disable_anthropic_translation is absent (default), the handler must
+        route to LiteLLMMessagesToCompletionTransformationHandler."""
+        from litellm.llms.anthropic.experimental_pass_through.messages import handler as h
+
+        mock_response = MagicMock()
+
+        with (
+            patch.object(
+                h.LiteLLMMessagesToCompletionTransformationHandler,
+                "anthropic_messages_handler",
+                return_value=mock_response,
+            ) as mock_translate,
+            patch("litellm.get_llm_provider", return_value=("qwen36-27b-fp8", "hosted_vllm", "key", "http://vllm/v1")),
+        ):
+            h.anthropic_messages_handler(
+                max_tokens=100,
+                messages=[{"role": "user", "content": "hi"}],
+                model="hosted_vllm/qwen36-27b-fp8",
+                api_key="key",
+                api_base="http://vllm/v1",
+            )
+
+        mock_translate.assert_called_once()