From 2f933c3f593081356ae7fcebe758e61429495d87 Mon Sep 17 00:00:00 2001 From: Val Date: Fri, 22 May 2026 14:58:43 +0300 Subject: [PATCH] feat(hosted_vllm): translate Anthropic prefill (`prefix: true`) to continue_final_message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the documented prefill-API coverage gap (#4881) for the hosted_vllm provider so /v1/messages requests routed to vLLM honor the Anthropic prefill semantic ("when the last message is role:assistant, the model continues from where that message leaves off"). Two minimal changes: * `litellm/llms/hosted_vllm/chat/transformation.py`: extend transform_request and async_transform_request to detect the unified prefill marker (`prefix: True` on the trailing assistant message) and inject vLLM's native continuation flags into extra_body — `continue_final_message: true` and `add_generation_prompt: false`. Idempotent (setdefault), no-op when the trigger is absent, never overrides caller-set values. * `litellm/llms/anthropic/experimental_pass_through/adapters/handler.py`: in `_prepare_completion_kwargs` — which is reached only when /v1/messages is routed to a non-Anthropic provider — auto-tag a trailing role:"assistant" message with `prefix: True` so the provider-side translator above fires. This makes the Anthropic /v1/messages prefill contract portable across providers. Tests added in test_hosted_vllm_chat_transformation.py: - prefill marker → continue_final_message injected - no marker → no-op - user-ending → no-op - existing extra_body preserved and merged - explicit caller-set continue_final_message:False respected Verified locally: 13/13 hosted_vllm transformation tests pass (8 pre-existing + 5 new). Refs: - #4881 — closed; defines the unified prefill API but only Anthropic provider implemented it - BerriAI/litellm#28580 — the bug report with empirical trace --- .../adapters/handler.py | 12 +++ .../llms/hosted_vllm/chat/transformation.py | 70 ++++++++++++++ .../test_hosted_vllm_chat_transformation.py | 91 +++++++++++++++++++ 3 files changed, 173 insertions(+) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py index 8ed6126d2eb..7a3edd5caf9 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/handler.py @@ -184,6 +184,18 @@ class LiteLLMMessagesToCompletionTransformationHandler: Logging as LiteLLMLoggingObject, ) + # Anthropic /v1/messages prefill contract: when the last message is + # role:"assistant" the model continues from there. The Anthropic + # provider config implements this via `prefix:true` (see #4881 + + # `litellm/llms/anthropic/chat/transformation.py::get_prefix_prompt`). + # When this adapter routes the call to a non-Anthropic provider (the + # whole reason this adapter exists), tag the trailing assistant + # message with prefix:true so provider configs that understand the + # unified marker — currently hosted_vllm — can translate it into + # their native continuation flag. Idempotent: setdefault only. + if messages and isinstance(messages[-1], dict) and messages[-1].get("role") == "assistant": + messages[-1].setdefault("prefix", True) + request_data = { "model": model, "messages": messages, diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 1824314865c..00e92ebea01 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -92,6 +92,76 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): params.extend(["reasoning_effort", "thinking"]) return params + @staticmethod + def _apply_prefill_translation( + messages: List[AllMessageValues], optional_params: dict + ) -> None: + """Translate LiteLLM's unified prefill marker to vLLM's native flags. + + When the last message is ``role:"assistant"`` and carries ``prefix: True`` + (the unified prefill marker introduced in #4881), vLLM's chat completions + endpoint expects ``continue_final_message: true`` and + ``add_generation_prompt: false`` to actually continue the assistant turn + rather than start a fresh one. Neither flag is part of the standard + OpenAI surface, so we inject them via ``extra_body`` — the openai SDK + forwards extra_body verbatim into the HTTP request body, and vLLM + honors them. + + Does not override caller-set values. No-op if the trigger pattern is + absent. Refs: #4881 (defines prefill API), upstream issue tracking the + hosted_vllm coverage gap. + """ + if not messages: + return + last_message = messages[-1] + if not isinstance(last_message, dict): + return + if last_message.get("role") != "assistant": + return + if not last_message.get("prefix"): + return + + extra_body = optional_params.get("extra_body") + if not isinstance(extra_body, dict): + extra_body = {} + extra_body.setdefault("continue_final_message", True) + extra_body.setdefault("add_generation_prompt", False) + optional_params["extra_body"] = extra_body + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + self._apply_prefill_translation(messages, optional_params) + return super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + async def async_transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + self._apply_prefill_translation(messages, optional_params) + return await super().async_transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + def map_openai_params( self, non_default_params: dict, diff --git a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 29ab3790609..c73d00111fe 100644 --- a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -319,3 +319,94 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema(): assert tools[0]["function"]["name"] == "search" assert tools[0]["function"]["description"] == "Search docs" assert tools[0]["function"]["parameters"] == input_schema + + +def test_hosted_vllm_prefill_translation_injects_continue_final_message(): + """Trailing assistant message with prefix:True translates into vLLM's + continue_final_message + add_generation_prompt flags inside extra_body.""" + config = HostedVLLMChatConfig() + messages = [ + {"role": "user", "content": "Count to three, comma-separated."}, + {"role": "assistant", "content": "1, 2,", "prefix": True}, + ] + transformed = config.transform_request( + model="hosted_vllm/qwen3.6-35b-a3b", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "extra_body" in transformed + assert transformed["extra_body"]["continue_final_message"] is True + assert transformed["extra_body"]["add_generation_prompt"] is False + + +def test_hosted_vllm_prefill_translation_no_prefix_marker_is_noop(): + """A trailing assistant message WITHOUT prefix:True must not inject + continuation flags — the caller did not opt in.""" + config = HostedVLLMChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + ] + transformed = config.transform_request( + model="hosted_vllm/qwen3.6-35b-a3b", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "extra_body" not in transformed or transformed.get("extra_body", {}) == {} + + +def test_hosted_vllm_prefill_translation_user_ending_is_noop(): + """Requests ending in a user message must be unaffected.""" + config = HostedVLLMChatConfig() + messages = [{"role": "user", "content": "Say hello in one word."}] + transformed = config.transform_request( + model="hosted_vllm/qwen3.6-35b-a3b", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "extra_body" not in transformed or transformed.get("extra_body", {}) == {} + + +def test_hosted_vllm_prefill_translation_preserves_existing_extra_body(): + """If the caller already set extra_body keys, prefill translation must + merge — not overwrite.""" + config = HostedVLLMChatConfig() + messages = [ + {"role": "user", "content": "Continue."}, + {"role": "assistant", "content": "X", "prefix": True}, + ] + transformed = config.transform_request( + model="hosted_vllm/qwen3.6-35b-a3b", + messages=messages, + optional_params={"extra_body": {"guided_choice": ["yes", "no"]}}, + litellm_params={}, + headers={}, + ) + eb = transformed["extra_body"] + assert eb["continue_final_message"] is True + assert eb["add_generation_prompt"] is False + assert eb["guided_choice"] == ["yes", "no"] + + +def test_hosted_vllm_prefill_translation_respects_explicit_caller_value(): + """If the caller already pinned continue_final_message=False, the helper + must not override their explicit choice (setdefault semantics).""" + config = HostedVLLMChatConfig() + messages = [ + {"role": "user", "content": "Continue."}, + {"role": "assistant", "content": "X", "prefix": True}, + ] + transformed = config.transform_request( + model="hosted_vllm/qwen3.6-35b-a3b", + messages=messages, + optional_params={"extra_body": {"continue_final_message": False}}, + litellm_params={}, + headers={}, + ) + assert transformed["extra_body"]["continue_final_message"] is False