diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 05db1544a2b..c4f6a03a909 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -2,7 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload +from typing import Any, AsyncIterator, Coroutine, Iterator, List, Literal, Optional, Tuple, Union, cast, overload from litellm.litellm_core_utils.prompt_templates.common_utils import ( _get_image_mime_type_from_url, @@ -15,9 +15,10 @@ from litellm.types.llms.openai import ( ChatCompletionVideoObject, ChatCompletionVideoUrlObject, ) +from litellm.types.utils import ModelResponse, ModelResponseStream from ....utils import _remove_additional_properties, _remove_strict_from_schema -from ...openai.chat.gpt_transformation import OpenAIGPTConfig +from ...openai.chat.gpt_transformation import OpenAIGPTConfig, OpenAIChatCompletionStreamingHandler class HostedVLLMChatConfig(OpenAIGPTConfig): @@ -26,6 +27,18 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): params.extend(["reasoning_effort", "thinking"]) return params + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + return HostedVLLMChatCompletionStreamingHandler( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) + def map_openai_params( self, non_default_params: dict, @@ -180,3 +193,27 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): return super()._transform_messages( messages, model, is_async=cast(Literal[False], False) ) + + +class HostedVLLMChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler): + """ + Streaming handler for hosted_vllm that maps vLLM's 'reasoning' field + to LiteLLM's 'reasoning_content' field. + + vLLM/SGLang returns delta.reasoning for thinking models, but LiteLLM + expects delta.reasoning_content. This handler performs the mapping + before the chunk is processed by the base OpenAI handler. + + See: https://github.com/BerriAI/litellm/issues/20246 + """ + + def chunk_parser(self, chunk: dict) -> ModelResponseStream: + # Map vLLM's 'reasoning' field to LiteLLM's 'reasoning_content' field + # vLLM returns delta.reasoning, but LiteLLM expects delta.reasoning_content + choices = chunk.get("choices", []) + for choice in choices: + delta = choice.get("delta", {}) + if "reasoning" in delta: + delta["reasoning_content"] = delta.pop("reasoning") + + return super().chunk_parser(chunk)