scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

fix: add HostedVLLMChatCompletionStreamingHandler to map reasoning→reasoning_content

Browse source 
vLLM/SGLang returns delta.reasoning for thinking models, but LiteLLM
expects delta.reasoning_content. Added a custom streaming handler
(like Groq's GroqChatCompletionStreamingHandler) that maps the field
before the chunk is processed.

This is the second part of the fix - the first part added reasoning
field check to is_chunk_non_empty() in streaming_handler.py.

Fixes: https://github.com/BerriAI/litellm/issues/20246
This commit is contained in:
ec-jt 2026-03-02 14:39:32 +00:00
parent da4f5cd9ed
commit 6f86e26e66
1 changed files with 39 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
litellm/llms/hosted_vllm/chat/transformation.py
View file

@ -2,7 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
from typing import Any, AsyncIterator, Coroutine, Iterator, List, Literal, Optional, Tuple, Union, cast, overload
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_get_image_mime_type_from_url,
@ -15,9 +15,10 @@ from litellm.types.llms.openai import (
ChatCompletionVideoObject,
ChatCompletionVideoUrlObject,
)
from litellm.types.utils import ModelResponse, ModelResponseStream
from ....utils import _remove_additional_properties, _remove_strict_from_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
from ...openai.chat.gpt_transformation import OpenAIGPTConfig, OpenAIChatCompletionStreamingHandler
class HostedVLLMChatConfig(OpenAIGPTConfig):
@ -26,6 +27,18 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
params.extend(["reasoning_effort", "thinking"])
return params
def get_model_response_iterator(
self,
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
sync_stream: bool,
json_mode: Optional[bool] = False,
) -> Any:
return HostedVLLMChatCompletionStreamingHandler(
streaming_response=streaming_response,
sync_stream=sync_stream,
json_mode=json_mode,
)
def map_openai_params(
self,
non_default_params: dict,
@ -180,3 +193,27 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
return super()._transform_messages(
messages, model, is_async=cast(Literal[False], False)
)
class HostedVLLMChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
"""
Streaming handler for hosted_vllm that maps vLLM's 'reasoning' field
to LiteLLM's 'reasoning_content' field.
vLLM/SGLang returns delta.reasoning for thinking models, but LiteLLM
expects delta.reasoning_content. This handler performs the mapping
before the chunk is processed by the base OpenAI handler.
See: https://github.com/BerriAI/litellm/issues/20246
"""
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
# Map vLLM's 'reasoning' field to LiteLLM's 'reasoning_content' field
# vLLM returns delta.reasoning, but LiteLLM expects delta.reasoning_content
choices = chunk.get("choices", [])
for choice in choices:
delta = choice.get("delta", {})
if "reasoning" in delta:
delta["reasoning_content"] = delta.pop("reasoning")
return super().chunk_parser(chunk)