From af2c199f91fd6cfabd76cf77db45890054804562 Mon Sep 17 00:00:00 2001 From: Aaron Deng <84045863+aarongdeng@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:08:31 +0800 Subject: [PATCH] fix(hosted_vllm): preserve vLLM choice fields --- .../llms/hosted_vllm/chat/transformation.py | 44 ++++++++++++++- .../test_hosted_vllm_chat_transformation.py | 56 ++++++++++++++++++- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 3386cd452da..e59bbb4dff5 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -5,7 +5,9 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` import copy import json from collections.abc import Coroutine -from typing import Final, Literal, cast, overload +from typing import TYPE_CHECKING, Final, Literal, cast, overload + +import httpx from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_image_mime_type_from_url, @@ -23,12 +25,52 @@ from litellm.types.llms.openai import ( ChatCompletionVideoObject, ChatCompletionVideoUrlObject, ) +from litellm.types.utils import ModelResponse from ....utils import remove_additional_properties, remove_strict_from_schema from ...openai.chat.gpt_transformation import OpenAIGPTConfig +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + class HostedVLLMChatConfig(OpenAIGPTConfig): + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict, + messages: list[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + response: Final = super().transform_response( + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + for choice in response.choices: + provider_fields: Final = getattr(choice, "provider_specific_fields", None) + if isinstance(provider_fields, dict): + for field in ("stop_reason", "token_ids"): + if field in provider_fields: + choice[field] = provider_fields[field] + return response + def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]: """ vLLM chat completions currently accepts only OpenAI function tools. diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index fc5ae3631ff..b1731487273 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,16 +1,68 @@ import copy import json from typing import Final -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +import httpx import pytest - from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, ) from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig +from litellm.types.utils import ModelResponse + + +def test_hosted_vllm_preserves_choice_fields(): + config = HostedVLLMChatConfig() + response = config.transform_response( + model="hosted_vllm/qwen3", + raw_response=httpx.Response( + status_code=200, + json={ + "id": "chatcmpl-vllm", + "model": "qwen3", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "done"}, + "finish_reason": "stop", + "stop_reason": "", + "token_ids": [13, 42], + "routed_experts": [1, 3], + }, + { + "index": 1, + "message": {"role": "assistant", "content": "also done"}, + "finish_reason": "stop", + "routed_experts": [2, 4], + }, + ], + }, + ), + model_response=ModelResponse(), + logging_obj=MagicMock(), + request_data={}, + messages=[{"role": "user", "content": "answer"}], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + choice = response.model_dump()["choices"][0] + assert choice["stop_reason"] == "" + assert choice["token_ids"] == [13, 42] + assert "routed_experts" not in choice + assert choice["provider_specific_fields"] == { + "stop_reason": "", + "token_ids": [13, 42], + "routed_experts": [1, 3], + } + unrelated_choice = response.model_dump()["choices"][1] + assert "stop_reason" not in unrelated_choice + assert "token_ids" not in unrelated_choice + assert unrelated_choice["provider_specific_fields"] == {"routed_experts": [2, 4]} def test_hosted_vllm_chat_transformation_file_url():