fix(hosted_vllm): preserve vLLM choice fields

This commit is contained in:
Aaron Deng 2026-10-25 14:08:31 +08:00
parent 60fc121ce8
commit af2c199f91
2 changed files with 97 additions and 3 deletions

View file

@ -5,7 +5,9 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
import copy
import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
from typing import TYPE_CHECKING, Final, Literal, cast, overload
import httpx
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_image_mime_type_from_url,
@ -23,12 +25,52 @@ from litellm.types.llms.openai import (
ChatCompletionVideoObject,
ChatCompletionVideoUrlObject,
)
from litellm.types.utils import ModelResponse
from ....utils import remove_additional_properties, remove_strict_from_schema
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class HostedVLLMChatConfig(OpenAIGPTConfig):
def transform_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ModelResponse,
logging_obj: "LiteLLMLoggingObj",
request_data: dict,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:
response: Final = super().transform_response(
model=model,
raw_response=raw_response,
model_response=model_response,
logging_obj=logging_obj,
request_data=request_data,
messages=messages,
optional_params=optional_params,
litellm_params=litellm_params,
encoding=encoding,
api_key=api_key,
json_mode=json_mode,
)
for choice in response.choices:
provider_fields: Final = getattr(choice, "provider_specific_fields", None)
if isinstance(provider_fields, dict):
for field in ("stop_reason", "token_ids"):
if field in provider_fields:
choice[field] = provider_fields[field]
return response
def _convert_custom_tools_to_function_tools(self, tools: list[dict[str, object]]) -> list[dict[str, object]]:
"""
vLLM chat completions currently accepts only OpenAI function tools.

View file

@ -1,16 +1,68 @@
import copy
import json
from typing import Final
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
import httpx
import pytest
from litellm.constants import (
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
)
from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig
from litellm.types.utils import ModelResponse
def test_hosted_vllm_preserves_choice_fields():
config = HostedVLLMChatConfig()
response = config.transform_response(
model="hosted_vllm/qwen3",
raw_response=httpx.Response(
status_code=200,
json={
"id": "chatcmpl-vllm",
"model": "qwen3",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "done"},
"finish_reason": "stop",
"stop_reason": "</answer>",
"token_ids": [13, 42],
"routed_experts": [1, 3],
},
{
"index": 1,
"message": {"role": "assistant", "content": "also done"},
"finish_reason": "stop",
"routed_experts": [2, 4],
},
],
},
),
model_response=ModelResponse(),
logging_obj=MagicMock(),
request_data={},
messages=[{"role": "user", "content": "answer"}],
optional_params={},
litellm_params={},
encoding=None,
)
choice = response.model_dump()["choices"][0]
assert choice["stop_reason"] == "</answer>"
assert choice["token_ids"] == [13, 42]
assert "routed_experts" not in choice
assert choice["provider_specific_fields"] == {
"stop_reason": "</answer>",
"token_ids": [13, 42],
"routed_experts": [1, 3],
}
unrelated_choice = response.model_dump()["choices"][1]
assert "stop_reason" not in unrelated_choice
assert "token_ids" not in unrelated_choice
assert unrelated_choice["provider_specific_fields"] == {"routed_experts": [2, 4]}
def test_hosted_vllm_chat_transformation_file_url():