diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..59138b23627 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -11,9 +11,34 @@ from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + ResponseInputParam, # pyright: ignore[reportAttributeAccessIssue,reportUnknownVariableType] # upstream openai export +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +HOSTED_VLLM_RESPONSES_SUPPORTED_PARAMS: Final = ( + "chat_template_kwargs", + "top_k", + "repetition_penalty", + "presence_penalty", + "frequency_penalty", + "min_p", + "best_of", + "stop_token_ids", + "spaces_between_special_tokens", + "prompt_logprobs", + "skip_special_tokens", + "include_stop_str_in_output", + "guided_json", + "guided_regex", + "guided_choice", + "guided_grammar", + "guided_decoding_backend", + "guided_whitespace_pattern", + "priority", +) + class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): """ @@ -29,6 +54,44 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.HOSTED_VLLM + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors base signature + params: Final[list[str]] = list( # mutable-ok: accumulator list + super().get_supported_openai_params(model) # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType] # base returns untyped list + ) + params.extend(HOSTED_VLLM_RESPONSES_SUPPORTED_PARAMS) + return params + + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, # pyright: ignore[reportUnknownParameterType] # mirrors OpenAIResponsesAPIConfig signature + response_api_optional_request_params: dict[str, object], # mutable-ok: mirrors base signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, object], # mutable-ok: mirrors base signature + ) -> dict[str, object]: # mutable-ok: mirrors base signature + request: Final[dict[str, object]] = dict( # mutable-ok: payload dict + super().transform_responses_api_request( # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType] # base returns untyped dict + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + ) + + for param in HOSTED_VLLM_RESPONSES_SUPPORTED_PARAMS: + if ( + param in response_api_optional_request_params + and response_api_optional_request_params[param] is not None + ): + request[param] = response_api_optional_request_params[param] + else: + val: object = getattr(litellm_params, param, None) + if val is not None: + request[param] = val + + return request + def validate_environment( self, headers: dict, diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index e32f876a6b9..a83ad492bdd 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -183,3 +183,119 @@ def test_hosted_vllm_validate_environment_custom_api_key(): ) assert headers.get("Authorization") == "Bearer my-custom-key" + + +def test_hosted_vllm_responses_supported_params(): + """Test get_supported_openai_params() includes vLLM provider-specific params.""" + config = HostedVLLMResponsesAPIConfig() + supported = config.get_supported_openai_params(model="Qwen/Qwen3-8B") + + assert "chat_template_kwargs" in supported + assert "top_k" in supported + assert "repetition_penalty" in supported + assert "presence_penalty" in supported + assert "frequency_penalty" in supported + assert "min_p" in supported + + +def test_hosted_vllm_responses_forwards_provider_specific_params(): + """ + Regression test for Issue #48362: + hosted_vllm Responses API must not drop provider-specific params + (chat_template_kwargs, top_k, repetition_penalty, presence_penalty) + that chat completions forwards. + """ + mock_client = _make_mock_http_client(_make_mock_responses_api_response("Test output")) + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.get_httpx_client", + return_value=mock_client, + ): + response = litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input="hi", + api_base="https://test-vllm.example.com/v1", + api_key="test-key", + chat_template_kwargs={"enable_thinking": False}, + top_k=20, + repetition_penalty=1.0, + presence_penalty=1.5, + ) + + assert response is not None + assert mock_client.post.call_count == 1 + call_json = mock_client.post.call_args[1]["json"] + assert call_json["model"] == "Qwen/Qwen3-8B" + assert call_json["input"] == "hi" + assert call_json["chat_template_kwargs"] == {"enable_thinking": False} + assert call_json["top_k"] == 20 + assert call_json["repetition_penalty"] == 1.0 + assert call_json["presence_penalty"] == 1.5 + + +def test_hosted_vllm_responses_transform_request_from_litellm_params(): + """Test transform_responses_api_request forwards provider params from litellm_params.""" + config = HostedVLLMResponsesAPIConfig() + litellm_params = GenericLiteLLMParams( + chat_template_kwargs={"enable_thinking": False}, + top_k=50, + min_p=0.05, + ) + + request = config.transform_responses_api_request( + model="Qwen/Qwen3-8B", + input="test input", + response_api_optional_request_params={}, + litellm_params=litellm_params, + headers={}, + ) + + assert request["chat_template_kwargs"] == {"enable_thinking": False} + assert request["top_k"] == 50 + assert request["min_p"] == 0.05 + + +def test_hosted_vllm_responses_transform_request_from_optional_request_params(): + """Test transform_responses_api_request forwards provider params from response_api_optional_request_params.""" + config = HostedVLLMResponsesAPIConfig() + + request = config.transform_responses_api_request( + model="Qwen/Qwen3-8B", + input="test input", + response_api_optional_request_params={ + "chat_template_kwargs": {"enable_thinking": False}, + "top_k": 30, + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request["chat_template_kwargs"] == {"enable_thinking": False} + assert request["top_k"] == 30 + + +def test_hosted_vllm_responses_provider_params_merge_with_request_extra_body(): + """ + A request-level extra_body must not drop the forwarded provider params: + unrelated keys from both reach vLLM, and extra_body wins on a shared key. + """ + mock_client = _make_mock_http_client(_make_mock_responses_api_response("Test output")) + + with patch( + "litellm.llms.custom_httpx.llm_http_handler.get_httpx_client", + return_value=mock_client, + ): + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input="hi", + api_base="https://test-vllm.example.com/v1", + api_key="test-key", + chat_template_kwargs={"enable_thinking": False}, + top_k=20, + extra_body={"min_p": 0.1, "top_k": 5}, + ) + + call_json = mock_client.post.call_args[1]["json"] + assert call_json["chat_template_kwargs"] == {"enable_thinking": False} + assert call_json["min_p"] == 0.1 + assert call_json["top_k"] == 5