From 272038ceebf0d8a5b671fc3017a7d91ddf5c482e Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 14 Oct 2026 13:58:21 +0200 Subject: [PATCH] fix(hosted_vllm): send developer input items as system on native Responses --- .../hosted_vllm/responses/transformation.py | 43 +++++- .../responses/test_hosted_vllm_responses.py | 131 +++++++++++++++++- 2 files changed, 172 insertions(+), 2 deletions(-) diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 20d755a2273..fafc9f96210 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -6,14 +6,36 @@ so this config enables direct routing instead of falling back to the chat completions → responses conversion pipeline. """ -from typing import Final +from collections.abc import Mapping +from typing import Final, cast # noqa: TID251 # a role-swapped input item has no typed rebuild across item shapes + +from openai.types.responses import ResponseInputItemParam from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +def _developer_item_as_system(item: ResponseInputItemParam) -> Mapping[str, object] | ResponseInputItemParam: + if "role" not in item or item["role"] != "developer": + return item + return {**item, "role": "system"} + + +def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + """vLLM hands input roles straight to the served model's chat template, and templates such as + Kimi K3's only know system/user/assistant/tool, so `developer` goes out as `system`, the same + mapping the chat path applies through `translate_developer_role_to_system_role`. + """ + if isinstance(input, str): + return input + return cast( # cast-ok: each item keeps its shape; only a developer role becomes system + "ResponseInputParam", [_developer_item_as_system(item) for item in input] + ) + + class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): """ Configuration for Hosted VLLM Responses API support. @@ -22,6 +44,9 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): but uses HOSTED_VLLM_API_BASE for the base URL and defaults to "fake-api-key" when no API key is provided (vLLM does not require authentication by default). + + Input `developer` items are sent as `system`, because vLLM passes roles to the + model's chat template and many open-weight templates don't know `developer`. """ @property @@ -68,6 +93,22 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return f"{api_base}/v1/responses" + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: overrides the base class signature + ) -> dict: # mutable-ok: overrides the base class signature + return super().transform_responses_api_request( + model=model, + input=_developer_items_as_system(self._validate_input_param(input)), + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index e32f876a6b9..af229aa516c 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -8,12 +8,16 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para """ import json +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest - +import respx +from openai.types.responses import EasyInputMessage import litellm +from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.hosted_vllm.responses.transformation import ( HostedVLLMResponsesAPIConfig, ) @@ -183,3 +187,128 @@ def test_hosted_vllm_validate_environment_custom_api_key(): ) assert headers.get("Authorization") == "Bearer my-custom-key" + + +VLLM_API_BASE: Final = "https://vllm.invalid/v1" +CODEX_INSTRUCTIONS: Final = "You are Codex, a coding agent. Follow the repository's AGENTS.md." + + +def _sent_input(input: object) -> object: + request: Final = HostedVLLMResponsesAPIConfig().transform_responses_api_request( + model="kimi-k3", + input=input, + response_api_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + return request["input"] + + +@pytest.mark.parametrize( + ("input", "expected"), + [ + pytest.param( + [ + { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": CODEX_INSTRUCTIONS}], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "fix the failing test"}], + }, + ], + [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": CODEX_INSTRUCTIONS}]}, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "fix the failing test"}], + }, + ], + id="codex-leading-developer-item", + ), + pytest.param( + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "developer", "content": "Answer in French from now on."}, + {"role": "user", "content": "how are you?"}, + ], + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "system", "content": "Answer in French from now on."}, + {"role": "user", "content": "how are you?"}, + ], + id="mid-conversation-developer-item", + ), + pytest.param( + [ + EasyInputMessage(role="developer", content=CODEX_INSTRUCTIONS, type="message"), + EasyInputMessage(role="user", content="hi", type="message"), + ], + [ + {"role": "system", "content": CODEX_INSTRUCTIONS, "type": "message"}, + {"role": "user", "content": "hi", "type": "message"}, + ], + id="pydantic-developer-item", + ), + ], +) +def test_hosted_vllm_responses_sends_developer_items_as_system(input: object, expected: object) -> None: + assert _sent_input(input) == expected + + +def test_hosted_vllm_responses_keeps_everything_but_the_developer_role() -> None: + input: Final = [ + {"type": "message", "role": "developer", "id": "msg_dev_1", "status": "completed", "content": "Be terse."}, + {"type": "message", "role": "system", "content": "You are helpful."}, + {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, + {"type": "function_call_output", "call_id": "call_1", "output": "21C"}, + ] + assert _sent_input(input) == [ + {"type": "message", "role": "system", "id": "msg_dev_1", "status": "completed", "content": "Be terse."}, + *input[1:], + ] + + +def test_hosted_vllm_responses_passes_string_input_and_instructions_through() -> None: + request: Final = HostedVLLMResponsesAPIConfig().transform_responses_api_request( + model="kimi-k3", + input="hello", + response_api_optional_request_params={"instructions": CODEX_INSTRUCTIONS}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert request["input"] == "hello" + assert request["instructions"] == CODEX_INSTRUCTIONS + + +@pytest.fixture +def vllm_responses_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + return respx_mock.post(f"{VLLM_API_BASE}/responses").mock( + return_value=httpx.Response(200, json=_make_mock_responses_api_response()) + ) + + +@pytest.mark.asyncio +async def test_hosted_vllm_aresponses_sends_codex_developer_item_as_system_on_the_wire( + vllm_responses_route: respx.Route, +) -> None: + response: Final = await litellm.aresponses( + model="hosted_vllm/kimi-k3", + input=[ + {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": CODEX_INSTRUCTIONS}]}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "fix the failing test"}]}, + ], + api_base=VLLM_API_BASE, + ) + body: Final = json.loads(vllm_responses_route.calls.last.request.content) + assert [item["role"] for item in body["input"]] == ["system", "user"] + assert body["input"][0]["content"] == [{"type": "input_text", "text": CODEX_INSTRUCTIONS}] + assert response.output[0].role == "assistant" # type: ignore[union-attr]