fix(hosted_vllm): send developer input items as system on native Responses

litellm/llms/hosted_vllm/responses/transformation.py

@@ -6,14 +6,36 @@ so this config enables direct routing instead of falling back to
 the chat completions → responses conversion pipeline.
 """
 
-from typing import Final
+from collections.abc import Mapping
+from typing import Final, cast  # noqa: TID251  # a role-swapped input item has no typed rebuild across item shapes
+
+from openai.types.responses import ResponseInputItemParam
 
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import ResponseInputParam
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
 
+def _developer_item_as_system(item: ResponseInputItemParam) -> Mapping[str, object] | ResponseInputItemParam:
+    if "role" not in item or item["role"] != "developer":
+        return item
+    return {**item, "role": "system"}
+
+
+def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam:
+    """vLLM hands input roles straight to the served model's chat template, and templates such as
+    Kimi K3's only know system/user/assistant/tool, so `developer` goes out as `system`, the same
+    mapping the chat path applies through `translate_developer_role_to_system_role`.
+    """
+    if isinstance(input, str):
+        return input
+    return cast(  # cast-ok: each item keeps its shape; only a developer role becomes system
+        "ResponseInputParam", [_developer_item_as_system(item) for item in input]
+    )
+
+
 class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
     """
     Configuration for Hosted VLLM Responses API support.
@@ -22,6 +44,9 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
     but uses HOSTED_VLLM_API_BASE for the base URL and defaults
     to "fake-api-key" when no API key is provided (vLLM does not
     require authentication by default).
+
+    Input `developer` items are sent as `system`, because vLLM passes roles to the
+    model's chat template and many open-weight templates don't know `developer`.
     """
 
     @property
@@ -68,6 +93,22 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
         return f"{api_base}/v1/responses"
 
+    def transform_responses_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: dict,  # mutable-ok: overrides the base class signature
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,  # mutable-ok: overrides the base class signature
+    ) -> dict:  # mutable-ok: overrides the base class signature
+        return super().transform_responses_api_request(
+            model=model,
+            input=_developer_items_as_system(self._validate_input_param(input)),
+            response_api_optional_request_params=response_api_optional_request_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     def supports_native_websocket(self) -> bool:
         """Hosted vLLM does not support native WebSocket for Responses API"""
         return False

tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py

@@ -8,12 +8,16 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para
 """
 
 import json
+from typing import Final
 from unittest.mock import MagicMock, patch
 
+import httpx
 import pytest
-
+import respx
+from openai.types.responses import EasyInputMessage
 
 import litellm
+from litellm.caching.llm_caching_handler import LLMClientCache
 from litellm.llms.hosted_vllm.responses.transformation import (
     HostedVLLMResponsesAPIConfig,
 )
@@ -183,3 +187,128 @@ def test_hosted_vllm_validate_environment_custom_api_key():
     )
 
     assert headers.get("Authorization") == "Bearer my-custom-key"
+
+
+VLLM_API_BASE: Final = "https://vllm.invalid/v1"
+CODEX_INSTRUCTIONS: Final = "You are Codex, a coding agent. Follow the repository's AGENTS.md."
+
+
+def _sent_input(input: object) -> object:
+    request: Final = HostedVLLMResponsesAPIConfig().transform_responses_api_request(
+        model="kimi-k3",
+        input=input,
+        response_api_optional_request_params={},
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+    return request["input"]
+
+
+@pytest.mark.parametrize(
+    ("input", "expected"),
+    [
+        pytest.param(
+            [
+                {
+                    "type": "message",
+                    "role": "developer",
+                    "content": [{"type": "input_text", "text": CODEX_INSTRUCTIONS}],
+                },
+                {
+                    "type": "message",
+                    "role": "user",
+                    "content": [{"type": "input_text", "text": "fix the failing test"}],
+                },
+            ],
+            [
+                {"type": "message", "role": "system", "content": [{"type": "input_text", "text": CODEX_INSTRUCTIONS}]},
+                {
+                    "type": "message",
+                    "role": "user",
+                    "content": [{"type": "input_text", "text": "fix the failing test"}],
+                },
+            ],
+            id="codex-leading-developer-item",
+        ),
+        pytest.param(
+            [
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "hello"},
+                {"role": "developer", "content": "Answer in French from now on."},
+                {"role": "user", "content": "how are you?"},
+            ],
+            [
+                {"role": "user", "content": "hi"},
+                {"role": "assistant", "content": "hello"},
+                {"role": "system", "content": "Answer in French from now on."},
+                {"role": "user", "content": "how are you?"},
+            ],
+            id="mid-conversation-developer-item",
+        ),
+        pytest.param(
+            [
+                EasyInputMessage(role="developer", content=CODEX_INSTRUCTIONS, type="message"),
+                EasyInputMessage(role="user", content="hi", type="message"),
+            ],
+            [
+                {"role": "system", "content": CODEX_INSTRUCTIONS, "type": "message"},
+                {"role": "user", "content": "hi", "type": "message"},
+            ],
+            id="pydantic-developer-item",
+        ),
+    ],
+)
+def test_hosted_vllm_responses_sends_developer_items_as_system(input: object, expected: object) -> None:
+    assert _sent_input(input) == expected
+
+
+def test_hosted_vllm_responses_keeps_everything_but_the_developer_role() -> None:
+    input: Final = [
+        {"type": "message", "role": "developer", "id": "msg_dev_1", "status": "completed", "content": "Be terse."},
+        {"type": "message", "role": "system", "content": "You are helpful."},
+        {"type": "function_call", "call_id": "call_1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
+        {"type": "function_call_output", "call_id": "call_1", "output": "21C"},
+    ]
+    assert _sent_input(input) == [
+        {"type": "message", "role": "system", "id": "msg_dev_1", "status": "completed", "content": "Be terse."},
+        *input[1:],
+    ]
+
+
+def test_hosted_vllm_responses_passes_string_input_and_instructions_through() -> None:
+    request: Final = HostedVLLMResponsesAPIConfig().transform_responses_api_request(
+        model="kimi-k3",
+        input="hello",
+        response_api_optional_request_params={"instructions": CODEX_INSTRUCTIONS},
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+    assert request["input"] == "hello"
+    assert request["instructions"] == CODEX_INSTRUCTIONS
+
+
+@pytest.fixture
+def vllm_responses_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> respx.Route:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
+    return respx_mock.post(f"{VLLM_API_BASE}/responses").mock(
+        return_value=httpx.Response(200, json=_make_mock_responses_api_response())
+    )
+
+
+@pytest.mark.asyncio
+async def test_hosted_vllm_aresponses_sends_codex_developer_item_as_system_on_the_wire(
+    vllm_responses_route: respx.Route,
+) -> None:
+    response: Final = await litellm.aresponses(
+        model="hosted_vllm/kimi-k3",
+        input=[
+            {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": CODEX_INSTRUCTIONS}]},
+            {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "fix the failing test"}]},
+        ],
+        api_base=VLLM_API_BASE,
+    )
+    body: Final = json.loads(vllm_responses_route.calls.last.request.content)
+    assert [item["role"] for item in body["input"]] == ["system", "user"]
+    assert body["input"][0]["content"] == [{"type": "input_text", "text": CODEX_INSTRUCTIONS}]
+    assert response.output[0].role == "assistant"  # type: ignore[union-attr]