feat(decisions): add hosted_vllm provider (#45501)

* feat(decisions): add hosted_vllm provider

Adds HostedVLLMDecisionsConfig so hosted_vllm/<served model> works on
POST /v1/systemone and POST /v1/decisions against a vLLM server that
serves the System One body (vllm-project/vllm#59299, after v0.31.0).
vLLM only supports choice questions and rejects others, so the base
decisions config gains a per-provider health_check_questions attribute
and the evaluation-mode health probe sends a choice question for
hosted_vllm while every other provider keeps the noul probe.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(decisions): move the hosted_vllm placeholder key to constants

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: kerry <kerry@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/__init__.py

@@ -1681,6 +1681,9 @@ if TYPE_CHECKING:
     from .llms.strands_decider.decisions.transformation import (
         StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig,
     )
+    from .llms.hosted_vllm.decisions.transformation import (
+        HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig,
+    )
     from .llms.openai.decisions.transformation import (
         OpenAIDecisionsConfig as OpenAIDecisionsConfig,
     )

litellm/_lazy_imports_registry.py

@@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = (
     "OpenRouterDecisionsConfig",
     "CloudflareDecisionsConfig",
     "StrandsDeciderDecisionsConfig",
+    "HostedVLLMDecisionsConfig",
     "OpenAIDecisionsConfig",
     "NvidiaNimRerankConfig",
     "NvidiaNimRankingConfig",
@@ -723,6 +724,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.strands_decider.decisions.transformation",
         "StrandsDeciderDecisionsConfig",
     ),
+    "HostedVLLMDecisionsConfig": (
+        ".llms.hosted_vllm.decisions.transformation",
+        "HostedVLLMDecisionsConfig",
+    ),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
     "NvidiaNimRerankConfig": (
         ".llms.nvidia_nim.rerank.transformation",

litellm/constants.py

@@ -285,6 +285,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int(
 
 # Provider-specific API base URLs
 XAI_API_BASE: Final = "https://api.x.ai/v1"
+PLACEHOLDER_API_KEY: Final = "fake-api-key"
 OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE"
 OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY"
 OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0"

litellm/litellm_core_utils/health_check_helpers.py

@@ -3,7 +3,7 @@ Helper functions for health check calls.
 """
 
 import base64
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping
 from typing import TYPE_CHECKING, Final, Literal
 
 from litellm.llms.base_llm.ocr.transformation import DocumentType
@@ -56,6 +56,21 @@ def get_image_file_for_health_check() -> bytes:
     return base64.b64decode(TEST_IMAGE_BASE64)
 
 
+def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]:
+    import litellm
+    from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+    from litellm.utils import ProviderConfigManager
+
+    provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None)
+    config: Final = (
+        None
+        if provider is None
+        else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider)
+    )
+    questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions
+    return {name: dict(question) for name, question in questions.items()}
+
+
 def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
     native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load()
     if native is None:
@@ -300,7 +315,9 @@ class HealthCheckHelpers:
                 **DECISIONS_CALL_PARAMS.validate_python(
                     {
                         "state": prompt or "health check",
-                        "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}},
+                        "questions": _decisions_health_check_questions(
+                            model=model, custom_llm_provider=custom_llm_provider
+                        ),
                         **_filter_model_params(model_params=model_params),
                     }
                 )

litellm/llms/base_llm/decisions/transformation.py

@@ -301,6 +301,9 @@ class BaseDecisionsConfig(ABC):
     api_key_env: tuple[str, ...] = ()
     api_base_env: tuple[str, ...] = ()
     api_key_required: bool = True
+    health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType(
+        {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})}
+    )
 
     def get_default_api_base(self) -> str | None:
         return None

litellm/llms/hosted_vllm/decisions/transformation.py

@@ -0,0 +1,26 @@
+from types import MappingProxyType
+from typing import Final
+
+from litellm.constants import PLACEHOLDER_API_KEY
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class HostedVLLMDecisionsConfig(BaseDecisionsConfig):
+    api_key_env = ("HOSTED_VLLM_API_KEY",)
+    api_base_env = ("HOSTED_VLLM_API_BASE",)
+    api_key_required = False
+    health_check_questions = MappingProxyType(
+        {
+            "reachable": MappingProxyType(
+                {
+                    "type": "choice",
+                    "instructions": "Is the service reachable?",
+                    "criteria": {"yes": None, "no": None},
+                }
+            )
+        }
+    )
+
+    def resolve_api_key(self, api_key: str | None) -> str | None:
+        resolved: Final = super().resolve_api_key(api_key)
+        return None if resolved == PLACEHOLDER_API_KEY else resolved

litellm/utils.py

@@ -8989,6 +8989,8 @@ class ProviderConfigManager:
             return litellm.CloudflareDecisionsConfig()
         if provider == LlmProviders.STRANDS_DECIDER:
             return litellm.StrandsDeciderDecisionsConfig()
+        if provider == LlmProviders.HOSTED_VLLM:
+            return litellm.HostedVLLMDecisionsConfig()
         if provider == LlmProviders.OPENAI:
             return litellm.OpenAIDecisionsConfig()
         return None

provider_endpoints_support.json

@@ -1370,7 +1370,8 @@
         "rerank": true,
         "a2a": true,
         "interactions": true,
-        "video_generations": true
+        "video_generations": true,
+        "systemone": true
       }
     },
     "huggingface": {

tests/integration/management/test_model_health_check.py

@@ -24,6 +24,16 @@ _CONFIGURED_PROBE_REPLY: Final = JsonResponse(
         "usage": {"input_tokens": 10, "output_tokens": 1},
     },
 )
+_VLLM_PROBE_REPLY: Final = JsonResponse(
+    content_type="application/json",
+    body={
+        "model": "Qwen/Qwen3-0.6B",
+        "answers": {
+            "reachable": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0, "no": 0.0}}
+        },
+        "usage": {"input_tokens": 10, "output_tokens": 1},
+    },
+)
 _STRANDS_PROBE_REPLY: Final = JsonResponse(
     content_type="application/json",
     body={
@@ -147,3 +157,32 @@ def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_
                 },
             )
         ]
+
+
+def test_evaluation_mode_health_check_of_hosted_vllm_sends_a_choice_probe(gateway: Gateway) -> None:
+    with gateway.scenario() as scenario:
+        handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _VLLM_PROBE_REPLY)
+        scenario.cleanups.callback(delete_scenario, handle)
+        model: Final = scenario.model(
+            model="hosted_vllm/Qwen/Qwen3-0.6B",
+            api_base=handle.api_base(),
+            model_info={"mode": "evaluation"},
+        )
+        report: Final = _health_report(gateway, model)
+        assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report
+        assert _probes_sent_to(gateway, handle) == [
+            (
+                f"/{handle.scenario_id}/v1/systemone",
+                {
+                    "model": "Qwen/Qwen3-0.6B",
+                    "state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"),
+                    "questions": {
+                        "reachable": {
+                            "type": "choice",
+                            "instructions": "Is the service reachable?",
+                            "criteria": {"yes": None, "no": None},
+                        }
+                    },
+                },
+            )
+        ]

tests/integration/providers/test_decisions_wire.py

@@ -94,6 +94,9 @@ _PROVIDERS: Final = (
     _Provider(
         "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None
     ),
+    _Provider(
+        "hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None
+    ),
     _Provider(
         "cloudflare",
         "cloudflare/clef",

tests/integration/proxy_config.yaml

@@ -346,6 +346,10 @@ model_list:
     litellm_params:
       model: strands_decider/strands-decider-2B-hobson-v19
       api_base: http://127.0.0.1:8191
+  - model_name: hosted_vllm/Qwen/Qwen3-0.6B
+    litellm_params:
+      model: hosted_vllm/Qwen/Qwen3-0.6B
+      api_base: http://127.0.0.1:8191
 general_settings:
   master_key: os.environ/LITELLM_MASTER_KEY
   database_url: os.environ/DATABASE_URL

tests/integration/translation/decisions/bases/hosted_vllm.py

@@ -0,0 +1,67 @@
+from typing import Final
+
+from integration.translation.case import TranslationTestCase
+
+"""Provider request and reply shape from a real vLLM serve (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-09 from a vLLM CPU build of vllm main 702ce313f serving Qwen/Qwen3-0.6B.
+"""
+QWEN3_0_6B_TEST_CASE: Final = TranslationTestCase(
+    scenario="basic",
+    litellm_endpoint="/v1/systemone",
+    litellm_request={
+        "model": "hosted_vllm/Qwen/Qwen3-0.6B",
+        "state": "Help! My payouts have been failing for 3 days!",
+        "questions": {
+            "is_urgent": {
+                "type": "choice",
+                "instructions": "Does this convey urgency?",
+                "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"},
+            }
+        },
+        "cache": {"no-cache": True},
+    },
+    expected_provider_endpoint="/v1/systemone",
+    expected_provider_headers={"content-type": "application/json"},
+    expected_provider_request={
+        "model": "Qwen/Qwen3-0.6B",
+        "state": "Help! My payouts have been failing for 3 days!",
+        "questions": {
+            "is_urgent": {
+                "type": "choice",
+                "instructions": "Does this convey urgency?",
+                "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"},
+            }
+        },
+    },
+    mock_provider_response={
+        "id": "decision-9c26c028d7f4457e",
+        "object": "structured_decision",
+        "created": 1791517765,
+        "model": "Qwen/Qwen3-0.6B",
+        "answers": {
+            "is_urgent": {
+                "type": "choice",
+                "choice": "yes",
+                "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374},
+                "confidence": 0.9649145337527952,
+            }
+        },
+        "usage": {"input_tokens": 61, "output_tokens": 1},
+        "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}},
+    },
+    expected_litellm_response={
+        "id": "decision-9c26c028d7f4457e",
+        "object": "structured_decision",
+        "created": 1791517765,
+        "model": "Qwen/Qwen3-0.6B",
+        "answers": {
+            "is_urgent": {
+                "type": "choice",
+                "choice": "yes",
+                "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374},
+                "confidence": 0.9649145337527952,
+            }
+        },
+        "usage": {"input_tokens": 61, "output_tokens": 1},
+        "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}},
+    },
+)

tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py

@@ -0,0 +1,11 @@
+import pytest
+from integration._support.client import Gateway
+from integration._support.provider import SharedProvider
+from integration.translation.case import TranslationTestCase
+from integration.translation.decisions.bases.hosted_vllm import QWEN3_0_6B_TEST_CASE
+from integration.translation.runner import assert_translation
+
+
+@pytest.mark.parametrize("case", [QWEN3_0_6B_TEST_CASE], ids=lambda case: case.id)
+def test_decisions_basic_hosted_vllm(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None:
+    assert_translation(case, gateway, provider)

tests/unit/decisions/test_main.py

@@ -733,6 +733,139 @@ async def test_strands_decider_provider_resolution_and_router_dispatch(
     assert response.model == _STRANDS_RESPONSE["model"]
 
 
+_VLLM_RESPONSE: Final[Mapping[str, object]] = {
+    "id": "systemone-4af3d2c1",
+    "object": "structured_decision",
+    "created": 1760012345,
+    "model": "Qwen/Qwen3-0.6B",
+    "answers": {
+        "is_defect": {
+            "type": "choice",
+            "choice": "yes",
+            "confidence": 0.91,
+            "probabilities": {"yes": 0.91, "no": 0.09},
+        }
+    },
+    "usage": {"input_tokens": 154, "output_tokens": 2},
+    "diagnostics": {"engine": "vllm", "logprobs_mode": "raw_logprobs"},
+}
+
+
+@pytest.mark.asyncio
+async def test_hosted_vllm_requires_api_base_before_http(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
+    monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
+
+    with pytest.raises(litellm.BadRequestError, match="api_base is required"):
+        await litellm.adecisions(
+            model="hosted_vllm/Qwen/Qwen3-0.6B",
+            state="review",
+            questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
+        )
+
+    assert len(respx_mock.calls) == 0
+
+
+@pytest.mark.asyncio
+async def test_hosted_vllm_without_key_sends_no_authorization_and_preserves_response_extras(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
+    monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
+    route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
+
+    response: Final = await litellm.adecisions(
+        model="hosted_vllm/Qwen/Qwen3-0.6B",
+        state="The package arrived broken.",
+        questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
+        api_base="http://vllm.local:8000",
+    )
+
+    assert route.called
+    assert "authorization" not in respx_mock.calls[0].request.headers
+    assert response.model_extra["id"] == _VLLM_RESPONSE["id"]
+    assert response.model_extra["object"] == _VLLM_RESPONSE["object"]
+    assert response.model_extra["diagnostics"] == _VLLM_RESPONSE["diagnostics"]
+
+
+@pytest.mark.asyncio
+async def test_hosted_vllm_uses_key_and_base_from_environment(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm.local:8000/v1")
+    monkeypatch.setenv("HOSTED_VLLM_API_KEY", "vllm-key")
+    route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
+
+    await litellm.adecisions(
+        model="hosted_vllm/Qwen/Qwen3-0.6B",
+        state="The package arrived broken.",
+        questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
+    )
+
+    assert route.called
+    assert respx_mock.calls[0].request.headers["authorization"] == "Bearer vllm-key"
+
+
+@pytest.mark.asyncio
+async def test_hosted_vllm_api_base_with_v1_suffix_posts_to_v1_systemone(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
+    monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
+    route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
+
+    await litellm.adecisions(
+        model="hosted_vllm/Qwen/Qwen3-0.6B",
+        state="The package arrived broken.",
+        questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}},
+        api_base="http://vllm.local:8000/v1",
+    )
+
+    assert route.called
+
+
+@pytest.mark.asyncio
+async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_served_model_name(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
+    monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
+    route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE)
+
+    await litellm.adecisions(
+        model="hosted_vllm/Qwen/Qwen3-0.6B",
+        state="The package arrived broken.",
+        questions={
+            "is_defect": {
+                "type": "choice",
+                "instructions": "Is this a defect?",
+                "criteria": {"yes": "defective", "no": "intact"},
+            }
+        },
+        api_base="http://vllm.local:8000",
+    )
+
+    assert route.called
+    assert json.loads(respx_mock.calls[0].request.content) == {
+        "model": "Qwen/Qwen3-0.6B",
+        "state": "The package arrived broken.",
+        "questions": {
+            "is_defect": {
+                "type": "choice",
+                "instructions": "Is this a defect?",
+                "criteria": {"yes": "defective", "no": "intact"},
+            }
+        },
+    }
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     ("api_base", "url"),

tests/unit/litellm_core_utils/test_health_check_helpers.py

@@ -679,6 +679,81 @@ async def test_ahealth_check_probes_strands_through_decisions_without_mode(
     assert "authorization" not in upstream.calls[0].request.headers
 
 
+@pytest.mark.asyncio
+async def test_ahealth_check_probes_hosted_vllm_with_a_choice_question(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
+    monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    litellm.in_memory_llm_clients_cache.flush_cache()
+    upstream: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(
+        json={
+            "model": "Qwen/Qwen3-0.6B",
+            "answers": {
+                "reachable": {
+                    "type": "choice",
+                    "choice": "yes",
+                    "confidence": 1.0,
+                    "probabilities": {"yes": 1.0, "no": 0.0},
+                }
+            },
+            "usage": {"input_tokens": 12, "output_tokens": 1},
+        }
+    )
+
+    result: Final = await ahealth_check(
+        {
+            "model": "hosted_vllm/Qwen/Qwen3-0.6B",
+            "api_base": "http://vllm.local:8000",
+        },
+        mode="evaluation",
+    )
+
+    assert "error" not in result, result
+    assert upstream.called
+    sent: Final = json.loads(upstream.calls[0].request.content)
+    assert sent["questions"] == {
+        "reachable": {
+            "type": "choice",
+            "instructions": "Is the service reachable?",
+            "criteria": {"yes": None, "no": None},
+        }
+    }
+
+
+@pytest.mark.asyncio
+async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers(
+    monkeypatch: pytest.MonkeyPatch,
+    respx_mock: respx.MockRouter,
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    litellm.in_memory_llm_clients_cache.flush_cache()
+    upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(
+        json={
+            "model": "jev-1.13",
+            "answers": {"reachable": {"type": "noul", "noul": 1.0}},
+            "usage": {"input_tokens": 12, "output_tokens": 1},
+        }
+    )
+
+    result: Final = await ahealth_check(
+        {
+            "model": "typesafe/jev-1.13",
+            "api_key": "sk-test",
+        },
+        mode="evaluation",
+    )
+
+    assert "error" not in result, result
+    assert upstream.called
+    sent: Final = json.loads(upstream.calls[0].request.content)
+    assert sent["questions"] == {
+        "reachable": {"type": "noul", "instructions": "Is the service reachable?"}
+    }
+
+
 @pytest.mark.parametrize(
     ("model", "custom_llm_provider", "expected"),
     (