diff --git a/litellm/__init__.py b/litellm/__init__.py index a44af3ceac8..7d3fb19d662 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1681,6 +1681,9 @@ if TYPE_CHECKING: from .llms.strands_decider.decisions.transformation import ( StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, ) + from .llms.hosted_vllm.decisions.transformation import ( + HostedVLLMDecisionsConfig as HostedVLLMDecisionsConfig, + ) from .llms.openai.decisions.transformation import ( OpenAIDecisionsConfig as OpenAIDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index dd41e49d320..77a45a18db2 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -162,6 +162,7 @@ LLM_CONFIG_NAMES: Final = ( "OpenRouterDecisionsConfig", "CloudflareDecisionsConfig", "StrandsDeciderDecisionsConfig", + "HostedVLLMDecisionsConfig", "OpenAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", @@ -723,6 +724,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.strands_decider.decisions.transformation", "StrandsDeciderDecisionsConfig", ), + "HostedVLLMDecisionsConfig": ( + ".llms.hosted_vllm.decisions.transformation", + "HostedVLLMDecisionsConfig", + ), "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", diff --git a/litellm/constants.py b/litellm/constants.py index 699872f32cc..290749e562f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -285,6 +285,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( # Provider-specific API base URLs XAI_API_BASE: Final = "https://api.x.ai/v1" +PLACEHOLDER_API_KEY: Final = "fake-api-key" OPEN_SANDBOX_API_BASE_ENV_VAR: Final = "OPEN_SANDBOX_API_BASE" OPEN_SANDBOX_API_KEY_ENV_VAR: Final = "OPEN_SANDBOX_API_KEY" OPEN_SANDBOX_DEFAULT_TEMPLATE: Final = "opensandbox/code-interpreter:v1.1.0" diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 60e4b7495ed..5d14a9761d5 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -3,7 +3,7 @@ Helper functions for health check calls. """ import base64 -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType @@ -56,6 +56,21 @@ def get_image_file_for_health_check() -> bytes: return base64.b64decode(TEST_IMAGE_BASE64) +def _decisions_health_check_questions(model: str, custom_llm_provider: str) -> Mapping[str, Mapping[str, object]]: + import litellm + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + from litellm.utils import ProviderConfigManager + + provider: Final = next((member for member in litellm.LlmProviders if member.value == custom_llm_provider), None) + config: Final = ( + None + if provider is None + else ProviderConfigManager.get_provider_decisions_config(model=model, provider=provider) + ) + questions: Final = BaseDecisionsConfig.health_check_questions if config is None else config.health_check_questions + return {name: dict(question) for name, question in questions.items()} + + def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType: native: Final = NATIVE_OCR_HEALTH_CHECK_DOCUMENT.load() if native is None: @@ -300,7 +315,9 @@ class HealthCheckHelpers: **DECISIONS_CALL_PARAMS.validate_python( { "state": prompt or "health check", - "questions": {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}}, + "questions": _decisions_health_check_questions( + model=model, custom_llm_provider=custom_llm_provider + ), **_filter_model_params(model_params=model_params), } ) diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index a8d14ca59a3..9d1ed188d7b 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -301,6 +301,9 @@ class BaseDecisionsConfig(ABC): api_key_env: tuple[str, ...] = () api_base_env: tuple[str, ...] = () api_key_required: bool = True + health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( + {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} + ) def get_default_api_base(self) -> str | None: return None diff --git a/litellm/llms/hosted_vllm/decisions/transformation.py b/litellm/llms/hosted_vllm/decisions/transformation.py new file mode 100644 index 00000000000..ef99f2de173 --- /dev/null +++ b/litellm/llms/hosted_vllm/decisions/transformation.py @@ -0,0 +1,26 @@ +from types import MappingProxyType +from typing import Final + +from litellm.constants import PLACEHOLDER_API_KEY +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig + + +class HostedVLLMDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("HOSTED_VLLM_API_KEY",) + api_base_env = ("HOSTED_VLLM_API_BASE",) + api_key_required = False + health_check_questions = MappingProxyType( + { + "reachable": MappingProxyType( + { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + ) + } + ) + + def resolve_api_key(self, api_key: str | None) -> str | None: + resolved: Final = super().resolve_api_key(api_key) + return None if resolved == PLACEHOLDER_API_KEY else resolved diff --git a/litellm/utils.py b/litellm/utils.py index dec57245ab2..b51f349c4e6 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8989,6 +8989,8 @@ class ProviderConfigManager: return litellm.CloudflareDecisionsConfig() if provider == LlmProviders.STRANDS_DECIDER: return litellm.StrandsDeciderDecisionsConfig() + if provider == LlmProviders.HOSTED_VLLM: + return litellm.HostedVLLMDecisionsConfig() if provider == LlmProviders.OPENAI: return litellm.OpenAIDecisionsConfig() return None diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 62e23388581..ef1fa270b06 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1370,7 +1370,8 @@ "rerank": true, "a2a": true, "interactions": true, - "video_generations": true + "video_generations": true, + "systemone": true } }, "huggingface": { diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 17bd1c2ebfe..11b902b540a 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -24,6 +24,16 @@ _CONFIGURED_PROBE_REPLY: Final = JsonResponse( "usage": {"input_tokens": 10, "output_tokens": 1}, }, ) +_VLLM_PROBE_REPLY: Final = JsonResponse( + content_type="application/json", + body={ + "model": "Qwen/Qwen3-0.6B", + "answers": { + "reachable": {"type": "choice", "choice": "yes", "confidence": 1.0, "probabilities": {"yes": 1.0, "no": 0.0}} + }, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, +) _STRANDS_PROBE_REPLY: Final = JsonResponse( content_type="application/json", body={ @@ -147,3 +157,32 @@ def test_evaluation_mode_health_check_of_the_self_hosted_strands_model_resolves_ }, ) ] + + +def test_evaluation_mode_health_check_of_hosted_vllm_sends_a_choice_probe(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + handle: Final = register_scenario(f"health-decisions-{uuid.uuid4().hex[:12]}", _VLLM_PROBE_REPLY) + scenario.cleanups.callback(delete_scenario, handle) + model: Final = scenario.model( + model="hosted_vllm/Qwen/Qwen3-0.6B", + api_base=handle.api_base(), + model_info={"mode": "evaluation"}, + ) + report: Final = _health_report(gateway, model) + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + assert _probes_sent_to(gateway, handle) == [ + ( + f"/{handle.scenario_id}/v1/systemone", + { + "model": "Qwen/Qwen3-0.6B", + "state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"), + "questions": { + "reachable": { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + }, + }, + ) + ] diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 07b02480fc8..3e922cfc314 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -94,6 +94,9 @@ _PROVIDERS: Final = ( _Provider( "strands_decider", "strands_decider/systemone-decider", "/v1/systemone", "systemone-decider", None, False, None ), + _Provider( + "hosted_vllm", "hosted_vllm/Qwen/Qwen3-0.6B", "/v1/systemone", "Qwen/Qwen3-0.6B", None, False, None + ), _Provider( "cloudflare", "cloudflare/clef", diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 4c027254838..46f2beb5d20 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -346,6 +346,10 @@ model_list: litellm_params: model: strands_decider/strands-decider-2B-hobson-v19 api_base: http://127.0.0.1:8191 + - model_name: hosted_vllm/Qwen/Qwen3-0.6B + litellm_params: + model: hosted_vllm/Qwen/Qwen3-0.6B + api_base: http://127.0.0.1:8191 general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/integration/translation/decisions/bases/hosted_vllm.py b/tests/integration/translation/decisions/bases/hosted_vllm.py new file mode 100644 index 00000000000..2da37f83a86 --- /dev/null +++ b/tests/integration/translation/decisions/bases/hosted_vllm.py @@ -0,0 +1,67 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from a real vLLM serve (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-09 from a vLLM CPU build of vllm main 702ce313f serving Qwen/Qwen3-0.6B. +""" +QWEN3_0_6B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "hosted_vllm/Qwen/Qwen3-0.6B", + "state": "Help! My payouts have been failing for 3 days!", + "questions": { + "is_urgent": { + "type": "choice", + "instructions": "Does this convey urgency?", + "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"}, + } + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "Qwen/Qwen3-0.6B", + "state": "Help! My payouts have been failing for 3 days!", + "questions": { + "is_urgent": { + "type": "choice", + "instructions": "Does this convey urgency?", + "criteria": {"yes": "the message conveys urgency", "no": "the message does not convey urgency"}, + } + }, + }, + mock_provider_response={ + "id": "decision-9c26c028d7f4457e", + "object": "structured_decision", + "created": 1791517765, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_urgent": { + "type": "choice", + "choice": "yes", + "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374}, + "confidence": 0.9649145337527952, + } + }, + "usage": {"input_tokens": 61, "output_tokens": 1}, + "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}}, + }, + expected_litellm_response={ + "id": "decision-9c26c028d7f4457e", + "object": "structured_decision", + "created": 1791517765, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_urgent": { + "type": "choice", + "choice": "yes", + "probabilities": {"yes": 0.9669140238756562, "no": 0.03308597612434374}, + "confidence": 0.9649145337527952, + } + }, + "usage": {"input_tokens": 61, "output_tokens": 1}, + "diagnostics": {"is_urgent": {"label_mass": 0.9979320910923947, "argmax_is_label": True}}, + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py b/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py new file mode 100644 index 00000000000..f52d9147273 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_hosted_vllm.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.hosted_vllm import QWEN3_0_6B_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [QWEN3_0_6B_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_hosted_vllm(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 820451ed943..c7a29ed8e0e 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -733,6 +733,139 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert response.model == _STRANDS_RESPONSE["model"] +_VLLM_RESPONSE: Final[Mapping[str, object]] = { + "id": "systemone-4af3d2c1", + "object": "structured_decision", + "created": 1760012345, + "model": "Qwen/Qwen3-0.6B", + "answers": { + "is_defect": { + "type": "choice", + "choice": "yes", + "confidence": 0.91, + "probabilities": {"yes": 0.91, "no": 0.09}, + } + }, + "usage": {"input_tokens": 154, "output_tokens": 2}, + "diagnostics": {"engine": "vllm", "logprobs_mode": "raw_logprobs"}, +} + + +@pytest.mark.asyncio +async def test_hosted_vllm_requires_api_base_before_http( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + + with pytest.raises(litellm.BadRequestError, match="api_base is required"): + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="review", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_hosted_vllm_without_key_sends_no_authorization_and_preserves_response_extras( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + response: Final = await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + api_base="http://vllm.local:8000", + ) + + assert route.called + assert "authorization" not in respx_mock.calls[0].request.headers + assert response.model_extra["id"] == _VLLM_RESPONSE["id"] + assert response.model_extra["object"] == _VLLM_RESPONSE["object"] + assert response.model_extra["diagnostics"] == _VLLM_RESPONSE["diagnostics"] + + +@pytest.mark.asyncio +async def test_hosted_vllm_uses_key_and_base_from_environment( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://vllm.local:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "vllm-key") + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + ) + + assert route.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer vllm-key" + + +@pytest.mark.asyncio +async def test_hosted_vllm_api_base_with_v1_suffix_posts_to_v1_systemone( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={"is_defect": {"type": "choice", "criteria": {"yes": None, "no": None}}}, + api_base="http://vllm.local:8000/v1", + ) + + assert route.called + + +@pytest.mark.asyncio +async def test_hosted_vllm_choice_question_goes_out_as_a_jev_choice_with_the_served_model_name( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond(json=_VLLM_RESPONSE) + + await litellm.adecisions( + model="hosted_vllm/Qwen/Qwen3-0.6B", + state="The package arrived broken.", + questions={ + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + api_base="http://vllm.local:8000", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content) == { + "model": "Qwen/Qwen3-0.6B", + "state": "The package arrived broken.", + "questions": { + "is_defect": { + "type": "choice", + "instructions": "Is this a defect?", + "criteria": {"yes": "defective", "no": "intact"}, + } + }, + } + + @pytest.mark.asyncio @pytest.mark.parametrize( ("api_base", "url"), diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index c0298bc1c80..9903ab71876 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -679,6 +679,81 @@ async def test_ahealth_check_probes_strands_through_decisions_without_mode( assert "authorization" not in upstream.calls[0].request.headers +@pytest.mark.asyncio +async def test_ahealth_check_probes_hosted_vllm_with_a_choice_question( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("http://vllm.local:8000/v1/systemone").respond( + json={ + "model": "Qwen/Qwen3-0.6B", + "answers": { + "reachable": { + "type": "choice", + "choice": "yes", + "confidence": 1.0, + "probabilities": {"yes": 1.0, "no": 0.0}, + } + }, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "hosted_vllm/Qwen/Qwen3-0.6B", + "api_base": "http://vllm.local:8000", + }, + mode="evaluation", + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["questions"] == { + "reachable": { + "type": "choice", + "instructions": "Is the service reachable?", + "criteria": {"yes": None, "no": None}, + } + } + + +@pytest.mark.asyncio +async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond( + json={ + "model": "jev-1.13", + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + } + ) + + result: Final = await ahealth_check( + { + "model": "typesafe/jev-1.13", + "api_key": "sk-test", + }, + mode="evaluation", + ) + + assert "error" not in result, result + assert upstream.called + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["questions"] == { + "reachable": {"type": "noul", "instructions": "Is the service reachable?"} + } + + @pytest.mark.parametrize( ("model", "custom_llm_provider", "expected"), (