diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 26280d35da0..fc3a61c078f 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -36,6 +36,7 @@ - {id: llm.chat_completions.azure_openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Azure OpenAI deployments"} - {id: llm.chat_completions.azure_openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Azure OpenAI function_calling"} - {id: llm.chat_completions.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Azure Foundry (azure_ai); newer, smoke"} +- {id: llm.chat_completions.hosted_vllm.passthrough.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_vllm_passthrough_e2e.py", rationale: "OpenAI-format chat via the raw /vllm/{endpoint} passthrough (/vllm/v1/chat/completions), forwarded to a self-hosted vLLM-compatible backend (VLLM_API_BASE); LIT-4751. Batch/file passthrough is not coverable on self-hosted vLLM, which serves no OpenAI Batch API"} - {id: llm.messages.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Core endpoint; Anthropic Messages native"} - {id: llm.messages.anthropic.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Streaming Messages API"} - {id: llm.messages.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "anthropic_endpoints/endpoints.py:64", rationale: "Cost logged on passthrough"} diff --git a/tests/e2e/llm_translation/passthrough_client.py b/tests/e2e/llm_translation/passthrough_client.py index 0576321ede1..c74da3e9abc 100644 --- a/tests/e2e/llm_translation/passthrough_client.py +++ b/tests/e2e/llm_translation/passthrough_client.py @@ -102,6 +102,12 @@ class AnthropicMessageBody(BaseModel): stream: bool = False +class VllmChatBody(BaseModel): + model: str + messages: list[ChatMessage] + max_tokens: int = 64 + + def _tags_header(tags: list[str] | None) -> str | None: return ",".join(tags) if tags else None @@ -185,6 +191,19 @@ class PassthroughClient: stream=stream, ) + def vllm_chat( + self, key: str, model: str, text: str, *, max_tokens: int = 64 + ) -> StreamingResponse: + return self.proxy.transport.send( + "/vllm/v1/chat/completions", + headers=self.proxy.transport.bearer(key), + json=VllmChatBody( + model=model, + max_tokens=max_tokens, + messages=[ChatMessage(role="user", content=text)], + ), + ) + def build_client(proxy: ProxyClient) -> PassthroughClient: return PassthroughClient(proxy=proxy) diff --git a/tests/e2e/llm_translation/test_vllm_passthrough_e2e.py b/tests/e2e/llm_translation/test_vllm_passthrough_e2e.py new file mode 100644 index 00000000000..3ea7d3147c1 --- /dev/null +++ b/tests/e2e/llm_translation/test_vllm_passthrough_e2e.py @@ -0,0 +1,46 @@ +"""Live e2e for the /vllm passthrough route. + +/vllm/{endpoint} is a raw passthrough: the client sends an OpenAI-format request +and litellm forwards it verbatim to the configured vLLM backend (VLLM_API_BASE), +with no per-request model registration (unlike the managed hosted_vllm path in +tests/e2e/batches). This drives /vllm/v1/chat/completions and asserts the +forwarded completion comes back with real content. + +On stage the backend is a CPU llama.cpp server standing in for vLLM (the cluster +is GPU-less and its CPU nodes lack the AVX512 vLLM's CPU build needs); from +litellm's side the passthrough code path is identical. Batch and file passthrough +(/vllm/v1/batches, /vllm/v1/files) is not covered: no self-hosted +vLLM-compatible server implements the OpenAI Batch API, so there is no backend to +forward those routes to. + +A passthrough call returning non-2xx fails hard (never a skip); once it is 2xx, a +missing or empty completion fails too. +""" + +import pytest + +from e2e_config import unique_marker +from e2e_http import require_successful_call +from models import ChatResponse +from passthrough_client import PassthroughClient + +pytestmark = pytest.mark.e2e + +VLLM_PASSTHROUGH_MODEL = "qwen2.5-0.5b-instruct" + + +class TestVllmChatPassthrough: + @pytest.mark.covers("llm.chat_completions.hosted_vllm.passthrough.nonstream.works") + def test_vllm_chat_passthrough_returns_completion( + self, client: PassthroughClient, scoped_key: str + ) -> None: + result = client.vllm_chat( + scoped_key, VLLM_PASSTHROUGH_MODEL, f"Say hello in one word ({unique_marker()})" + ) + require_successful_call(result) + + parsed = ChatResponse.model_validate_json(result.body) + assert parsed.choices, f"/vllm chat passthrough returned no choices: {result.body[:300]}" + message = parsed.choices[0].message + content = (message.content if message else None) or "" + assert content.strip(), f"/vllm chat passthrough returned empty content: {result.body[:300]}"