feat(proxy): add /xai pass-through route

litellm ships pass-through routes for anthropic, openai, gemini,
vertex_ai, bedrock, mistral, cohere, vllm, azure, assemblyai, watsonx
and a few others, and it already has a full xai provider for the
unified api, so xai missing from that list reads like an oversight.
/anthropic/v1/messages worked out of the box while /xai/v1/responses
returned 404

callers using xai's own sdk, or the openai sdk pointed at
{proxy}/xai/v1, need the native surface, the translated unified route
can't give them provider specific behaviour like native error shapes

the implementation follows the sibling routes: resolve the base url
from XAI_API_BASE with a https://api.x.ai default, normalize the
endpoint path, join it onto the base, pull the credential through the
existing pass-through credential router (XAI_API_KEY or a deployment
marked use_in_pass_through), then delegate to create_pass_through_route

the XAI_API_BASE constant in constants.py already carries a /v1
suffix, so it's not reused here on purpose, the caller supplies the
version path same as every other pass-through route

registers the route in the proxy route list and the lazy loading path
prefixes, and refreshes the committed openapi snapshot

adds tests for the constructed upstream url (default base, explicit
base, base with a path prefix), streaming detection from the request
body, and an end to end forward asserting the auth header reaches xai
This commit is contained in:
Mario Apra 2026-09-18 08:12:12 +01:00
parent c256c3c1a6
commit 73c4f689c5
No known key found for this signature in database
GPG key ID: E96EBEB3725C0F0A
5 changed files with 371 additions and 0 deletions

View file

@ -213,6 +213,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/vertex_ai/",
"/vllm/",
"/watsonx/",
"/xai/",
),
),
LazyFeature(

View file

@ -21457,6 +21457,228 @@
"llm_passthrough"
]
}
},
"/xai/{endpoint}": {
"delete": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
"operationId": "xai_proxy_route_xai__endpoint__delete",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Xai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"get": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
"operationId": "xai_proxy_route_xai__endpoint__get",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Xai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"patch": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
"operationId": "xai_proxy_route_xai__endpoint__patch",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Xai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"post": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
"operationId": "xai_proxy_route_xai__endpoint__post",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Xai Proxy Route",
"tags": [
"llm_passthrough"
]
},
"put": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
"operationId": "xai_proxy_route_xai__endpoint__put",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Xai Proxy Route",
"tags": [
"llm_passthrough"
]
}
}
}
},

View file

@ -485,6 +485,7 @@ class LiteLLMRoutes(enum.Enum):
"/vllm",
"/mistral",
"/typesafe",
"/xai",
"/milvus",
"/gigachat",
"/watsonx",

View file

@ -561,6 +561,40 @@ async def typesafe_proxy_route(
return await endpoint_func(request, fastapi_response, user_api_key_dict)
@router.api_route(
"/xai/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
tags=["xAI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
)
async def xai_proxy_route(
endpoint: str,
request: Request,
fastapi_response: Response,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""[Docs](https://docs.litellm.ai/docs/pass_through/xai)"""
base_target_url: Final = get_secret_str("XAI_API_BASE") or "https://api.x.ai"
encoded_endpoint: Final = httpx.URL(endpoint).path
normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
base_url: Final = httpx.URL(base_target_url)
updated_url: Final = base_url.copy_with(
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
)
xai_api_key: Final = passthrough_endpoint_router.get_credentials(
custom_llm_provider="xai",
region_name=None,
)
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping
"Authorization": f"Bearer {xai_api_key}",
},
is_streaming_request=await is_streaming_request_fn(request),
)
return await endpoint_func(request, fastapi_response, user_api_key_dict)
@router.api_route(
"/milvus/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

View file

@ -48,6 +48,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
vertex_discovery_proxy_route,
vertex_proxy_route,
vllm_proxy_route,
xai_proxy_route,
)
from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth
from litellm.proxy.auth.handle_jwt import JWTHandler
@ -6222,3 +6223,115 @@ class TestTypeSafePassthroughRoute:
custom_llm_provider="typesafe",
is_streaming_request=False,
)
class TestXAIPassthroughRoute:
@staticmethod
def _request(body: Mapping[str, object], path: str) -> Request:
payload: Final = json.dumps(body).encode("utf-8")
async def receive() -> dict[str, object]:
return {"type": "http.request", "body": payload, "more_body": False}
return Request(
{
"type": "http",
"method": "POST",
"path": path,
"headers": [(b"content-type", b"application/json")],
"query_string": b"",
},
receive=receive,
)
@staticmethod
def _install(monkeypatch: pytest.MonkeyPatch, api_base: str | None) -> Mock:
monkeypatch.setenv("XAI_API_KEY", "xai-test-key")
if api_base is None:
monkeypatch.delenv("XAI_API_BASE", raising=False)
else:
monkeypatch.setenv("XAI_API_BASE", api_base)
create_route: Final = Mock(return_value=AsyncMock(return_value={"ok": True}))
monkeypatch.setattr(
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
create_route,
)
return create_route
@pytest.fixture
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
from litellm.proxy.proxy_server import app
monkeypatch.setenv("XAI_API_KEY", "xai-test-key")
monkeypatch.delenv("XAI_API_BASE", raising=False)
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
yield TestClient(app)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"api_base, expected_target",
[
(None, "https://api.x.ai/v1/responses"),
("https://api.x.ai", "https://api.x.ai/v1/responses"),
("https://xai.example/gateway", "https://xai.example/gateway/v1/responses"),
],
)
async def test_target_carries_the_callers_version_path_exactly_once(
self,
monkeypatch: pytest.MonkeyPatch,
api_base: str | None,
expected_target: str,
) -> None:
create_route: Final = self._install(monkeypatch, api_base)
result: Final = await xai_proxy_route(
endpoint="v1/responses",
request=self._request({"model": "grok-4.20", "input": "ping"}, "/xai/v1/responses"),
fastapi_response=MagicMock(spec=Response),
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
)
assert result == {"ok": True}
create_route.assert_called_once_with(
endpoint="v1/responses",
target=expected_target,
custom_headers={"Authorization": "Bearer xai-test-key"},
is_streaming_request=False,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("stream, expected", [(True, True), (False, False)])
async def test_streaming_flag_follows_the_request_body(
self,
monkeypatch: pytest.MonkeyPatch,
stream: bool,
expected: bool,
) -> None:
create_route: Final = self._install(monkeypatch, None)
await xai_proxy_route(
endpoint="v1/chat/completions",
request=self._request(
{"model": "grok-4.20", "messages": [], "stream": stream},
"/xai/v1/chat/completions",
),
fastapi_response=MagicMock(spec=Response),
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
)
assert create_route.call_args.kwargs["is_streaming_request"] is expected
def test_native_responses_call_reaches_xai_with_the_resolved_credential(self, client: TestClient) -> None:
with respx.mock(assert_all_called=True) as upstream:
route = upstream.post("https://api.x.ai/v1/responses").mock(
return_value=httpx.Response(200, json={"id": "resp_123"})
)
response = client.post("/xai/v1/responses", json={"model": "grok-4.20", "input": "ping"})
assert (response.status_code, response.json()) == (200, {"id": "resp_123"})
sent: Final = route.calls.last.request
assert sent.headers["authorization"] == "Bearer xai-test-key"
assert json.loads(sent.content) == {"model": "grok-4.20", "input": "ping"}