From 73c4f689c50983a85cee4a462869dfdb8d2571f3 Mon Sep 17 00:00:00 2001 From: Mario Apra Date: Fri, 18 Sep 2026 08:12:12 +0100 Subject: [PATCH] feat(proxy): add /xai pass-through route litellm ships pass-through routes for anthropic, openai, gemini, vertex_ai, bedrock, mistral, cohere, vllm, azure, assemblyai, watsonx and a few others, and it already has a full xai provider for the unified api, so xai missing from that list reads like an oversight. /anthropic/v1/messages worked out of the box while /xai/v1/responses returned 404 callers using xai's own sdk, or the openai sdk pointed at {proxy}/xai/v1, need the native surface, the translated unified route can't give them provider specific behaviour like native error shapes the implementation follows the sibling routes: resolve the base url from XAI_API_BASE with a https://api.x.ai default, normalize the endpoint path, join it onto the base, pull the credential through the existing pass-through credential router (XAI_API_KEY or a deployment marked use_in_pass_through), then delegate to create_pass_through_route the XAI_API_BASE constant in constants.py already carries a /v1 suffix, so it's not reused here on purpose, the caller supplies the version path same as every other pass-through route registers the route in the proxy route list and the lazy loading path prefixes, and refreshes the committed openapi snapshot adds tests for the constructed upstream url (default base, explicit base, base with a path prefix), streaming detection from the request body, and an end to end forward asserting the auth header reaches xai --- litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 222 ++++++++++++++++++ litellm/proxy/_types.py | 1 + .../llm_passthrough_endpoints.py | 34 +++ .../test_llm_pass_through_endpoints.py | 113 +++++++++ 5 files changed, 371 insertions(+) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 2a28ea3763f..b166cf76282 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -213,6 +213,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/vertex_ai/", "/vllm/", "/watsonx/", + "/xai/", ), ), LazyFeature( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b244678e201..ebe208a1861 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -21457,6 +21457,228 @@ "llm_passthrough" ] } + }, + "/xai/{endpoint}": { + "delete": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b0d31df92ce..31c393220a8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -485,6 +485,7 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", + "/xai", "/milvus", "/gigachat", "/watsonx", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 1c763db2146..2efa80b2f92 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -561,6 +561,40 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +@router.api_route( + "/xai/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["xAI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def xai_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + """[Docs](https://docs.litellm.ai/docs/pass_through/xai)""" + base_target_url: Final = get_secret_str("XAI_API_BASE") or "https://api.x.ai" + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + xai_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="xai", + region_name=None, + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping + "Authorization": f"Bearer {xai_api_key}", + }, + is_streaming_request=await is_streaming_request_fn(request), + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/milvus/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9394a13fee4..2e955f8f5df 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -48,6 +48,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( vertex_discovery_proxy_route, vertex_proxy_route, vllm_proxy_route, + xai_proxy_route, ) from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth from litellm.proxy.auth.handle_jwt import JWTHandler @@ -6222,3 +6223,115 @@ class TestTypeSafePassthroughRoute: custom_llm_provider="typesafe", is_streaming_request=False, ) + + +class TestXAIPassthroughRoute: + @staticmethod + def _request(body: Mapping[str, object], path: str) -> Request: + payload: Final = json.dumps(body).encode("utf-8") + + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "headers": [(b"content-type", b"application/json")], + "query_string": b"", + }, + receive=receive, + ) + + @staticmethod + def _install(monkeypatch: pytest.MonkeyPatch, api_base: str | None) -> Mock: + monkeypatch.setenv("XAI_API_KEY", "xai-test-key") + if api_base is None: + monkeypatch.delenv("XAI_API_BASE", raising=False) + else: + monkeypatch.setenv("XAI_API_BASE", api_base) + create_route: Final = Mock(return_value=AsyncMock(return_value={"ok": True})) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + return create_route + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("XAI_API_KEY", "xai-test-key") + monkeypatch.delenv("XAI_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "api_base, expected_target", + [ + (None, "https://api.x.ai/v1/responses"), + ("https://api.x.ai", "https://api.x.ai/v1/responses"), + ("https://xai.example/gateway", "https://xai.example/gateway/v1/responses"), + ], + ) + async def test_target_carries_the_callers_version_path_exactly_once( + self, + monkeypatch: pytest.MonkeyPatch, + api_base: str | None, + expected_target: str, + ) -> None: + create_route: Final = self._install(monkeypatch, api_base) + + result: Final = await xai_proxy_route( + endpoint="v1/responses", + request=self._request({"model": "grok-4.20", "input": "ping"}, "/xai/v1/responses"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert result == {"ok": True} + create_route.assert_called_once_with( + endpoint="v1/responses", + target=expected_target, + custom_headers={"Authorization": "Bearer xai-test-key"}, + is_streaming_request=False, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("stream, expected", [(True, True), (False, False)]) + async def test_streaming_flag_follows_the_request_body( + self, + monkeypatch: pytest.MonkeyPatch, + stream: bool, + expected: bool, + ) -> None: + create_route: Final = self._install(monkeypatch, None) + + await xai_proxy_route( + endpoint="v1/chat/completions", + request=self._request( + {"model": "grok-4.20", "messages": [], "stream": stream}, + "/xai/v1/chat/completions", + ), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["is_streaming_request"] is expected + + def test_native_responses_call_reaches_xai_with_the_resolved_credential(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://api.x.ai/v1/responses").mock( + return_value=httpx.Response(200, json={"id": "resp_123"}) + ) + response = client.post("/xai/v1/responses", json={"model": "grok-4.20", "input": "ping"}) + + assert (response.status_code, response.json()) == (200, {"id": "resp_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer xai-test-key" + assert json.loads(sent.content) == {"model": "grok-4.20", "input": "ping"}