diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 2a28ea3763f..b166cf76282 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -213,6 +213,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( "/vertex_ai/", "/vllm/", "/watsonx/", + "/xai/", ), ), LazyFeature( diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index b244678e201..ebe208a1861 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -21457,6 +21457,228 @@ "llm_passthrough" ] } + }, + "/xai/{endpoint}": { + "delete": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__delete", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "get": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__get", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "patch": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__patch", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "post": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__post", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + }, + "put": { + "description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)", + "operationId": "xai_proxy_route_xai__endpoint__put", + "parameters": [ + { + "in": "path", + "name": "endpoint", + "required": true, + "schema": { + "title": "Endpoint", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Xai Proxy Route", + "tags": [ + "llm_passthrough" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b0d31df92ce..31c393220a8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -485,6 +485,7 @@ class LiteLLMRoutes(enum.Enum): "/vllm", "/mistral", "/typesafe", + "/xai", "/milvus", "/gigachat", "/watsonx", diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 1c763db2146..2efa80b2f92 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -561,6 +561,40 @@ async def typesafe_proxy_route( return await endpoint_func(request, fastapi_response, user_api_key_dict) +@router.api_route( + "/xai/{endpoint:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list + tags=["xAI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list +) +async def xai_proxy_route( + endpoint: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + """[Docs](https://docs.litellm.ai/docs/pass_through/xai)""" + base_target_url: Final = get_secret_str("XAI_API_BASE") or "https://api.x.ai" + encoded_endpoint: Final = httpx.URL(endpoint).path + normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}" + base_url: Final = httpx.URL(base_target_url) + updated_url: Final = base_url.copy_with( + path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint), + ) + xai_api_key: Final = passthrough_endpoint_router.get_credentials( + custom_llm_provider="xai", + region_name=None, + ) + endpoint_func: Final = create_pass_through_route( + endpoint=endpoint, + target=str(updated_url), + custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping + "Authorization": f"Bearer {xai_api_key}", + }, + is_streaming_request=await is_streaming_request_fn(request), + ) + return await endpoint_func(request, fastapi_response, user_api_key_dict) + + @router.api_route( "/milvus/{endpoint:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"], diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 9394a13fee4..2e955f8f5df 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -48,6 +48,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( vertex_discovery_proxy_route, vertex_proxy_route, vllm_proxy_route, + xai_proxy_route, ) from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth from litellm.proxy.auth.handle_jwt import JWTHandler @@ -6222,3 +6223,115 @@ class TestTypeSafePassthroughRoute: custom_llm_provider="typesafe", is_streaming_request=False, ) + + +class TestXAIPassthroughRoute: + @staticmethod + def _request(body: Mapping[str, object], path: str) -> Request: + payload: Final = json.dumps(body).encode("utf-8") + + async def receive() -> dict[str, object]: + return {"type": "http.request", "body": payload, "more_body": False} + + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "headers": [(b"content-type", b"application/json")], + "query_string": b"", + }, + receive=receive, + ) + + @staticmethod + def _install(monkeypatch: pytest.MonkeyPatch, api_base: str | None) -> Mock: + monkeypatch.setenv("XAI_API_KEY", "xai-test-key") + if api_base is None: + monkeypatch.delenv("XAI_API_BASE", raising=False) + else: + monkeypatch.setenv("XAI_API_BASE", api_base) + create_route: Final = Mock(return_value=AsyncMock(return_value={"ok": True})) + monkeypatch.setattr( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route", + create_route, + ) + return create_route + + @pytest.fixture + def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: + from litellm.proxy.proxy_server import app + + monkeypatch.setenv("XAI_API_KEY", "xai-test-key") + monkeypatch.delenv("XAI_API_BASE", raising=False) + monkeypatch.delenv("SERVER_ROOT_PATH", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual")) + yield TestClient(app) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "api_base, expected_target", + [ + (None, "https://api.x.ai/v1/responses"), + ("https://api.x.ai", "https://api.x.ai/v1/responses"), + ("https://xai.example/gateway", "https://xai.example/gateway/v1/responses"), + ], + ) + async def test_target_carries_the_callers_version_path_exactly_once( + self, + monkeypatch: pytest.MonkeyPatch, + api_base: str | None, + expected_target: str, + ) -> None: + create_route: Final = self._install(monkeypatch, api_base) + + result: Final = await xai_proxy_route( + endpoint="v1/responses", + request=self._request({"model": "grok-4.20", "input": "ping"}, "/xai/v1/responses"), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert result == {"ok": True} + create_route.assert_called_once_with( + endpoint="v1/responses", + target=expected_target, + custom_headers={"Authorization": "Bearer xai-test-key"}, + is_streaming_request=False, + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("stream, expected", [(True, True), (False, False)]) + async def test_streaming_flag_follows_the_request_body( + self, + monkeypatch: pytest.MonkeyPatch, + stream: bool, + expected: bool, + ) -> None: + create_route: Final = self._install(monkeypatch, None) + + await xai_proxy_route( + endpoint="v1/chat/completions", + request=self._request( + {"model": "grok-4.20", "messages": [], "stream": stream}, + "/xai/v1/chat/completions", + ), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"), + ) + + assert create_route.call_args.kwargs["is_streaming_request"] is expected + + def test_native_responses_call_reaches_xai_with_the_resolved_credential(self, client: TestClient) -> None: + with respx.mock(assert_all_called=True) as upstream: + route = upstream.post("https://api.x.ai/v1/responses").mock( + return_value=httpx.Response(200, json={"id": "resp_123"}) + ) + response = client.post("/xai/v1/responses", json={"model": "grok-4.20", "input": "ping"}) + + assert (response.status_code, response.json()) == (200, {"id": "resp_123"}) + sent: Final = route.calls.last.request + assert sent.headers["authorization"] == "Bearer xai-test-key" + assert json.loads(sent.content) == {"model": "grok-4.20", "input": "ping"}