mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(proxy): add /xai pass-through route
litellm ships pass-through routes for anthropic, openai, gemini,
vertex_ai, bedrock, mistral, cohere, vllm, azure, assemblyai, watsonx
and a few others, and it already has a full xai provider for the
unified api, so xai missing from that list reads like an oversight.
/anthropic/v1/messages worked out of the box while /xai/v1/responses
returned 404
callers using xai's own sdk, or the openai sdk pointed at
{proxy}/xai/v1, need the native surface, the translated unified route
can't give them provider specific behaviour like native error shapes
the implementation follows the sibling routes: resolve the base url
from XAI_API_BASE with a https://api.x.ai default, normalize the
endpoint path, join it onto the base, pull the credential through the
existing pass-through credential router (XAI_API_KEY or a deployment
marked use_in_pass_through), then delegate to create_pass_through_route
the XAI_API_BASE constant in constants.py already carries a /v1
suffix, so it's not reused here on purpose, the caller supplies the
version path same as every other pass-through route
registers the route in the proxy route list and the lazy loading path
prefixes, and refreshes the committed openapi snapshot
adds tests for the constructed upstream url (default base, explicit
base, base with a path prefix), streaming detection from the request
body, and an end to end forward asserting the auth header reaches xai
This commit is contained in:
parent
c256c3c1a6
commit
73c4f689c5
5 changed files with 371 additions and 0 deletions
|
|
@ -213,6 +213,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
"/vertex_ai/",
|
||||
"/vllm/",
|
||||
"/watsonx/",
|
||||
"/xai/",
|
||||
),
|
||||
),
|
||||
LazyFeature(
|
||||
|
|
|
|||
|
|
@ -21457,6 +21457,228 @@
|
|||
"llm_passthrough"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/xai/{endpoint}": {
|
||||
"delete": {
|
||||
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
|
||||
"operationId": "xai_proxy_route_xai__endpoint__delete",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Xai Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
},
|
||||
"get": {
|
||||
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
|
||||
"operationId": "xai_proxy_route_xai__endpoint__get",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Xai Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
},
|
||||
"patch": {
|
||||
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
|
||||
"operationId": "xai_proxy_route_xai__endpoint__patch",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Xai Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
},
|
||||
"post": {
|
||||
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
|
||||
"operationId": "xai_proxy_route_xai__endpoint__post",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Xai Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
},
|
||||
"put": {
|
||||
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/xai)",
|
||||
"operationId": "xai_proxy_route_xai__endpoint__put",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Xai Proxy Route",
|
||||
"tags": [
|
||||
"llm_passthrough"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -485,6 +485,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/vllm",
|
||||
"/mistral",
|
||||
"/typesafe",
|
||||
"/xai",
|
||||
"/milvus",
|
||||
"/gigachat",
|
||||
"/watsonx",
|
||||
|
|
|
|||
|
|
@ -561,6 +561,40 @@ async def typesafe_proxy_route(
|
|||
return await endpoint_func(request, fastapi_response, user_api_key_dict)
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/xai/{endpoint:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE", "PATCH"], # mutable-ok: FastAPI route metadata requires a list
|
||||
tags=["xAI Pass-through", "pass-through"], # mutable-ok: FastAPI route metadata requires a list
|
||||
)
|
||||
async def xai_proxy_route(
|
||||
endpoint: str,
|
||||
request: Request,
|
||||
fastapi_response: Response,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
"""[Docs](https://docs.litellm.ai/docs/pass_through/xai)"""
|
||||
base_target_url: Final = get_secret_str("XAI_API_BASE") or "https://api.x.ai"
|
||||
encoded_endpoint: Final = httpx.URL(endpoint).path
|
||||
normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
|
||||
base_url: Final = httpx.URL(base_target_url)
|
||||
updated_url: Final = base_url.copy_with(
|
||||
path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
|
||||
)
|
||||
xai_api_key: Final = passthrough_endpoint_router.get_credentials(
|
||||
custom_llm_provider="xai",
|
||||
region_name=None,
|
||||
)
|
||||
endpoint_func: Final = create_pass_through_route(
|
||||
endpoint=endpoint,
|
||||
target=str(updated_url),
|
||||
custom_headers={ # mutable-ok: pass-through request headers require a mutable mapping
|
||||
"Authorization": f"Bearer {xai_api_key}",
|
||||
},
|
||||
is_streaming_request=await is_streaming_request_fn(request),
|
||||
)
|
||||
return await endpoint_func(request, fastapi_response, user_api_key_dict)
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/milvus/{endpoint:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
|
|||
vertex_discovery_proxy_route,
|
||||
vertex_proxy_route,
|
||||
vllm_proxy_route,
|
||||
xai_proxy_route,
|
||||
)
|
||||
from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler
|
||||
|
|
@ -6222,3 +6223,115 @@ class TestTypeSafePassthroughRoute:
|
|||
custom_llm_provider="typesafe",
|
||||
is_streaming_request=False,
|
||||
)
|
||||
|
||||
|
||||
class TestXAIPassthroughRoute:
|
||||
@staticmethod
|
||||
def _request(body: Mapping[str, object], path: str) -> Request:
|
||||
payload: Final = json.dumps(body).encode("utf-8")
|
||||
|
||||
async def receive() -> dict[str, object]:
|
||||
return {"type": "http.request", "body": payload, "more_body": False}
|
||||
|
||||
return Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": path,
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
"query_string": b"",
|
||||
},
|
||||
receive=receive,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _install(monkeypatch: pytest.MonkeyPatch, api_base: str | None) -> Mock:
|
||||
monkeypatch.setenv("XAI_API_KEY", "xai-test-key")
|
||||
if api_base is None:
|
||||
monkeypatch.delenv("XAI_API_BASE", raising=False)
|
||||
else:
|
||||
monkeypatch.setenv("XAI_API_BASE", api_base)
|
||||
create_route: Final = Mock(return_value=AsyncMock(return_value={"ok": True}))
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
|
||||
create_route,
|
||||
)
|
||||
return create_route
|
||||
|
||||
@pytest.fixture
|
||||
def client(self, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]:
|
||||
from litellm.proxy.proxy_server import app
|
||||
|
||||
monkeypatch.setenv("XAI_API_KEY", "xai-test-key")
|
||||
monkeypatch.delenv("XAI_API_BASE", raising=False)
|
||||
monkeypatch.delenv("SERVER_ROOT_PATH", raising=False)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, lambda: UserAPIKeyAuth(api_key="sk-virtual"))
|
||||
yield TestClient(app)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"api_base, expected_target",
|
||||
[
|
||||
(None, "https://api.x.ai/v1/responses"),
|
||||
("https://api.x.ai", "https://api.x.ai/v1/responses"),
|
||||
("https://xai.example/gateway", "https://xai.example/gateway/v1/responses"),
|
||||
],
|
||||
)
|
||||
async def test_target_carries_the_callers_version_path_exactly_once(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
api_base: str | None,
|
||||
expected_target: str,
|
||||
) -> None:
|
||||
create_route: Final = self._install(monkeypatch, api_base)
|
||||
|
||||
result: Final = await xai_proxy_route(
|
||||
endpoint="v1/responses",
|
||||
request=self._request({"model": "grok-4.20", "input": "ping"}, "/xai/v1/responses"),
|
||||
fastapi_response=MagicMock(spec=Response),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
|
||||
)
|
||||
|
||||
assert result == {"ok": True}
|
||||
create_route.assert_called_once_with(
|
||||
endpoint="v1/responses",
|
||||
target=expected_target,
|
||||
custom_headers={"Authorization": "Bearer xai-test-key"},
|
||||
is_streaming_request=False,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream, expected", [(True, True), (False, False)])
|
||||
async def test_streaming_flag_follows_the_request_body(
|
||||
self,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
stream: bool,
|
||||
expected: bool,
|
||||
) -> None:
|
||||
create_route: Final = self._install(monkeypatch, None)
|
||||
|
||||
await xai_proxy_route(
|
||||
endpoint="v1/chat/completions",
|
||||
request=self._request(
|
||||
{"model": "grok-4.20", "messages": [], "stream": stream},
|
||||
"/xai/v1/chat/completions",
|
||||
),
|
||||
fastapi_response=MagicMock(spec=Response),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
|
||||
)
|
||||
|
||||
assert create_route.call_args.kwargs["is_streaming_request"] is expected
|
||||
|
||||
def test_native_responses_call_reaches_xai_with_the_resolved_credential(self, client: TestClient) -> None:
|
||||
with respx.mock(assert_all_called=True) as upstream:
|
||||
route = upstream.post("https://api.x.ai/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json={"id": "resp_123"})
|
||||
)
|
||||
response = client.post("/xai/v1/responses", json={"model": "grok-4.20", "input": "ping"})
|
||||
|
||||
assert (response.status_code, response.json()) == (200, {"id": "resp_123"})
|
||||
sent: Final = route.calls.last.request
|
||||
assert sent.headers["authorization"] == "Bearer xai-test-key"
|
||||
assert json.loads(sent.content) == {"model": "grok-4.20", "input": "ping"}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue