From f4d012c9a3836dea2fd3aca82fcb1bea9a7f3baa Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Tue, 8 Sep 2026 10:55:50 +0000 Subject: [PATCH 1/9] feat(xai): add grok imagine image generation, edit, and video download Map OpenAI /images/generations, /images/edits, and /videos to xAI Imagine, including CDN video download and existing SuperGrok OAuth or API-key auth Co-authored-by: HX --- litellm/images/main.py | 3 +- litellm/llms/xai/image_edit/__init__.py | 3 + litellm/llms/xai/image_edit/transformation.py | 237 ++++++++++ litellm/llms/xai/image_generation/__init__.py | 11 + .../xai/image_generation/transformation.py | 207 +++++++++ litellm/llms/xai/videos/__init__.py | 3 + litellm/llms/xai/videos/transformation.py | 409 ++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 75 +++- .../provider_endpoints_support_backup.json | 6 +- litellm/proxy/image_endpoints/endpoints.py | 22 +- litellm/proxy/video_endpoints/endpoints.py | 27 +- litellm/proxy/video_endpoints/utils.py | 23 +- litellm/utils.py | 14 + litellm/videos/main.py | 66 ++- model_prices_and_context_window.json | 75 +++- provider_endpoints_support.json | 6 +- .../llms/xai/test_xai_image_edit.py | 167 +++++++ .../llms/xai/test_xai_image_generation.py | 129 ++++++ .../llms/xai/test_xai_video_generation.py | 206 +++++++++ .../proxy/image_endpoints/test_endpoints.py | 48 ++ .../proxy/video_endpoints/test_endpoints.py | 70 +++ .../proxy/video_endpoints/test_utils.py | 47 ++ tests/test_litellm/videos/test_main.py | 24 + 23 files changed, 1826 insertions(+), 52 deletions(-) create mode 100644 litellm/llms/xai/image_edit/__init__.py create mode 100644 litellm/llms/xai/image_edit/transformation.py create mode 100644 litellm/llms/xai/image_generation/__init__.py create mode 100644 litellm/llms/xai/image_generation/transformation.py create mode 100644 litellm/llms/xai/videos/__init__.py create mode 100644 litellm/llms/xai/videos/transformation.py create mode 100644 tests/test_litellm/llms/xai/test_xai_image_edit.py create mode 100644 tests/test_litellm/llms/xai/test_xai_image_generation.py create mode 100644 tests/test_litellm/llms/xai/test_xai_video_generation.py diff --git a/litellm/images/main.py b/litellm/images/main.py index 6a94e7c8df2..3da6b4aa030 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -388,6 +388,7 @@ def image_generation( litellm.LlmProviders.DASHSCOPE, litellm.LlmProviders.QWENCLOUD, litellm.LlmProviders.QWEN_AI_PLATFORM, + litellm.LlmProviders.XAI, ): if image_generation_config is None: raise ValueError(f"image generation config is not supported for {custom_llm_provider}") @@ -397,7 +398,7 @@ def image_generation( litellm_params_dict["api_base"] = _api_base return llm_http_handler.image_generation_handler( - api_key=api_key, + api_key=api_key or dynamic_api_key, model=model, prompt=prompt, image_generation_provider_config=image_generation_config, diff --git a/litellm/llms/xai/image_edit/__init__.py b/litellm/llms/xai/image_edit/__init__.py new file mode 100644 index 00000000000..5cae7020a92 --- /dev/null +++ b/litellm/llms/xai/image_edit/__init__.py @@ -0,0 +1,3 @@ +from .transformation import XAIImageEditConfig + +__all__ = ["XAIImageEditConfig"] diff --git a/litellm/llms/xai/image_edit/transformation.py b/litellm/llms/xai/image_edit/transformation.py new file mode 100644 index 00000000000..4ba55f6c339 --- /dev/null +++ b/litellm/llms/xai/image_edit/transformation.py @@ -0,0 +1,237 @@ +import base64 +from io import BufferedReader, BytesIO +from typing import Any, Final + +import httpx +from httpx._types import RequestFiles + +from litellm.constants import XAI_API_BASE +from litellm.exceptions import AuthenticationError +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import FileTypes, ImageObject, ImageResponse + +_SIZE_TO_ASPECT_RATIO: Final = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1536x1024": "3:2", + "1024x1536": "2:3", + "1280x720": "16:9", + "720x1280": "9:16", + "1920x1080": "16:9", + "1080x1920": "9:16", +} +_XAI_NATIVE_PARAMS: Final = frozenset({"aspect_ratio", "n", "resolution"}) + + +def _read_seekable(image: BytesIO | BufferedReader) -> bytes: + current_pos: Final = image.tell() + image.seek(0) + data: Final = image.read() + image.seek(current_pos) + return data + + +class XAIImageEditConfig(BaseImageEditConfig): + def get_supported_openai_params(self, model: str) -> list: + return ["n", "response_format", "size", "user"] + + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: + supported: Final = frozenset(self.get_supported_openai_params(model)) + allowed: Final = supported | _XAI_NATIVE_PARAMS + incoming: Final = dict(image_edit_optional_params) + unknown: Final = tuple(key for key in incoming if key not in allowed) + if unknown and not drop_params: + raise ValueError( + f"Parameter {unknown[0]} is not supported for model {model}. " + f"Supported parameters are {sorted(allowed)}. " + "Set drop_params=True to drop unsupported parameters." + ) + + mapped: Final = {key: value for key, value in incoming.items() if key in allowed} + size: Final = mapped.get("size") + aspect_ratio: Final = mapped.get("aspect_ratio") or ( + _SIZE_TO_ASPECT_RATIO.get(str(size), "1:1") if size else None + ) + n: Final = mapped.get("n") + resolution: Final = mapped.get("resolution") + return { + **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), + **({"n": int(n)} if n is not None else {}), + **({"resolution": resolution} if resolution is not None else {}), + } + + def use_multipart_form_data(self) -> bool: + return False + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, + ) -> str: + from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth + + api_key: Final = litellm_params.get("api_key") if isinstance(litellm_params, dict) else None + resolved_base: Final = ( + XAIOAuthAuthenticator().get_api_base() + if should_use_xai_oauth(litellm_params) and not XAIModelInfo.get_api_key(api_key) + else ( + api_base + or get_secret_str("XAI_API_BASE") + or get_secret_str("XAI_OAUTH_API_BASE") + or XAI_API_BASE + ) + ) + base: Final = (resolved_base or XAI_API_BASE).rstrip("/") + if base.endswith("/v1"): + return f"{base}/images/edits" + return f"{base}/v1/images/edits" + + def validate_environment( + self, + headers: dict, + model: str, + api_key: str | None = None, + litellm_params: dict | None = None, + api_base: str | None = None, + ) -> dict: + from litellm.llms.xai.oauth import ( + XAIOAuthAuthenticator, + XAIOAuthError, + should_use_xai_oauth, + ) + + params: Final = litellm_params or {} + dynamic_api_key: Final = XAIModelInfo.get_api_key(api_key) + if should_use_xai_oauth(params) and not dynamic_api_key: + try: + headers["Authorization"] = f"Bearer {XAIOAuthAuthenticator().get_access_token()}" + except XAIOAuthError as exc: + raise AuthenticationError( + model=model, + llm_provider="xai", + message=str(exc), + ) from exc + else: + if not dynamic_api_key: + raise AuthenticationError( + model=model, + llm_provider="xai", + message=( + "Missing xAI credentials for image edit. " + "Pass api_key / XAI_API_KEY, or set use_xai_oauth=True." + ), + ) + headers["Authorization"] = f"Bearer {dynamic_api_key}" + + if "content-type" not in headers and "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + return headers + + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: FileTypes | None, + image_edit_optional_request_params: dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[dict, RequestFiles]: + if image is None: + raise ValueError("xAI image edit requires at least one reference image.") + + image_payloads: Final = tuple(self._to_image_url(item) for item in self._as_image_list(image)) + if not image_payloads: + raise ValueError("xAI image edit requires at least one reference image.") + + n: Final = image_edit_optional_request_params.get("n") + request: Final[dict[str, Any]] = { + "model": XAIModelInfo.get_base_model(model) or model, + **({"prompt": prompt} if prompt is not None else {}), + **( + {"image": image_payloads[0]} + if len(image_payloads) == 1 + else {"images": list(image_payloads)} + ), + **{ + key: image_edit_optional_request_params[key] + for key in ("aspect_ratio", "resolution") + if image_edit_optional_request_params.get(key) is not None + }, + **({"n": int(n)} if n is not None else {}), + } + return request, [] + + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: Any, + ) -> ImageResponse: + try: + response_data: Final = raw_response.json() + except Exception: + raise self.get_error_class( + error_message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + images: Final = tuple( + ImageObject( + url=item.get("url"), + b64_json=item.get("b64_json") or item.get("b64"), + ) + for item in response_data.get("data") or () + if isinstance(item, dict) + ) + if not images: + raise self.get_error_class( + error_message=f"xAI image edit returned no image data: {response_data}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + return ImageResponse(data=list(images)) + + def _as_image_list(self, image: FileTypes | list[FileTypes]) -> tuple[FileTypes, ...]: + if isinstance(image, list): + return tuple(item for item in image if item is not None) + return (image,) + + def _to_image_url(self, image: FileTypes) -> dict[str, str]: + if isinstance(image, str): + return {"url": image} + if isinstance(image, dict): + if image.get("url"): + return {"url": str(image["url"])} + if image.get("file_id"): + return {"file_id": str(image["file_id"])} + + mime: Final = ImageEditRequestUtils.get_image_content_type(image) + encoded: Final = base64.b64encode(self._read_all_bytes(image)).decode("utf-8") + return {"url": f"data:{mime};base64,{encoded}"} + + def _read_all_bytes(self, image: FileTypes) -> bytes: + if isinstance(image, bytes): + return image + if isinstance(image, bytearray): + return bytes(image) + if isinstance(image, (BytesIO, BufferedReader)): + return _read_seekable(image) + if hasattr(image, "read"): + raw: Final = image.read() + if isinstance(raw, str): + return raw.encode("utf-8") + return bytes(raw) + raise ValueError(f"Unsupported image input type for xAI image edit: {type(image)}") diff --git a/litellm/llms/xai/image_generation/__init__.py b/litellm/llms/xai/image_generation/__init__.py new file mode 100644 index 00000000000..cc249fdf9ff --- /dev/null +++ b/litellm/llms/xai/image_generation/__init__.py @@ -0,0 +1,11 @@ +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) + +from .transformation import XAIImageGenerationConfig + +__all__ = ["XAIImageGenerationConfig", "get_xai_image_generation_config"] + + +def get_xai_image_generation_config(model: str) -> BaseImageGenerationConfig: + return XAIImageGenerationConfig() diff --git a/litellm/llms/xai/image_generation/transformation.py b/litellm/llms/xai/image_generation/transformation.py new file mode 100644 index 00000000000..6dd8318a139 --- /dev/null +++ b/litellm/llms/xai/image_generation/transformation.py @@ -0,0 +1,207 @@ +from typing import TYPE_CHECKING, Any, Final + +import httpx + +from litellm.constants import XAI_API_BASE +from litellm.exceptions import AuthenticationError +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +_SIZE_TO_ASPECT_RATIO: Final = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1536x1024": "3:2", + "1024x1536": "2:3", + "1280x720": "16:9", + "720x1280": "9:16", + "1920x1080": "16:9", + "1080x1920": "9:16", +} +_XAI_NATIVE_PARAMS: Final = frozenset({"aspect_ratio", "n"}) + + +class XAIImageGenerationConfig(BaseImageGenerationConfig): + def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: + return ["n", "response_format", "size", "user"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params: Final = frozenset(self.get_supported_openai_params(model)) + allowed: Final = supported_params | _XAI_NATIVE_PARAMS + unknown: Final = tuple( + key + for key in non_default_params + if key not in optional_params and key not in allowed + ) + if unknown and not drop_params: + raise ValueError( + f"Parameter {unknown[0]} is not supported for model {model}. " + f"Supported parameters are {sorted(allowed)}. " + "Set drop_params=True to drop unsupported parameters." + ) + + merged: Final = {**optional_params, **{k: v for k, v in non_default_params.items() if k in allowed}} + size: Final = merged.get("size") + aspect_ratio: Final = merged.get("aspect_ratio") or ( + _SIZE_TO_ASPECT_RATIO.get(str(size), "1:1") if size else None + ) + n: Final = merged.get("n") + return { + **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), + **({"n": int(n)} if n is not None else {}), + } + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict, + litellm_params: dict, + stream: bool | None = None, + ) -> str: + from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth + + resolved_base: Final = ( + XAIOAuthAuthenticator().get_api_base() + if should_use_xai_oauth(litellm_params) and not XAIModelInfo.get_api_key(api_key) + else ( + api_base + or get_secret_str("XAI_API_BASE") + or get_secret_str("XAI_OAUTH_API_BASE") + or XAI_API_BASE + ) + ) + base: Final = (resolved_base or XAI_API_BASE).rstrip("/") + if base.endswith("/v1"): + return f"{base}/images/generations" + return f"{base}/v1/images/generations" + + def validate_environment( + self, + headers: dict, + model: str, + messages: list[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict: + from litellm.llms.xai.oauth import ( + XAIOAuthAuthenticator, + XAIOAuthError, + should_use_xai_oauth, + ) + + dynamic_api_key: Final = XAIModelInfo.get_api_key(api_key) + if should_use_xai_oauth(litellm_params) and not dynamic_api_key: + try: + headers["Authorization"] = f"Bearer {XAIOAuthAuthenticator().get_access_token()}" + except XAIOAuthError as exc: + raise AuthenticationError( + model=model, + llm_provider="xai", + message=str(exc), + ) from exc + else: + if not dynamic_api_key: + raise AuthenticationError( + model=model, + llm_provider="xai", + message=( + "Missing xAI credentials for image generation. " + "Pass api_key / XAI_API_KEY, or set use_xai_oauth=True." + ), + ) + headers["Authorization"] = f"Bearer {dynamic_api_key}" + + if "content-type" not in headers and "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + return headers + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + n: Final = optional_params.get("n") + return { + "model": XAIModelInfo.get_base_model(model) or model, + "prompt": prompt, + **( + {"aspect_ratio": optional_params["aspect_ratio"]} + if optional_params.get("aspect_ratio") is not None + else {} + ), + **({"n": int(n)} if n is not None else {}), + } + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ImageResponse: + try: + response_data: Final = raw_response.json() + except Exception: + raise self.get_error_class( + error_message=raw_response.text, + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + + logging_obj.post_call( + input=request_data.get("prompt", ""), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=response_data, + ) + + images: Final = tuple( + ImageObject( + url=item.get("url"), + b64_json=item.get("b64_json") or item.get("b64"), + ) + for item in response_data.get("data") or () + if isinstance(item, dict) + ) + if not images: + raise self.get_error_class( + error_message=f"xAI image generation returned no image data: {response_data}", + status_code=raw_response.status_code, + headers=raw_response.headers, + ) + model_response.data = list(images) + return model_response diff --git a/litellm/llms/xai/videos/__init__.py b/litellm/llms/xai/videos/__init__.py new file mode 100644 index 00000000000..1e9330ffba1 --- /dev/null +++ b/litellm/llms/xai/videos/__init__.py @@ -0,0 +1,3 @@ +from .transformation import XAIVideoConfig + +__all__ = ["XAIVideoConfig"] diff --git a/litellm/llms/xai/videos/transformation.py b/litellm/llms/xai/videos/transformation.py new file mode 100644 index 00000000000..6d933a31357 --- /dev/null +++ b/litellm/llms/xai/videos/transformation.py @@ -0,0 +1,409 @@ +import time +from typing import TYPE_CHECKING, Any, Final + +import httpx +from httpx._types import RequestFiles + +import litellm +from litellm.constants import XAI_API_BASE +from litellm.exceptions import AuthenticationError +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.llms.xai.common_utils import XAIModelInfo +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import ( + encode_video_id_with_provider, + extract_original_video_id, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + +_SIZE_TO_ASPECT_RATIO: Final = { + "1024x1024": "1:1", + "1792x1024": "16:9", + "1024x1792": "9:16", + "1280x720": "16:9", + "720x1280": "9:16", + "1920x1080": "16:9", + "1080x1920": "9:16", +} +def _duration_from_seconds(seconds: object) -> int: + try: + return int(seconds) if seconds is not None else 6 + except (TypeError, ValueError): + return 6 + + +_STATUS_MAP: Final = { + "done": "completed", + "completed": "completed", + "succeeded": "completed", + "failed": "failed", + "expired": "failed", + "pending": "processing", + "processing": "processing", + "in_progress": "processing", +} + + +class XAIVideoConfig(BaseVideoConfig): + def get_supported_openai_params(self, model: str) -> list: + return [ + "model", + "prompt", + "input_reference", + "seconds", + "size", + "user", + "extra_headers", + ] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: + incoming: Final = dict(video_create_optional_params) + size: Final = incoming.get("size") + return { + **{ + key: value + for key, value in incoming.items() + if key not in {"seconds", "size", "input_reference", "user", "extra_headers", "model"} + }, + **( + {"duration": _duration_from_seconds(incoming.get("seconds"))} + if "seconds" in incoming and "duration" not in incoming + else {} + ), + **( + {"aspect_ratio": incoming.get("aspect_ratio") or _SIZE_TO_ASPECT_RATIO.get(str(size), "16:9")} + if size and "aspect_ratio" not in incoming + else {} + ), + **( + {"image": incoming.get("image") or incoming.get("input_reference")} + if incoming.get("input_reference") and "image" not in incoming + else {} + ), + } + + def _resolve_api_base( + self, + api_base: str | None, + api_key: str | None, + litellm_params: GenericLiteLLMParams | dict | None, + ) -> str: + from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth + + params: Final = ( + litellm_params.model_dump() + if isinstance(litellm_params, GenericLiteLLMParams) + else (litellm_params or {}) + ) + if should_use_xai_oauth(params) and not XAIModelInfo.get_api_key(api_key): + return XAIOAuthAuthenticator().get_api_base().rstrip("/") + + resolved: Final = ( + api_base + or (params.get("api_base") if isinstance(params, dict) else None) + or get_secret_str("XAI_API_BASE") + or get_secret_str("XAI_OAUTH_API_BASE") + or XAI_API_BASE + ) + return str(resolved).rstrip("/") + + def _v1_root(self, api_base: str) -> str: + base: Final = api_base.rstrip("/") + if base.endswith("/v1"): + return base + return f"{base}/v1" + + def validate_environment( + self, + headers: dict, + model: str, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> dict: + from litellm.llms.xai.oauth import ( + XAIOAuthAuthenticator, + XAIOAuthError, + should_use_xai_oauth, + ) + + params: Final = litellm_params.model_dump() if litellm_params is not None else {} + resolved_api_key: Final = api_key or (litellm_params.api_key if litellm_params else None) + dynamic_api_key: Final = XAIModelInfo.get_api_key(resolved_api_key) + if should_use_xai_oauth(params) and not dynamic_api_key: + try: + headers["Authorization"] = f"Bearer {XAIOAuthAuthenticator().get_access_token()}" + except XAIOAuthError as exc: + raise AuthenticationError( + model=model or "xai-video", + llm_provider="xai", + message=str(exc), + ) from exc + else: + if not dynamic_api_key: + raise AuthenticationError( + model=model or "xai-video", + llm_provider="xai", + message=( + "Missing xAI credentials for video generation. " + "Pass api_key / XAI_API_KEY, or set use_xai_oauth=True." + ), + ) + headers["Authorization"] = f"Bearer {dynamic_api_key}" + + if "content-type" not in headers and "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, + ) -> str: + resolved: Final = self._resolve_api_base( + api_base=api_base, + api_key=litellm_params.get("api_key") if litellm_params else None, + litellm_params=litellm_params, + ) + if not model: + return self._v1_root(resolved) + return f"{self._v1_root(resolved)}/videos/generations" + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[dict, RequestFiles, str]: + copied: Final = { + key: video_create_optional_request_params[key] + for key in ( + "image", + "images", + "duration", + "resolution_name", + "aspect_ratio", + "size", + ) + if video_create_optional_request_params.get(key) is not None + } + return ( + { + "model": XAIModelInfo.get_base_model(model) or model, + **({"prompt": prompt} if prompt else {}), + **copied, + **({"duration": 6} if "duration" not in copied else {}), + }, + [], + api_base, + ) + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + request_data: dict | None = None, + ) -> VideoObject: + response_data: Final = raw_response.json() + request_id: Final = response_data.get("request_id") or response_data.get("id") + if not request_id: + raise ValueError(f"xAI video generation response missing request_id: {response_data}") + + usage: Final = response_data.get("usage") or {} + video_obj: Final = VideoObject( + id=str(request_id), + object="video", + status="processing", + created_at=int(time.time()), + model=XAIModelInfo.get_base_model(model) or model, + progress=0, + ) + if custom_llm_provider: + video_obj.id = encode_video_id_with_provider( + video_obj.id, custom_llm_provider, model + ) + video_obj.usage = usage if isinstance(usage, dict) else {} + video_obj._hidden_params["video_url"] = None + return video_obj + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[str, dict]: + original_id: Final = extract_original_video_id(video_id) + return f"{self._v1_root(api_base)}/videos/{original_id}", {} + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> VideoObject: + response_data: Final = raw_response.json() + status_raw: Final = str(response_data.get("status") or "processing").lower() + status: Final = _STATUS_MAP.get(status_raw, status_raw) + video_meta: Final = response_data.get("video") or {} + video_url: Final = video_meta.get("url") if isinstance(video_meta, dict) else None + seconds: Final = ( + str(video_meta.get("duration")) + if isinstance(video_meta, dict) and video_meta.get("duration") is not None + else None + ) + request_id: Final = ( + response_data.get("request_id") + or response_data.get("id") + or (video_url.split("/")[-1].replace(".mp4", "") if video_url else "unknown") + ) + video_obj: Final = VideoObject( + id=str(request_id), + object="video", + status=status, + created_at=response_data.get("created_at") or int(time.time()), + completed_at=int(time.time()) if status == "completed" else None, + model=response_data.get("model"), + progress=response_data.get("progress"), + seconds=seconds, + usage=response_data.get("usage") if isinstance(response_data.get("usage"), dict) else {}, + ) + video_obj._hidden_params["video_url"] = video_url + if custom_llm_provider and video_obj.id and video_obj.id != "unknown": + video_obj.id = encode_video_id_with_provider( + video_obj.id, custom_llm_provider, response_data.get("model") + ) + return video_obj + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + variant: str | None = None, + ) -> tuple[str, dict]: + original_id: Final = extract_original_video_id(video_id) + return f"{self._v1_root(api_base)}/videos/{original_id}", {} + + def _video_cdn_url(self, raw_response: httpx.Response) -> str | None: + content_type: Final = (raw_response.headers.get("content-type") or "").lower() + if "application/json" not in content_type and raw_response.content[:1] != b"{": + return None + payload: Final = raw_response.json() + if not isinstance(payload, dict): + return None + video_meta: Final = payload.get("video") or {} + url: Final = video_meta.get("url") if isinstance(video_meta, dict) else None + if isinstance(url, str) and url: + return url + raise ValueError( + f"xAI video not ready for download (status={payload.get('status')}): {payload}" + ) + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + url: Final = self._video_cdn_url(raw_response) + if url is None: + return raw_response.content + httpx_client: Final[HTTPHandler] = _get_httpx_client() + video_response: Final = httpx_client.get(url) + video_response.raise_for_status() + return video_response.content + + async def async_transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> bytes: + url: Final = self._video_cdn_url(raw_response) + if url is None: + return raw_response.content + async_httpx_client: Final[AsyncHTTPHandler] = get_async_httpx_client( + llm_provider=litellm.LlmProviders.XAI, + ) + video_response: Final = await async_httpx_client.get(url) + video_response.raise_for_status() + return video_response.content + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: dict[str, Any] | None = None, + ) -> tuple[str, dict]: + raise NotImplementedError("Video remix is not supported by xAI Imagine API") + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("Video remix is not supported by xAI Imagine API") + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, Any] | None = None, + ) -> tuple[str, dict]: + raise NotImplementedError("Video listing is not supported by xAI Imagine API") + + def transform_video_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> dict[str, str]: + raise NotImplementedError("Video listing is not supported by xAI Imagine API") + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> tuple[str, dict]: + raise NotImplementedError("Video delete is not supported by xAI Imagine API") + + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> VideoObject: + raise NotImplementedError("Video delete is not supported by xAI Imagine API") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b1ffc1583e4..38bab1291da 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59300,7 +59300,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59316,7 +59317,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59332,7 +59334,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59348,7 +59351,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59364,7 +59368,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59380,7 +59385,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59397,7 +59403,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59413,7 +59420,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -63792,5 +63800,56 @@ "supports_tool_choice": false, "supports_response_schema": true, "supports_vision": false + }, + "xai/grok-imagine-video": { + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/videos", + "/v1/videos/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5": { + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.08, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/videos", + "/v1/videos/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-preview": { + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.08, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/videos", + "/v1/videos/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index dbeaccdda2d..7da6a2e9a9a 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2342,7 +2342,8 @@ "messages": true, "responses": true, "embeddings": false, - "image_generations": false, + "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, "moderations": false, @@ -2350,7 +2351,8 @@ "rerank": false, "a2a": true, "interactions": true, - "realtime": true + "realtime": true, + "video_generations": true } }, "xinference": { diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 06f99e4ae9c..ca3f9333d83 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -299,12 +299,22 @@ async def image_edit_api( if mask_files: data["mask"] = mask_files - for _field in ("image", "mask"): - if _field in data and isinstance(data[_field], str): - raise HTTPException( - status_code=422, - detail=f"'{_field}' must be provided as a multipart file upload, not a string.", - ) + invalid_image_fields: Final = tuple( + field + for field in ("image", "mask") + if field in data + and isinstance(data[field], str) + and not ( + data[field].startswith("http://") + or data[field].startswith("https://") + or data[field].startswith("data:image/") + ) + ) + if invalid_image_fields: + raise HTTPException( + status_code=422, + detail=f"'{invalid_image_fields[0]}' must be a multipart file, http(s) URL, or data:image URI.", + ) # Ensure prompt exists in data (default to None for models that don't require it) if "prompt" not in data: diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 66071c05b4f..9cc1f033326 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -20,6 +20,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + resolve_video_request_model, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -262,12 +263,13 @@ async def video_status( if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) - if resolved_model: - data["model"] = resolved_model + resolved_model: Final = resolve_video_request_model( + model_id_from_decoded=model_id_from_decoded, + query_model=request.query_params.get("model"), + llm_router=llm_router, + ) + if resolved_model: + data["model"] = resolved_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) @@ -360,12 +362,13 @@ async def video_content( if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider - # Resolve model_name from model_id if available - # This allows the router to automatically inject litellm_params from the model config - if model_id_from_decoded and llm_router: - resolved_model: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) - if resolved_model: - data["model"] = resolved_model + resolved_content_model: Final = resolve_video_request_model( + model_id_from_decoded=model_id_from_decoded, + query_model=request.query_params.get("model"), + llm_router=llm_router, + ) + if resolved_content_model: + data["model"] = resolved_content_model # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index a38226cc253..af0e9548568 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -1,10 +1,31 @@ -from typing import Any, Final +from typing import Any, Final, Protocol import orjson from litellm.types.videos.utils import encode_character_id_with_provider +class VideoModelIdResolver(Protocol): + def resolve_model_name_from_model_id(self, model_id: str | None) -> str | None: ... + + +def resolve_video_request_model( + *, + model_id_from_decoded: str | None, + query_model: str | None, + llm_router: VideoModelIdResolver | None, +) -> str | None: + if model_id_from_decoded: + if llm_router is not None: + resolved: Final = llm_router.resolve_model_name_from_model_id(model_id_from_decoded) + if isinstance(resolved, str) and resolved: + return resolved + return model_id_from_decoded + if isinstance(query_model, str) and query_model: + return query_model + return None + + def extract_model_from_target_model_names(target_model_names: Any) -> str | None: if isinstance(target_model_names, str): target_model_names = [m.strip() for m in target_model_names.split(",") if m.strip()] diff --git a/litellm/utils.py b/litellm/utils.py index d0e11bc9551..56141a16b5c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9136,6 +9136,12 @@ class ProviderConfigManager: ) return get_modelscope_image_generation_config(model) + elif LlmProviders.XAI == provider: + from litellm.llms.xai.image_generation import ( + get_xai_image_generation_config, + ) + + return get_xai_image_generation_config(model) return None @staticmethod @@ -9163,6 +9169,10 @@ class ProviderConfigManager: from litellm.llms.runwayml.videos.transformation import RunwayMLVideoConfig return RunwayMLVideoConfig() + elif LlmProviders.XAI == provider: + from litellm.llms.xai.videos.transformation import XAIVideoConfig + + return XAIVideoConfig() elif LlmProviders.HOSTED_VLLM == provider: from litellm.llms.hosted_vllm.videos import get_hosted_vllm_video_config @@ -9287,6 +9297,10 @@ class ProviderConfigManager: ) return get_openrouter_image_edit_config(model) + elif LlmProviders.XAI == provider: + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + return XAIImageEditConfig() return None @staticmethod diff --git a/litellm/videos/main.py b/litellm/videos/main.py index 445435a30fa..b95225d04c5 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -30,6 +30,51 @@ from litellm.videos.utils import VideoGenerationRequestUtils llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() +def _litellm_provider_from_cost_entry(info: object) -> str | None: + if isinstance(info, dict): + catalog_provider: Final = info.get("litellm_provider") + if isinstance(catalog_provider, str) and catalog_provider: + return catalog_provider + return None + + +def _provider_from_prefixed_model(model: str) -> str | None: + if "/" not in model: + return None + try: + _, provider, _, _ = get_llm_provider(model=model) + except Exception: + return None + return provider + + +def _custom_llm_provider_from_model(model: str) -> str | None: + return ( + _provider_from_prefixed_model(model) + or _litellm_provider_from_cost_entry(litellm.model_cost.get(model)) + or _litellm_provider_from_cost_entry(litellm.model_cost.get(f"xai/{model}")) + or ("xai" if model.startswith("grok-imagine-video") else None) + ) + + +def _provider_for_video_id( + video_id: str, + custom_llm_provider: str | None, + model: object | None = None, +) -> str: + if custom_llm_provider is not None: + return custom_llm_provider + decoded: Final = decode_video_id_with_provider(video_id) + from_id: Final = decoded.get("custom_llm_provider") + if from_id: + return from_id + if isinstance(model, str) and model: + from_model: Final = _custom_llm_provider_from_model(model) + if from_model: + return from_model + return "openai" + + ##### Video Generation ####################### @client async def avideo_generation( @@ -317,10 +362,9 @@ def video_content( litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("async_call", False) is True - # Try to decode provider from video_id if not explicitly provided - if custom_llm_provider is None: - decoded: Final = decode_video_id_with_provider(video_id) - custom_llm_provider = decoded.get("custom_llm_provider") or "openai" + custom_llm_provider = _provider_for_video_id( + video_id, custom_llm_provider, kwargs.get("model") + ) # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) @@ -412,10 +456,9 @@ async def avideo_content( loop: Final = asyncio.get_event_loop() kwargs["async_call"] = True - # Try to decode provider from video_id if not explicitly provided - if custom_llm_provider is None: - decoded: Final = decode_video_id_with_provider(video_id) - custom_llm_provider = decoded.get("custom_llm_provider") or "openai" + custom_llm_provider = _provider_for_video_id( + video_id, custom_llm_provider, kwargs.get("model") + ) func: Final = partial( video_content, @@ -1019,10 +1062,9 @@ def video_status( response: Final = VideoObject(**mock_response) return response - # Try to decode provider from video_id if not explicitly provided - if custom_llm_provider is None: - decoded: Final = decode_video_id_with_provider(video_id) - custom_llm_provider = decoded.get("custom_llm_provider") or "openai" + custom_llm_provider = _provider_for_video_id( + video_id, custom_llm_provider, kwargs.get("model") + ) # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1ffc1583e4..38bab1291da 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59300,7 +59300,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59316,7 +59317,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59332,7 +59334,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59348,7 +59351,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59364,7 +59368,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59380,7 +59385,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59397,7 +59403,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -59413,7 +59420,8 @@ "mode": "image_generation", "source": "https://docs.x.ai/docs/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ], "supported_modalities": [ "text", @@ -63792,5 +63800,56 @@ "supports_tool_choice": false, "supports_response_schema": true, "supports_vision": false + }, + "xai/grok-imagine-video": { + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.05, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/videos", + "/v1/videos/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5": { + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.08, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/videos", + "/v1/videos/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, + "xai/grok-imagine-video-1.5-preview": { + "litellm_provider": "xai", + "mode": "video_generation", + "output_cost_per_video_per_second": 0.08, + "source": "https://docs.x.ai/docs/models", + "supported_endpoints": [ + "/v1/videos", + "/v1/videos/generations" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index c71f4a82a4a..c3855a7a7de 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2653,7 +2653,8 @@ "messages": true, "responses": true, "embeddings": false, - "image_generations": false, + "image_generations": true, + "image_edits": true, "audio_transcriptions": false, "audio_speech": false, "moderations": false, @@ -2661,7 +2662,8 @@ "rerank": false, "a2a": true, "interactions": true, - "realtime": true + "realtime": true, + "video_generations": true } }, "xinference": { diff --git a/tests/test_litellm/llms/xai/test_xai_image_edit.py b/tests/test_litellm/llms/xai/test_xai_image_edit.py new file mode 100644 index 00000000000..575cbf459b6 --- /dev/null +++ b/tests/test_litellm/llms/xai/test_xai_image_edit.py @@ -0,0 +1,167 @@ +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +def test_provider_config_manager_returns_xai_image_edit_config(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + config = ProviderConfigManager.get_provider_image_edit_config( + model="grok-imagine-image", + provider=LlmProviders.XAI, + ) + assert isinstance(config, XAIImageEditConfig) + + +def test_get_complete_url_default(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + url = XAIImageEditConfig().get_complete_url( + model="grok-imagine-image", + api_base=None, + litellm_params={}, + ) + assert url.endswith("/v1/images/edits") + + +def test_uses_json_not_multipart(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + assert XAIImageEditConfig().use_multipart_form_data() is False + + +def test_map_size_to_aspect_ratio(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + mapped = XAIImageEditConfig().map_openai_params( + image_edit_optional_params={"size": "1024x1792", "n": 1}, + model="grok-imagine-image", + drop_params=True, + ) + assert mapped["aspect_ratio"] == "9:16" + assert mapped["n"] == 1 + assert "size" not in mapped + + +def test_validate_environment_requires_credentials(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + with pytest.raises(Exception, match="Missing xAI credentials"): + XAIImageEditConfig().validate_environment( + headers={}, + model="grok-imagine-image", + api_key=None, + litellm_params={}, + ) + + +def test_validate_environment_oauth_injects_bearer(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + with patch( + "litellm.llms.xai.oauth.XAIOAuthAuthenticator.get_access_token", + return_value="oauth-token", + ): + headers = XAIImageEditConfig().validate_environment( + headers={}, + model="grok-imagine-image", + api_key=None, + litellm_params={"use_xai_oauth": True}, + ) + assert headers["Authorization"] == "Bearer oauth-token" + + +def test_map_string_n_to_int(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + mapped = XAIImageEditConfig().map_openai_params( + image_edit_optional_params={"n": "1"}, + model="grok-imagine-image", + drop_params=True, + ) + assert mapped["n"] == 1 + assert isinstance(mapped["n"], int) + + +def test_transform_string_n_to_int(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + data, _ = XAIImageEditConfig().transform_image_edit_request( + model="xai/grok-imagine-image", + prompt="make the cube red", + image="https://imgen.x.ai/source.jpeg", + image_edit_optional_request_params={"n": "1"}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert data["n"] == 1 + assert isinstance(data["n"], int) + + +def test_transform_bytes_to_data_uri_and_response(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + config = XAIImageEditConfig() + data, files = config.transform_image_edit_request( + model="xai/grok-imagine-image", + prompt="make the cube red", + image=b"\xff\xd8\xfffakejpeg", + image_edit_optional_request_params={"aspect_ratio": "1:1"}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert files == [] + assert data["model"] == "grok-imagine-image" + assert data["prompt"] == "make the cube red" + assert data["aspect_ratio"] == "1:1" + assert data["image"]["url"].startswith("data:image/jpeg;base64,") + + raw = httpx.Response( + 200, + json={"data": [{"url": "https://imgen.x.ai/edited.jpeg", "mime_type": "image/jpeg"}]}, + ) + response = config.transform_image_edit_response( + model="grok-imagine-image", + raw_response=raw, + logging_obj=MagicMock(), + ) + assert response.data is not None + assert response.data[0].url == "https://imgen.x.ai/edited.jpeg" + + +def test_transform_http_url_passthrough(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + data, files = XAIImageEditConfig().transform_image_edit_request( + model="grok-imagine-image", + prompt="make it night", + image="https://imgen.x.ai/source.jpeg", + image_edit_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert files == [] + assert data["image"] == {"url": "https://imgen.x.ai/source.jpeg"} + + +def test_transform_multiple_images_uses_images_array(): + from litellm.llms.xai.image_edit.transformation import XAIImageEditConfig + + data, _ = XAIImageEditConfig().transform_image_edit_request( + model="grok-imagine-image", + prompt="combine styles", + image=["https://imgen.x.ai/a.jpeg", "https://imgen.x.ai/b.jpeg"], + image_edit_optional_request_params={}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert "image" not in data + assert data["images"] == [ + {"url": "https://imgen.x.ai/a.jpeg"}, + {"url": "https://imgen.x.ai/b.jpeg"}, + ] diff --git a/tests/test_litellm/llms/xai/test_xai_image_generation.py b/tests/test_litellm/llms/xai/test_xai_image_generation.py new file mode 100644 index 00000000000..a617388f34c --- /dev/null +++ b/tests/test_litellm/llms/xai/test_xai_image_generation.py @@ -0,0 +1,129 @@ +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +from litellm.llms.xai.image_generation.transformation import XAIImageGenerationConfig +from litellm.types.utils import ImageResponse, LlmProviders +from litellm.utils import ProviderConfigManager + + +def test_provider_config_manager_returns_xai_image_config(): + config = ProviderConfigManager.get_provider_image_generation_config( + model="grok-imagine-image", + provider=LlmProviders.XAI, + ) + assert isinstance(config, XAIImageGenerationConfig) + + +def test_map_size_to_aspect_ratio(): + mapped = XAIImageGenerationConfig().map_openai_params( + non_default_params={"size": "1024x1792"}, + optional_params={}, + model="grok-imagine-image", + drop_params=True, + ) + assert mapped["aspect_ratio"] == "9:16" + assert "size" not in mapped + + +def test_get_complete_url_default(): + url = XAIImageGenerationConfig().get_complete_url( + api_base=None, + api_key=None, + model="grok-imagine-image", + optional_params={}, + litellm_params={}, + ) + assert url.endswith("/v1/images/generations") + + +def test_validate_environment_requires_credentials(): + with pytest.raises(Exception, match="Missing xAI credentials"): + XAIImageGenerationConfig().validate_environment( + headers={}, + model="grok-imagine-image", + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + +def test_validate_environment_oauth_injects_bearer(): + with patch( + "litellm.llms.xai.oauth.XAIOAuthAuthenticator.get_access_token", + return_value="oauth-token", + ): + headers = XAIImageGenerationConfig().validate_environment( + headers={}, + model="grok-imagine-image", + messages=[], + optional_params={}, + litellm_params={"use_xai_oauth": True}, + api_key=None, + ) + assert headers["Authorization"] == "Bearer oauth-token" + + +def test_map_string_n_to_int(): + mapped = XAIImageGenerationConfig().map_openai_params( + non_default_params={"n": "1"}, + optional_params={}, + model="grok-imagine-image", + drop_params=True, + ) + assert mapped["n"] == 1 + assert isinstance(mapped["n"], int) + + +def test_transform_string_n_to_int(): + request = XAIImageGenerationConfig().transform_image_generation_request( + model="xai/grok-imagine-image", + prompt="a red apple", + optional_params={"n": "1"}, + litellm_params={}, + headers={}, + ) + assert request["n"] == 1 + assert isinstance(request["n"], int) + + +def test_transform_request_and_response(): + config = XAIImageGenerationConfig() + request = config.transform_image_generation_request( + model="xai/grok-imagine-image", + prompt="a red apple", + optional_params={"aspect_ratio": "1:1"}, + litellm_params={}, + headers={}, + ) + assert request == { + "model": "grok-imagine-image", + "prompt": "a red apple", + "aspect_ratio": "1:1", + } + + raw = httpx.Response( + 200, + json={ + "data": [ + { + "url": "https://imgen.x.ai/example.jpeg", + "mime_type": "image/jpeg", + } + ] + }, + ) + response = config.transform_image_generation_response( + model="grok-imagine-image", + raw_response=raw, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data=request, + optional_params={}, + litellm_params={}, + encoding=None, + ) + assert response.data is not None + assert response.data[0].url == "https://imgen.x.ai/example.jpeg" diff --git a/tests/test_litellm/llms/xai/test_xai_video_generation.py b/tests/test_litellm/llms/xai/test_xai_video_generation.py new file mode 100644 index 00000000000..fee57425264 --- /dev/null +++ b/tests/test_litellm/llms/xai/test_xai_video_generation.py @@ -0,0 +1,206 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from litellm.llms.xai.videos.transformation import XAIVideoConfig +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +def test_provider_config_manager_returns_xai_video_config(): + config = ProviderConfigManager.get_provider_video_config( + model="grok-imagine-video", + provider=LlmProviders.XAI, + ) + assert isinstance(config, XAIVideoConfig) + + +def test_map_seconds_and_size(): + mapped = XAIVideoConfig().map_openai_params( + video_create_optional_params={"seconds": "10", "size": "1280x720"}, + model="grok-imagine-video", + drop_params=True, + ) + assert mapped["duration"] == 10 + assert mapped["aspect_ratio"] == "16:9" + + +def test_map_seconds_nine_is_not_clamped_to_six(): + mapped = XAIVideoConfig().map_openai_params( + video_create_optional_params={"seconds": "9"}, + model="grok-imagine-video-1.5", + drop_params=True, + ) + assert mapped["duration"] == 9 + assert "seconds" not in mapped + + +def test_get_complete_url_create_and_status_root(): + config = XAIVideoConfig() + create_url = config.get_complete_url( + model="grok-imagine-video", + api_base="https://api.x.ai/v1", + litellm_params={}, + ) + assert create_url == "https://api.x.ai/v1/videos/generations" + + status_root = config.get_complete_url( + model="", + api_base="https://api.x.ai/v1", + litellm_params={}, + ) + assert status_root == "https://api.x.ai/v1" + + +def test_validate_environment_oauth_injects_bearer(): + with patch( + "litellm.llms.xai.oauth.XAIOAuthAuthenticator.get_access_token", + return_value="oauth-token", + ): + headers = XAIVideoConfig().validate_environment( + headers={}, + model="grok-imagine-video", + api_key=None, + litellm_params=GenericLiteLLMParams(use_xai_oauth=True), + ) + assert headers["Authorization"] == "Bearer oauth-token" + + +def test_transform_create_and_status_response(): + config = XAIVideoConfig() + data, files, api_base = config.transform_video_create_request( + model="xai/grok-imagine-video", + prompt="a cat walking", + api_base="https://api.x.ai/v1/videos/generations", + video_create_optional_request_params={"duration": 6}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert data["model"] == "grok-imagine-video" + assert data["prompt"] == "a cat walking" + assert data["duration"] == 6 + assert files == [] + + created = config.transform_video_create_response( + model="grok-imagine-video", + raw_response=httpx.Response(200, json={"request_id": "req-123"}), + logging_obj=MagicMock(), + custom_llm_provider="xai", + ) + assert created.status == "processing" + assert created.id + + status_url, params = config.transform_video_status_retrieve_request( + video_id=created.id, + api_base="https://api.x.ai/v1", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert status_url.endswith("/videos/req-123") + assert params == {} + + status = config.transform_video_status_retrieve_response( + raw_response=httpx.Response( + 200, + json={ + "status": "done", + "request_id": "req-123", + "video": {"url": "https://vidgen.x.ai/x.mp4", "duration": 6}, + "progress": 100, + "model": "grok-imagine-video", + }, + ), + logging_obj=MagicMock(), + custom_llm_provider="xai", + ) + assert status.status == "completed" + assert status.seconds == "6" + assert status._hidden_params.get("video_url") == "https://vidgen.x.ai/x.mp4" + + +def test_validate_environment_requires_credentials(): + with pytest.raises(Exception, match="Missing xAI credentials"): + XAIVideoConfig().validate_environment( + headers={}, + model="grok-imagine-video", + api_key=None, + litellm_params=GenericLiteLLMParams(), + ) + + +def test_content_request_is_get_status(): + url, params = XAIVideoConfig().transform_video_content_request( + video_id="req-123", + api_base="https://api.x.ai/v1", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert url == "https://api.x.ai/v1/videos/req-123" + assert params == {} + + +def test_content_response_fetches_cdn_via_shared_client(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "done", "video": {"url": "https://vidgen.x.ai/x.mp4"}}, + ) + cdn = MagicMock() + video_resp = MagicMock() + video_resp.content = b"mp4-bytes" + video_resp.raise_for_status.return_value = None + cdn.get.return_value = video_resp + with patch( + "litellm.llms.xai.videos.transformation._get_httpx_client", + return_value=cdn, + ): + body = XAIVideoConfig().transform_video_content_response(status, logging_obj=MagicMock()) + assert body == b"mp4-bytes" + cdn.get.assert_called_once_with("https://vidgen.x.ai/x.mp4") + + +@pytest.mark.asyncio +async def test_async_content_response_does_not_use_sync_client(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "done", "video": {"url": "https://vidgen.x.ai/x.mp4"}}, + ) + async_client = MagicMock() + video_resp = MagicMock() + video_resp.content = b"async-mp4" + video_resp.raise_for_status.return_value = None + async_client.get = AsyncMock(return_value=video_resp) + with patch( + "litellm.llms.xai.videos.transformation.get_async_httpx_client", + return_value=async_client, + ), patch( + "litellm.llms.xai.videos.transformation._get_httpx_client", + ) as sync_client: + body = await XAIVideoConfig().async_transform_video_content_response( + status, logging_obj=MagicMock() + ) + assert body == b"async-mp4" + sync_client.assert_not_called() + async_client.get.assert_awaited_once_with("https://vidgen.x.ai/x.mp4") + + +def test_content_response_raises_when_status_has_no_url(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "pending"}, + ) + with pytest.raises(ValueError, match="not ready"): + XAIVideoConfig().transform_video_content_response(status, logging_obj=MagicMock()) + + +def test_content_response_returns_raw_bytes_when_not_json(): + raw = httpx.Response( + 200, + headers={"content-type": "video/mp4"}, + content=b"already-mp4", + ) + assert XAIVideoConfig().transform_video_content_response(raw, logging_obj=MagicMock()) == b"already-mp4" diff --git a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py index 203391aadad..f017dedf9cd 100644 --- a/tests/test_litellm/proxy/image_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/image_endpoints/test_endpoints.py @@ -167,3 +167,51 @@ def test_image_edit_multipart_n_that_is_not_a_number_is_left_alone(monkeypatch): assert response.status_code == 200 assert captured["n"] == "two" + + +def test_image_edit_http_url_is_accepted(monkeypatch): + captured: Dict[str, Any] = {} + + response = _image_edit_client(monkeypatch, captured).post( + "/v1/images/edits", + data={ + "model": "grok-imagine-image", + "prompt": "make it night", + "image": "https://imgen.x.ai/source.jpeg", + }, + ) + + assert response.status_code == 200 + assert captured["image"] == "https://imgen.x.ai/source.jpeg" + + +def test_image_edit_data_uri_is_accepted(monkeypatch): + captured: Dict[str, Any] = {} + + response = _image_edit_client(monkeypatch, captured).post( + "/v1/images/edits", + data={ + "model": "grok-imagine-image", + "prompt": "make it red", + "image": "data:image/jpeg;base64,abc", + }, + ) + + assert response.status_code == 200 + assert captured["image"] == "data:image/jpeg;base64,abc" + + +def test_image_edit_plain_string_image_is_rejected(monkeypatch): + captured: Dict[str, Any] = {} + + response = _image_edit_client(monkeypatch, captured).post( + "/v1/images/edits", + data={ + "model": "grok-imagine-image", + "prompt": "make it red", + "image": "not-a-url-or-file", + }, + ) + + assert response.status_code == 422 + assert "multipart file" in response.json()["detail"] diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index c5996f95f54..eeb59e31988 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -309,6 +309,40 @@ async def test_status__header_provider_beats_decoded_id(harness): assert data["model"] == "azure-sora" +@pytest.mark.asyncio +async def test_status__resolve_fail_keeps_decoded_model_id(harness): + encoded = encode_video_id_with_provider( + "9b444cea-aaaa-bbbb-cccc-dddddddddddd", + "xai", + "grok-imagine-video-1.5", + ) + + await call_status(harness, encoded) + + harness.resolve_model.assert_called_once_with("grok-imagine-video-1.5") + assert harness.processor_data() == { + "video_id": encoded, + "custom_llm_provider": "xai", + "model": "grok-imagine-video-1.5", + } + + +@pytest.mark.asyncio +async def test_status__query_model_on_plain_id(harness): + await call_status( + harness, + "9b444cea-aaaa-bbbb-cccc-dddddddddddd", + query={"model": "grok-imagine-video-1.5"}, + ) + + harness.resolve_model.assert_not_called() + assert harness.processor_data() == { + "video_id": "9b444cea-aaaa-bbbb-cccc-dddddddddddd", + "custom_llm_provider": "openai", + "model": "grok-imagine-video-1.5", + } + + # =========================================================================== # # GET /v1/videos/{video_id}/content - video_content # # =========================================================================== # @@ -365,6 +399,42 @@ async def test_content__model_encoded_id(harness): } +@pytest.mark.asyncio +async def test_content__query_model_on_plain_id(harness): + harness.base_process.return_value = b"x" + + await call_content( + harness, + "9b444cea-aaaa-bbbb-cccc-dddddddddddd", + query={"model": "grok-imagine-video-1.5"}, + ) + + harness.resolve_model.assert_not_called() + assert harness.processor_data() == { + "video_id": "9b444cea-aaaa-bbbb-cccc-dddddddddddd", + "model": "grok-imagine-video-1.5", + } + + +@pytest.mark.asyncio +async def test_content__resolve_fail_keeps_decoded_model_id(harness): + harness.base_process.return_value = b"x" + encoded = encode_video_id_with_provider( + "9b444cea-aaaa-bbbb-cccc-dddddddddddd", + "xai", + "grok-imagine-video-1.5", + ) + + await call_content(harness, encoded) + + harness.resolve_model.assert_called_once_with("grok-imagine-video-1.5") + assert harness.processor_data() == { + "video_id": encoded, + "custom_llm_provider": "xai", + "model": "grok-imagine-video-1.5", + } + + # =========================================================================== # # POST /v1/videos/edits - video_edit # # =========================================================================== # diff --git a/tests/test_litellm/proxy/video_endpoints/test_utils.py b/tests/test_litellm/proxy/video_endpoints/test_utils.py index 9a2c208c075..5944fd7a394 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_utils.py +++ b/tests/test_litellm/proxy/video_endpoints/test_utils.py @@ -21,6 +21,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + resolve_video_request_model, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -28,6 +29,52 @@ from litellm.types.videos.utils import ( encode_character_id_with_provider, ) +# =========================================================================== # +# resolve_video_request_model +# =========================================================================== # + + +class _Resolver: + def __init__(self, mapping: dict[str, str | None]): + self.mapping = mapping + + def resolve_model_name_from_model_id(self, model_id: str | None) -> str | None: + return self.mapping.get(model_id) if model_id else None + + +def test_resolve_video_request_model__router_hit(): + assert ( + resolve_video_request_model( + model_id_from_decoded="deployment-123", + query_model="ignored", + llm_router=_Resolver({"deployment-123": "azure-sora"}), + ) + == "azure-sora" + ) + + +def test_resolve_video_request_model__keeps_decoded_id_when_router_misses(): + assert ( + resolve_video_request_model( + model_id_from_decoded="grok-imagine-video-1.5", + query_model=None, + llm_router=_Resolver({}), + ) + == "grok-imagine-video-1.5" + ) + + +def test_resolve_video_request_model__query_model_on_plain_id(): + assert ( + resolve_video_request_model( + model_id_from_decoded=None, + query_model="grok-imagine-video-1.5", + llm_router=None, + ) + == "grok-imagine-video-1.5" + ) + + # =========================================================================== # # extract_model_from_target_model_names # =========================================================================== # diff --git a/tests/test_litellm/videos/test_main.py b/tests/test_litellm/videos/test_main.py index 22e1e5c05eb..88e659490f6 100644 --- a/tests/test_litellm/videos/test_main.py +++ b/tests/test_litellm/videos/test_main.py @@ -178,6 +178,15 @@ def test_video_content__plain_id_defaults_to_openai(seams): assert seams.kwargs_of("video_content_handler")["custom_llm_provider"] == "openai" +def test_video_content__plain_id_with_grok_model_uses_xai(seams): + videos_main.video_content( + video_id="9b444cea-aaaa-bbbb-cccc-dddddddddddd", + model="grok-imagine-video-1.5", + ) + + assert seams.kwargs_of("video_content_handler")["custom_llm_provider"] == "xai" + + def test_video_remix__dispatch_and_provider_from_id(seams): result = videos_main.video_remix(video_id=AZURE_VIDEO_ID, prompt="new colors") @@ -377,6 +386,21 @@ async def test_avideo_content__pre_decodes_provider_before_delegating(): assert sync.call_args.kwargs["custom_llm_provider"] == "azure" +@pytest.mark.asyncio +async def test_avideo_content__plain_id_with_grok_model_uses_xai(): + sentinel = b"mp4-bytes" + with patch.object( + videos_main, "video_content", MagicMock(return_value=sentinel) + ) as sync: + result = await videos_main.avideo_content( + video_id="9b444cea-aaaa-bbbb-cccc-dddddddddddd", + model="grok-imagine-video-1.5", + ) + + assert result is sentinel + assert sync.call_args.kwargs["custom_llm_provider"] == "xai" + + # =========================================================================== # # Credential passthrough - DB/YAML model-config credentials the router injects # via kwargs must reach the provider call for EVERY video handler, carried in From c25883d55a0756f5cc32ace835fa5c9d784f114c Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Tue, 8 Sep 2026 11:09:29 +0000 Subject: [PATCH 2/9] fix(proxy): accept image-edit URL and data-URI form fields FastAPI File() rejected string image/mask values before the handler ran, so Grok Imagine edits that pass an http(s) URL or data:image URI never reached xAI. Parse those fields from the form instead and keep multipart uploads working Co-authored-by: HX --- litellm/llms/xai/videos/transformation.py | 18 +-- litellm/proxy/image_endpoints/endpoints.py | 161 +++++++++++++-------- 2 files changed, 110 insertions(+), 69 deletions(-) diff --git a/litellm/llms/xai/videos/transformation.py b/litellm/llms/xai/videos/transformation.py index 6d933a31357..bbb207fcc56 100644 --- a/litellm/llms/xai/videos/transformation.py +++ b/litellm/llms/xai/videos/transformation.py @@ -39,6 +39,8 @@ _SIZE_TO_ASPECT_RATIO: Final = { "1920x1080": "16:9", "1080x1920": "9:16", } + + def _duration_from_seconds(seconds: object) -> int: try: return int(seconds) if seconds is not None else 6 @@ -110,9 +112,7 @@ class XAIVideoConfig(BaseVideoConfig): from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth params: Final = ( - litellm_params.model_dump() - if isinstance(litellm_params, GenericLiteLLMParams) - else (litellm_params or {}) + litellm_params.model_dump() if isinstance(litellm_params, GenericLiteLLMParams) else (litellm_params or {}) ) if should_use_xai_oauth(params) and not XAIModelInfo.get_api_key(api_key): return XAIOAuthAuthenticator().get_api_base().rstrip("/") @@ -243,9 +243,7 @@ class XAIVideoConfig(BaseVideoConfig): progress=0, ) if custom_llm_provider: - video_obj.id = encode_video_id_with_provider( - video_obj.id, custom_llm_provider, model - ) + video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) video_obj.usage = usage if isinstance(usage, dict) else {} video_obj._hidden_params["video_url"] = None return video_obj @@ -294,9 +292,7 @@ class XAIVideoConfig(BaseVideoConfig): ) video_obj._hidden_params["video_url"] = video_url if custom_llm_provider and video_obj.id and video_obj.id != "unknown": - video_obj.id = encode_video_id_with_provider( - video_obj.id, custom_llm_provider, response_data.get("model") - ) + video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, response_data.get("model")) return video_obj def transform_video_content_request( @@ -321,9 +317,7 @@ class XAIVideoConfig(BaseVideoConfig): url: Final = video_meta.get("url") if isinstance(video_meta, dict) else None if isinstance(url, str) and url: return url - raise ValueError( - f"xAI video not ready for download (status={payload.get('status')}): {payload}" - ) + raise ValueError(f"xAI video not ready for download (status={payload.get('status')}): {payload}") def transform_video_content_response( self, diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index ca3f9333d83..35c6db4584e 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -5,8 +5,9 @@ from collections.abc import Sequence from typing import Final, get_type_hints import orjson -from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status from fastapi.responses import ORJSONResponse +from starlette.datastructures import UploadFile import litellm from litellm._logging import verbose_proxy_logger @@ -17,6 +18,7 @@ from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.http_parsing_utils import ( + _is_form_content_type, coerce_numeric_form_fields, numeric_form_fields, ) @@ -27,6 +29,11 @@ from litellm.types.llms.openai import ChatCompletionUserMessage router: Final = APIRouter() IMAGE_EDIT_NUMERIC_FORM_FIELDS: Final = numeric_form_fields(get_type_hints(ImageEditRequestParams)) +_IMAGE_REFERENCE_PREFIXES: Final = ("http://", "https://", "data:image/") +_IMAGE_EDIT_FILE_FIELDS: Final = ( + ("image", "image[]"), + ("mask", "mask[]"), +) async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO: @@ -51,6 +58,93 @@ async def batch_to_bytesio( return [await uploadfile_to_bytesio(u) for u in uploads] +def _is_image_reference_string(value: str) -> bool: + return value.startswith(_IMAGE_REFERENCE_PREFIXES) + + +def _invalid_image_field_error(field: str) -> HTTPException: + return HTTPException( + status_code=422, + detail=f"'{field}' must be a multipart file, http(s) URL, or data:image URI.", + ) + + +def _form_field_values(form: object, name: str) -> tuple[object, ...]: + getlist: Final = getattr(form, "getlist", None) + if not callable(getlist): + return () + return tuple(getlist(name)) + + +async def _coerce_image_part(value: object, field: str) -> io.BytesIO | str: + if isinstance(value, UploadFile): + return await uploadfile_to_bytesio(value) + if isinstance(value, str) and _is_image_reference_string(value): + return value + raise _invalid_image_field_error(field) + + +async def _normalize_image_values(values: tuple[object, ...], field: str) -> object | None: + if not values: + return None + coerced: Final = tuple([await _coerce_image_part(value, field) for value in values]) + if len(coerced) == 1 and isinstance(coerced[0], str): + return coerced[0] + return list(coerced) + + +def _json_image_values(data: dict[str, object], field: str) -> tuple[object, ...]: + if field not in data: + return () + raw: Final = data[field] + if isinstance(raw, list): + return tuple(raw) + return (raw,) + + +async def _normalized_image_edit_fields( + values_by_field: dict[str, tuple[object, ...]], +) -> dict[str, object]: + image: Final = await _normalize_image_values(values_by_field["image"], "image") + mask: Final = await _normalize_image_values(values_by_field["mask"], "mask") + return { + **({"image": image} if image is not None else {}), + **({"mask": mask} if mask is not None else {}), + } + + +async def _image_edit_assets_from_request( + request: Request, + data: dict[str, object], +) -> dict[str, object]: + form: Final = await request.form() if _is_form_content_type(request.headers.get("content-type", "")) else None + if form is None: + return { + **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, + **await _normalized_image_edit_fields( + {field: _json_image_values(data, field) for field, _alias in _IMAGE_EDIT_FILE_FIELDS} + ), + } + + form_values: Final = { + name: _form_field_values(form, name) for field, alias in _IMAGE_EDIT_FILE_FIELDS for name in (field, alias) + } + conflicts: Final = tuple( + field for field, alias in _IMAGE_EDIT_FILE_FIELDS if form_values[field] and form_values[alias] + ) + if conflicts: + raise HTTPException( + status_code=422, + detail=f"Cannot specify both '{conflicts[0]}' and '{conflicts[0]}[]'", + ) + return { + **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, + **await _normalized_image_edit_fields( + {field: form_values[field] or form_values[alias] for field, alias in _IMAGE_EDIT_FILE_FIELDS} + ), + } + + @router.post( "/v1/images/generations", dependencies=[Depends(user_api_key_auth)], @@ -235,10 +329,6 @@ async def image_edit_api( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), - image: list[UploadFile] | None = File(None), - image_array: list[UploadFile] | None = File(None, alias="image[]"), - mask: list[UploadFile] | None = File(None), - mask_array: list[UploadFile] | None = File(None, alias="mask[]"), model: str | None = None, ): """ @@ -254,20 +344,6 @@ async def image_edit_api( -F 'prompt=Create a studio ghibli image of this' ``` """ - if image is not None and image_array is not None: - raise HTTPException(status_code=422, detail="Cannot specify both 'image' and 'image[]'") - if mask is not None and mask_array is not None: - raise HTTPException(status_code=422, detail="Cannot specify both 'mask' and 'mask[]'") - if image is None and image_array is not None: - image = image_array - if mask is None and mask_array is not None: - mask = mask_array - - # if image is None: - # raise HTTPException(status_code=422, detail="Field required: image") - # Note: Image is optional for some models (e.g., Bedrock Stability style-transfer) - # The validation will be done at the model level if image is truly required - from litellm.proxy.proxy_server import ( _read_request_body, general_settings, @@ -283,49 +359,20 @@ async def image_edit_api( version, ) - ######################################################### - # Read request body and convert UploadFiles to BytesIO - ######################################################### - data: Final = dict( + parsed_body: Final = dict( coerce_numeric_form_fields( parsed_body=await _read_request_body(request=request), numeric_fields=IMAGE_EDIT_NUMERIC_FORM_FIELDS, ) ) - image_files: Final = await batch_to_bytesio(image) - mask_files: Final = await batch_to_bytesio(mask) - if image_files: - data["image"] = image_files - if mask_files: - data["mask"] = mask_files - - invalid_image_fields: Final = tuple( - field - for field in ("image", "mask") - if field in data - and isinstance(data[field], str) - and not ( - data[field].startswith("http://") - or data[field].startswith("https://") - or data[field].startswith("data:image/") - ) - ) - if invalid_image_fields: - raise HTTPException( - status_code=422, - detail=f"'{invalid_image_fields[0]}' must be a multipart file, http(s) URL, or data:image URI.", - ) - - # Ensure prompt exists in data (default to None for models that don't require it) - if "prompt" not in data: - data["prompt"] = None - - data["model"] = ( - model - or general_settings.get("image_generation_model", None) # server default - or user_model # model name passed via cli args - or data.get("model", None) # default passed in http request - ) + with_assets: Final = await _image_edit_assets_from_request(request, parsed_body) + data: Final = { + **with_assets, + **({} if "prompt" in with_assets else {"prompt": None}), + "model": ( + model or general_settings.get("image_generation_model", None) or user_model or with_assets.get("model") + ), + } ######################################################### # Process request ######################################################### From 1d8942f70b5a2bbe1a1bbb325ba8bb4ee448d563 Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Wed, 9 Sep 2026 14:59:29 +0800 Subject: [PATCH 3/9] fix(xai): encode video ids and validate CDN download URLs --- litellm/llms/xai/videos/transformation.py | 18 ++- litellm/proxy/video_endpoints/endpoints.py | 12 +- litellm/proxy/video_endpoints/utils.py | 9 ++ .../llms/xai/test_xai_video_generation.py | 143 +++++++++++++++++- .../proxy/video_endpoints/test_endpoints.py | 16 +- .../proxy/video_endpoints/test_utils.py | 16 ++ 6 files changed, 197 insertions(+), 17 deletions(-) diff --git a/litellm/llms/xai/videos/transformation.py b/litellm/llms/xai/videos/transformation.py index bbb207fcc56..b871f4a6011 100644 --- a/litellm/llms/xai/videos/transformation.py +++ b/litellm/llms/xai/videos/transformation.py @@ -7,6 +7,7 @@ from httpx._types import RequestFiles import litellm from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import async_safe_get, encode_url_path_segment, safe_get from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -248,6 +249,13 @@ class XAIVideoConfig(BaseVideoConfig): video_obj._hidden_params["video_url"] = None return video_obj + def _video_resource_url(self, api_base: str, video_id: str) -> str: + encoded_video_id: Final = encode_url_path_segment( + extract_original_video_id(video_id), + field_name="video_id", + ) + return f"{self._v1_root(api_base)}/videos/{encoded_video_id}" + def transform_video_status_retrieve_request( self, video_id: str, @@ -255,8 +263,7 @@ class XAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> tuple[str, dict]: - original_id: Final = extract_original_video_id(video_id) - return f"{self._v1_root(api_base)}/videos/{original_id}", {} + return self._video_resource_url(api_base, video_id), {} def transform_video_status_retrieve_response( self, @@ -303,8 +310,7 @@ class XAIVideoConfig(BaseVideoConfig): headers: dict, variant: str | None = None, ) -> tuple[str, dict]: - original_id: Final = extract_original_video_id(video_id) - return f"{self._v1_root(api_base)}/videos/{original_id}", {} + return self._video_resource_url(api_base, video_id), {} def _video_cdn_url(self, raw_response: httpx.Response) -> str | None: content_type: Final = (raw_response.headers.get("content-type") or "").lower() @@ -328,7 +334,7 @@ class XAIVideoConfig(BaseVideoConfig): if url is None: return raw_response.content httpx_client: Final[HTTPHandler] = _get_httpx_client() - video_response: Final = httpx_client.get(url) + video_response: Final = safe_get(httpx_client, url) video_response.raise_for_status() return video_response.content @@ -343,7 +349,7 @@ class XAIVideoConfig(BaseVideoConfig): async_httpx_client: Final[AsyncHTTPHandler] = get_async_httpx_client( llm_provider=litellm.LlmProviders.XAI, ) - video_response: Final = await async_httpx_client.get(url) + video_response: Final = await async_safe_get(async_httpx_client, url) video_response.raise_for_status() return video_response.content diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 9cc1f033326..6f13d88341f 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -20,6 +20,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + infer_video_provider_from_model, resolve_video_request_model, video_reference_to_id, ) @@ -253,15 +254,12 @@ async def video_status( provider_from_id: Final = decoded.get("custom_llm_provider") model_id_from_decoded: Final = decoded.get("model_id") - custom_llm_provider: Final = ( + explicit_provider: Final = ( get_custom_llm_provider_from_request_headers(request=request) or get_custom_llm_provider_from_request_query(request=request) or await get_custom_llm_provider_from_request_body(request=request) or provider_from_id - or "openai" ) - if custom_llm_provider: - data["custom_llm_provider"] = custom_llm_provider resolved_model: Final = resolve_video_request_model( model_id_from_decoded=model_id_from_decoded, @@ -271,6 +269,12 @@ async def video_status( if resolved_model: data["model"] = resolved_model + custom_llm_provider: Final = ( + explicit_provider or infer_video_provider_from_model(resolved_model) or "openai" + ) + if custom_llm_provider: + data["custom_llm_provider"] = custom_llm_provider + # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index af0e9548568..7d0357371d8 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -9,6 +9,15 @@ class VideoModelIdResolver(Protocol): def resolve_model_name_from_model_id(self, model_id: str | None) -> str | None: ... +def infer_video_provider_from_model(model: str | None) -> str | None: + if not isinstance(model, str) or not model: + return None + unprefixed: Final = model.split("/", 1)[-1] + if unprefixed.startswith("grok-imagine-video"): + return "xai" + return None + + def resolve_video_request_model( *, model_id_from_decoded: str | None, diff --git a/tests/test_litellm/llms/xai/test_xai_video_generation.py b/tests/test_litellm/llms/xai/test_xai_video_generation.py index fee57425264..261ce2e88c0 100644 --- a/tests/test_litellm/llms/xai/test_xai_video_generation.py +++ b/tests/test_litellm/llms/xai/test_xai_video_generation.py @@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +from litellm.litellm_core_utils.url_utils import SSRFError from litellm.llms.xai.videos.transformation import XAIVideoConfig from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -141,6 +142,50 @@ def test_content_request_is_get_status(): assert params == {} +@pytest.mark.parametrize("video_id", ["..", ""]) +@pytest.mark.parametrize( + "transform_name", + ["transform_video_status_retrieve_request", "transform_video_content_request"], +) +def test_video_id_rejects_empty_and_dot_path_segments(video_id, transform_name): + transform = getattr(XAIVideoConfig(), transform_name) + with pytest.raises(ValueError, match="video_id"): + transform( + video_id=video_id, + api_base="https://api.x.ai/v1", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + +@pytest.mark.parametrize( + "transform_name", + ["transform_video_status_retrieve_request", "transform_video_content_request"], +) +def test_video_id_parent_path_is_one_encoded_segment(transform_name): + transform = getattr(XAIVideoConfig(), transform_name) + url, params = transform( + video_id="../models", + api_base="https://api.x.ai/v1", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert url == "https://api.x.ai/v1/videos/..%2Fmodels" + assert "/videos/../" not in url + assert params == {} + + +def test_video_id_is_percent_encoded_as_one_segment(): + url, params = XAIVideoConfig().transform_video_status_retrieve_request( + video_id="req-123?x=1#frag", + api_base="https://api.x.ai/v1", + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + assert url == "https://api.x.ai/v1/videos/req-123%3Fx%3D1%23frag" + assert params == {} + + def test_content_response_fetches_cdn_via_shared_client(): status = httpx.Response( 200, @@ -151,14 +196,16 @@ def test_content_response_fetches_cdn_via_shared_client(): video_resp = MagicMock() video_resp.content = b"mp4-bytes" video_resp.raise_for_status.return_value = None - cdn.get.return_value = video_resp with patch( "litellm.llms.xai.videos.transformation._get_httpx_client", return_value=cdn, - ): + ), patch( + "litellm.llms.xai.videos.transformation.safe_get", + return_value=video_resp, + ) as safe_get: body = XAIVideoConfig().transform_video_content_response(status, logging_obj=MagicMock()) assert body == b"mp4-bytes" - cdn.get.assert_called_once_with("https://vidgen.x.ai/x.mp4") + safe_get.assert_called_once_with(cdn, "https://vidgen.x.ai/x.mp4") @pytest.mark.asyncio @@ -172,19 +219,21 @@ async def test_async_content_response_does_not_use_sync_client(): video_resp = MagicMock() video_resp.content = b"async-mp4" video_resp.raise_for_status.return_value = None - async_client.get = AsyncMock(return_value=video_resp) with patch( "litellm.llms.xai.videos.transformation.get_async_httpx_client", return_value=async_client, ), patch( "litellm.llms.xai.videos.transformation._get_httpx_client", - ) as sync_client: + ) as sync_client, patch( + "litellm.llms.xai.videos.transformation.async_safe_get", + new=AsyncMock(return_value=video_resp), + ) as async_safe_get: body = await XAIVideoConfig().async_transform_video_content_response( status, logging_obj=MagicMock() ) assert body == b"async-mp4" sync_client.assert_not_called() - async_client.get.assert_awaited_once_with("https://vidgen.x.ai/x.mp4") + async_safe_get.assert_awaited_once_with(async_client, "https://vidgen.x.ai/x.mp4") def test_content_response_raises_when_status_has_no_url(): @@ -204,3 +253,85 @@ def test_content_response_returns_raw_bytes_when_not_json(): content=b"already-mp4", ) assert XAIVideoConfig().transform_video_content_response(raw, logging_obj=MagicMock()) == b"already-mp4" + + +def test_content_response_rejects_internal_cdn_host(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "done", "video": {"url": "http://127.0.0.1/secret.mp4"}}, + ) + + def boom(*args, **kwargs): + raise AssertionError("unsafe CDN fetch must not run") + + with patch( + "litellm.llms.xai.videos.transformation._get_httpx_client", + return_value=MagicMock(get=boom), + ): + with pytest.raises((SSRFError, ValueError)): + XAIVideoConfig().transform_video_content_response(status, logging_obj=MagicMock()) + + +def test_content_response_fetches_public_cdn_via_safe_get(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "done", "video": {"url": "https://vidgen.x.ai/x.mp4"}}, + ) + video_resp = MagicMock() + video_resp.content = b"safe-mp4" + video_resp.raise_for_status.return_value = None + with patch( + "litellm.llms.xai.videos.transformation.safe_get", + return_value=video_resp, + create=True, + ) as safe_get: + body = XAIVideoConfig().transform_video_content_response(status, logging_obj=MagicMock()) + assert body == b"safe-mp4" + assert safe_get.call_count == 1 + assert safe_get.call_args.args[1] == "https://vidgen.x.ai/x.mp4" + + +@pytest.mark.asyncio +async def test_async_content_response_rejects_internal_cdn_host(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "done", "video": {"url": "http://127.0.0.1/secret.mp4"}}, + ) + + async def boom(*args, **kwargs): + raise AssertionError("unsafe async CDN fetch must not run") + + with patch( + "litellm.llms.xai.videos.transformation.get_async_httpx_client", + return_value=MagicMock(get=boom), + ): + with pytest.raises((SSRFError, ValueError)): + await XAIVideoConfig().async_transform_video_content_response( + status, logging_obj=MagicMock() + ) + + +@pytest.mark.asyncio +async def test_async_content_response_fetches_public_cdn_via_async_safe_get(): + status = httpx.Response( + 200, + headers={"content-type": "application/json"}, + json={"status": "done", "video": {"url": "https://vidgen.x.ai/x.mp4"}}, + ) + video_resp = MagicMock() + video_resp.content = b"async-safe-mp4" + video_resp.raise_for_status.return_value = None + with patch( + "litellm.llms.xai.videos.transformation.async_safe_get", + new=AsyncMock(return_value=video_resp), + create=True, + ) as async_safe_get: + body = await XAIVideoConfig().async_transform_video_content_response( + status, logging_obj=MagicMock() + ) + assert body == b"async-safe-mp4" + async_safe_get.assert_awaited() + assert async_safe_get.call_args.args[1] == "https://vidgen.x.ai/x.mp4" diff --git a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py index eeb59e31988..78877b84f6c 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/video_endpoints/test_endpoints.py @@ -338,11 +338,25 @@ async def test_status__query_model_on_plain_id(harness): harness.resolve_model.assert_not_called() assert harness.processor_data() == { "video_id": "9b444cea-aaaa-bbbb-cccc-dddddddddddd", - "custom_llm_provider": "openai", + "custom_llm_provider": "xai", "model": "grok-imagine-video-1.5", } +@pytest.mark.asyncio +async def test_status__query_model_grok_imagine_does_not_default_openai_before_inference( + harness, +): + await call_status( + harness, + "video_plain_xai", + query={"model": "grok-imagine-video"}, + ) + + assert harness.processor_data()["custom_llm_provider"] == "xai" + assert harness.processor_data()["model"] == "grok-imagine-video" + + # =========================================================================== # # GET /v1/videos/{video_id}/content - video_content # # =========================================================================== # diff --git a/tests/test_litellm/proxy/video_endpoints/test_utils.py b/tests/test_litellm/proxy/video_endpoints/test_utils.py index 5944fd7a394..af8af6cdcf2 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_utils.py +++ b/tests/test_litellm/proxy/video_endpoints/test_utils.py @@ -21,6 +21,7 @@ from litellm.proxy.video_endpoints.utils import ( encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, + infer_video_provider_from_model, resolve_video_request_model, video_reference_to_id, ) @@ -75,6 +76,21 @@ def test_resolve_video_request_model__query_model_on_plain_id(): ) +@pytest.mark.parametrize( + "model,expected", + [ + ("grok-imagine-video", "xai"), + ("grok-imagine-video-1.5", "xai"), + ("xai/grok-imagine-video", "xai"), + ("sora-2", None), + (None, None), + ("", None), + ], +) +def test_infer_video_provider_from_model(model, expected): + assert infer_video_provider_from_model(model) == expected + + # =========================================================================== # # extract_model_from_target_model_names # =========================================================================== # From d1f7eee7921d3e346c82f736e3ab89312c346a2c Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Thu, 10 Sep 2026 10:42:21 +0800 Subject: [PATCH 4/9] fix(proxy): authorize video query model before status and content dispatch --- litellm/proxy/auth/auth_utils.py | 1 + .../proxy/auth/test_auth_utils.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index f78c4221f5a..791db5edca4 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1684,6 +1684,7 @@ _MODEL_ROUTING_HEADER_OR_QUERY_ROUTE_MARKERS: Final = ( "/batches", "/skills", "/evals", + "/videos", ) _MODEL_ROUTING_QUERY_TARGET_MODEL_ROUTE_MARKERS: Final = ( "/files", diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index aaf630ad29b..c7cd5cc6d73 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -658,6 +658,25 @@ def test_get_model_from_request_includes_fine_tuning_target_model_query(): ) +def test_get_model_from_request_includes_video_query_model_for_plain_id(): + result = get_model_from_request( + request_data={"video_id": "plain-xai-id"}, + route="/v1/videos/{video_id}", + request_query_params={"model": "grok-imagine-video-1.5"}, + ) + assert result == "grok-imagine-video-1.5" + + +def test_get_model_from_request_includes_video_query_model_on_content_route(): + result = get_model_from_request( + request_data={"video_id": "plain-xai-id"}, + route="/v1/videos/{video_id}/content", + request_query_params={"model": "restricted-xai-model"}, + request_headers={"x-litellm-model": "also-restricted"}, + ) + assert result == ["restricted-xai-model", "also-restricted"] + + def test_get_model_from_request_extracts_video_id_model(): from litellm.types.videos.utils import encode_video_id_with_provider From d6bb450932c1e3af24208d12b08e4418861ec908 Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Wed, 16 Sep 2026 10:00:57 +0800 Subject: [PATCH 5/9] style: apply ruff format to xai image and video files --- litellm/llms/xai/image_edit/transformation.py | 16 +++------------- .../llms/xai/image_generation/transformation.py | 13 ++----------- litellm/proxy/video_endpoints/endpoints.py | 4 +--- litellm/videos/main.py | 12 +++--------- 4 files changed, 9 insertions(+), 36 deletions(-) diff --git a/litellm/llms/xai/image_edit/transformation.py b/litellm/llms/xai/image_edit/transformation.py index 4ba55f6c339..9ef0b51dca7 100644 --- a/litellm/llms/xai/image_edit/transformation.py +++ b/litellm/llms/xai/image_edit/transformation.py @@ -86,12 +86,7 @@ class XAIImageEditConfig(BaseImageEditConfig): resolved_base: Final = ( XAIOAuthAuthenticator().get_api_base() if should_use_xai_oauth(litellm_params) and not XAIModelInfo.get_api_key(api_key) - else ( - api_base - or get_secret_str("XAI_API_BASE") - or get_secret_str("XAI_OAUTH_API_BASE") - or XAI_API_BASE - ) + else (api_base or get_secret_str("XAI_API_BASE") or get_secret_str("XAI_OAUTH_API_BASE") or XAI_API_BASE) ) base: Final = (resolved_base or XAI_API_BASE).rstrip("/") if base.endswith("/v1"): @@ -129,8 +124,7 @@ class XAIImageEditConfig(BaseImageEditConfig): model=model, llm_provider="xai", message=( - "Missing xAI credentials for image edit. " - "Pass api_key / XAI_API_KEY, or set use_xai_oauth=True." + "Missing xAI credentials for image edit. Pass api_key / XAI_API_KEY, or set use_xai_oauth=True." ), ) headers["Authorization"] = f"Bearer {dynamic_api_key}" @@ -159,11 +153,7 @@ class XAIImageEditConfig(BaseImageEditConfig): request: Final[dict[str, Any]] = { "model": XAIModelInfo.get_base_model(model) or model, **({"prompt": prompt} if prompt is not None else {}), - **( - {"image": image_payloads[0]} - if len(image_payloads) == 1 - else {"images": list(image_payloads)} - ), + **({"image": image_payloads[0]} if len(image_payloads) == 1 else {"images": list(image_payloads)}), **{ key: image_edit_optional_request_params[key] for key in ("aspect_ratio", "resolution") diff --git a/litellm/llms/xai/image_generation/transformation.py b/litellm/llms/xai/image_generation/transformation.py index 6dd8318a139..4e220e82386 100644 --- a/litellm/llms/xai/image_generation/transformation.py +++ b/litellm/llms/xai/image_generation/transformation.py @@ -49,11 +49,7 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): ) -> dict: supported_params: Final = frozenset(self.get_supported_openai_params(model)) allowed: Final = supported_params | _XAI_NATIVE_PARAMS - unknown: Final = tuple( - key - for key in non_default_params - if key not in optional_params and key not in allowed - ) + unknown: Final = tuple(key for key in non_default_params if key not in optional_params and key not in allowed) if unknown and not drop_params: raise ValueError( f"Parameter {unknown[0]} is not supported for model {model}. " @@ -86,12 +82,7 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): resolved_base: Final = ( XAIOAuthAuthenticator().get_api_base() if should_use_xai_oauth(litellm_params) and not XAIModelInfo.get_api_key(api_key) - else ( - api_base - or get_secret_str("XAI_API_BASE") - or get_secret_str("XAI_OAUTH_API_BASE") - or XAI_API_BASE - ) + else (api_base or get_secret_str("XAI_API_BASE") or get_secret_str("XAI_OAUTH_API_BASE") or XAI_API_BASE) ) base: Final = (resolved_base or XAI_API_BASE).rstrip("/") if base.endswith("/v1"): diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 6f13d88341f..6da6301aac5 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -269,9 +269,7 @@ async def video_status( if resolved_model: data["model"] = resolved_model - custom_llm_provider: Final = ( - explicit_provider or infer_video_provider_from_model(resolved_model) or "openai" - ) + custom_llm_provider: Final = explicit_provider or infer_video_provider_from_model(resolved_model) or "openai" if custom_llm_provider: data["custom_llm_provider"] = custom_llm_provider diff --git a/litellm/videos/main.py b/litellm/videos/main.py index b95225d04c5..c01d1343c91 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -362,9 +362,7 @@ def video_content( litellm_call_id: Final[str | None] = kwargs.get("litellm_call_id", None) _is_async: Final = kwargs.pop("async_call", False) is True - custom_llm_provider = _provider_for_video_id( - video_id, custom_llm_provider, kwargs.get("model") - ) + custom_llm_provider = _provider_for_video_id(video_id, custom_llm_provider, kwargs.get("model")) # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) @@ -456,9 +454,7 @@ async def avideo_content( loop: Final = asyncio.get_event_loop() kwargs["async_call"] = True - custom_llm_provider = _provider_for_video_id( - video_id, custom_llm_provider, kwargs.get("model") - ) + custom_llm_provider = _provider_for_video_id(video_id, custom_llm_provider, kwargs.get("model")) func: Final = partial( video_content, @@ -1062,9 +1058,7 @@ def video_status( response: Final = VideoObject(**mock_response) return response - custom_llm_provider = _provider_for_video_id( - video_id, custom_llm_provider, kwargs.get("model") - ) + custom_llm_provider = _provider_for_video_id(video_id, custom_llm_provider, kwargs.get("model")) # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) From 6ab547e35ab9e502666a546b5392716ff7cbf617 Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Fri, 18 Sep 2026 15:32:00 +0800 Subject: [PATCH 6/9] fix(xai): drop banned typing.Any from image and video transforms --- litellm/llms/xai/image_edit/transformation.py | 7 +++-- .../xai/image_generation/transformation.py | 13 ++++------ litellm/llms/xai/videos/transformation.py | 26 ++++++++----------- litellm/proxy/video_endpoints/utils.py | 21 ++++++++------- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/litellm/llms/xai/image_edit/transformation.py b/litellm/llms/xai/image_edit/transformation.py index 9ef0b51dca7..c82c810442e 100644 --- a/litellm/llms/xai/image_edit/transformation.py +++ b/litellm/llms/xai/image_edit/transformation.py @@ -1,6 +1,6 @@ import base64 from io import BufferedReader, BytesIO -from typing import Any, Final +from typing import TYPE_CHECKING, Final import httpx from httpx._types import RequestFiles @@ -15,6 +15,9 @@ from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import FileTypes, ImageObject, ImageResponse +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + _SIZE_TO_ASPECT_RATIO: Final = { "1024x1024": "1:1", "1792x1024": "16:9", @@ -167,7 +170,7 @@ class XAIImageEditConfig(BaseImageEditConfig): self, model: str, raw_response: httpx.Response, - logging_obj: Any, + logging_obj: "LiteLLMLoggingObj", ) -> ImageResponse: try: response_data: Final = raw_response.json() diff --git a/litellm/llms/xai/image_generation/transformation.py b/litellm/llms/xai/image_generation/transformation.py index 4e220e82386..62dd7a7f707 100644 --- a/litellm/llms/xai/image_generation/transformation.py +++ b/litellm/llms/xai/image_generation/transformation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import httpx @@ -16,11 +16,8 @@ from litellm.types.llms.openai import ( from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - - LiteLLMLoggingObj = _LiteLLMLoggingObj -else: - LiteLLMLoggingObj = Any + import tiktoken + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _SIZE_TO_ASPECT_RATIO: Final = { "1024x1024": "1:1", @@ -156,11 +153,11 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): model: str, raw_response: httpx.Response, model_response: ImageResponse, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", request_data: dict, optional_params: dict, litellm_params: dict, - encoding: Any, + encoding: "tiktoken.Encoding | None", api_key: str | None = None, json_mode: bool | None = None, ) -> ImageResponse: diff --git a/litellm/llms/xai/videos/transformation.py b/litellm/llms/xai/videos/transformation.py index b871f4a6011..2755fe6660d 100644 --- a/litellm/llms/xai/videos/transformation.py +++ b/litellm/llms/xai/videos/transformation.py @@ -1,5 +1,5 @@ import time -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Final import httpx from httpx._types import RequestFiles @@ -25,11 +25,7 @@ from litellm.types.videos.utils import ( ) if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj - - LiteLLMLoggingObj = _LiteLLMLoggingObj -else: - LiteLLMLoggingObj = Any + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _SIZE_TO_ASPECT_RATIO: Final = { "1024x1024": "1:1", @@ -225,7 +221,7 @@ class XAIVideoConfig(BaseVideoConfig): self, model: str, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, request_data: dict | None = None, ) -> VideoObject: @@ -268,7 +264,7 @@ class XAIVideoConfig(BaseVideoConfig): def transform_video_status_retrieve_response( self, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, ) -> VideoObject: response_data: Final = raw_response.json() @@ -328,7 +324,7 @@ class XAIVideoConfig(BaseVideoConfig): def transform_video_content_response( self, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", ) -> bytes: url: Final = self._video_cdn_url(raw_response) if url is None: @@ -341,7 +337,7 @@ class XAIVideoConfig(BaseVideoConfig): async def async_transform_video_content_response( self, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", ) -> bytes: url: Final = self._video_cdn_url(raw_response) if url is None: @@ -360,14 +356,14 @@ class XAIVideoConfig(BaseVideoConfig): api_base: str, litellm_params: GenericLiteLLMParams, headers: dict, - extra_body: dict[str, Any] | None = None, + extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: raise NotImplementedError("Video remix is not supported by xAI Imagine API") def transform_video_remix_response( self, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, ) -> VideoObject: raise NotImplementedError("Video remix is not supported by xAI Imagine API") @@ -380,14 +376,14 @@ class XAIVideoConfig(BaseVideoConfig): after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, Any] | None = None, + extra_query: dict[str, object] | None = None, ) -> tuple[str, dict]: raise NotImplementedError("Video listing is not supported by xAI Imagine API") def transform_video_list_response( self, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, ) -> dict[str, str]: raise NotImplementedError("Video listing is not supported by xAI Imagine API") @@ -404,6 +400,6 @@ class XAIVideoConfig(BaseVideoConfig): def transform_video_delete_response( self, raw_response: httpx.Response, - logging_obj: LiteLLMLoggingObj, + logging_obj: "LiteLLMLoggingObj", ) -> VideoObject: raise NotImplementedError("Video delete is not supported by xAI Imagine API") diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 7d0357371d8..9795e925863 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -1,4 +1,5 @@ -from typing import Any, Final, Protocol +from collections.abc import Mapping, Sequence +from typing import Final, Protocol import orjson @@ -35,12 +36,14 @@ def resolve_video_request_model( return None -def extract_model_from_target_model_names(target_model_names: Any) -> str | None: +def extract_model_from_target_model_names(target_model_names: object) -> str | None: if isinstance(target_model_names, str): - target_model_names = [m.strip() for m in target_model_names.split(",") if m.strip()] - elif not isinstance(target_model_names, list): - return None - return target_model_names[0] if target_model_names else None + names: Final = tuple(m.strip() for m in target_model_names.split(",") if m.strip()) + return names[0] if names else None + if isinstance(target_model_names, Sequence) and not isinstance(target_model_names, (str, bytes)): + first: Final = target_model_names[0] if target_model_names else None + return first if isinstance(first, str) else None + return None def video_reference_to_id(video_ref: object) -> str: @@ -55,9 +58,9 @@ def video_reference_to_id(video_ref: object) -> str: return parsed_ref.get("id", "") if isinstance(parsed_ref, dict) else video_ref -def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: +def get_custom_provider_from_data(data: Mapping[str, object]) -> str | None: custom_llm_provider: Final = data.get("custom_llm_provider") - if custom_llm_provider: + if isinstance(custom_llm_provider, str) and custom_llm_provider: return custom_llm_provider extra_body = data.get("extra_body") @@ -77,7 +80,7 @@ def get_custom_provider_from_data(data: dict[str, Any]) -> str | None: return None -def encode_character_id_in_response(response: Any, custom_llm_provider: str, model_id: str | None) -> Any: +def encode_character_id_in_response(response: object, custom_llm_provider: str, model_id: str | None) -> object: if isinstance(response, dict) and response.get("id"): response["id"] = encode_character_id_with_provider( character_id=response["id"], From b89657c029d9cdc76045558a755d87fba231df9e Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Mon, 21 Sep 2026 10:52:25 +0800 Subject: [PATCH 7/9] fix(xai): replace leftover Any and sort TYPE_CHECKING imports --- litellm/llms/xai/image_edit/transformation.py | 2 +- litellm/llms/xai/image_generation/transformation.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/llms/xai/image_edit/transformation.py b/litellm/llms/xai/image_edit/transformation.py index c82c810442e..15762d76391 100644 --- a/litellm/llms/xai/image_edit/transformation.py +++ b/litellm/llms/xai/image_edit/transformation.py @@ -153,7 +153,7 @@ class XAIImageEditConfig(BaseImageEditConfig): raise ValueError("xAI image edit requires at least one reference image.") n: Final = image_edit_optional_request_params.get("n") - request: Final[dict[str, Any]] = { + request: Final[dict[str, object]] = { "model": XAIModelInfo.get_base_model(model) or model, **({"prompt": prompt} if prompt is not None else {}), **({"image": image_payloads[0]} if len(image_payloads) == 1 else {"images": list(image_payloads)}), diff --git a/litellm/llms/xai/image_generation/transformation.py b/litellm/llms/xai/image_generation/transformation.py index 62dd7a7f707..327f7891ae2 100644 --- a/litellm/llms/xai/image_generation/transformation.py +++ b/litellm/llms/xai/image_generation/transformation.py @@ -17,6 +17,7 @@ from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: import tiktoken + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _SIZE_TO_ASPECT_RATIO: Final = { From 55a34c313fb911cb5d75c0767173222a3084f629 Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Tue, 22 Sep 2026 10:43:15 +0800 Subject: [PATCH 8/9] fix(xai): mark provider JSON dicts mutable-ok for type discipline --- litellm/llms/xai/image_edit/__init__.py | 2 +- litellm/llms/xai/image_edit/transformation.py | 62 +++++------ litellm/llms/xai/image_generation/__init__.py | 2 +- .../xai/image_generation/transformation.py | 60 +++++------ litellm/llms/xai/videos/__init__.py | 2 +- litellm/llms/xai/videos/transformation.py | 100 +++++++++--------- litellm/proxy/image_endpoints/endpoints.py | 74 ++++++------- 7 files changed, 151 insertions(+), 151 deletions(-) diff --git a/litellm/llms/xai/image_edit/__init__.py b/litellm/llms/xai/image_edit/__init__.py index 5cae7020a92..79664e14d6b 100644 --- a/litellm/llms/xai/image_edit/__init__.py +++ b/litellm/llms/xai/image_edit/__init__.py @@ -1,3 +1,3 @@ from .transformation import XAIImageEditConfig -__all__ = ["XAIImageEditConfig"] +__all__ = ["XAIImageEditConfig"] # mutable-ok: provider JSON body and base-class dict signature diff --git a/litellm/llms/xai/image_edit/transformation.py b/litellm/llms/xai/image_edit/transformation.py index 15762d76391..a109f1051b2 100644 --- a/litellm/llms/xai/image_edit/transformation.py +++ b/litellm/llms/xai/image_edit/transformation.py @@ -18,7 +18,7 @@ from litellm.types.utils import FileTypes, ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -_SIZE_TO_ASPECT_RATIO: Final = { +_SIZE_TO_ASPECT_RATIO: Final = { # mutable-ok: provider JSON body and base-class dict signature "1024x1024": "1:1", "1792x1024": "16:9", "1024x1792": "9:16", @@ -41,18 +41,18 @@ def _read_seekable(image: BytesIO | BufferedReader) -> bytes: class XAIImageEditConfig(BaseImageEditConfig): - def get_supported_openai_params(self, model: str) -> list: - return ["n", "response_format", "size", "user"] + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: provider JSON body and base-class dict signature + return ["n", "response_format", "size", "user"] # mutable-ok: provider JSON body and base-class dict signature def map_openai_params( self, image_edit_optional_params: ImageEditOptionalRequestParams, model: str, drop_params: bool, - ) -> dict: + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature supported: Final = frozenset(self.get_supported_openai_params(model)) allowed: Final = supported | _XAI_NATIVE_PARAMS - incoming: Final = dict(image_edit_optional_params) + incoming: Final = dict(image_edit_optional_params) # mutable-ok: provider JSON body and base-class dict signature unknown: Final = tuple(key for key in incoming if key not in allowed) if unknown and not drop_params: raise ValueError( @@ -61,17 +61,17 @@ class XAIImageEditConfig(BaseImageEditConfig): "Set drop_params=True to drop unsupported parameters." ) - mapped: Final = {key: value for key, value in incoming.items() if key in allowed} + mapped: Final = {key: value for key, value in incoming.items() if key in allowed} # mutable-ok: provider JSON body and base-class dict signature size: Final = mapped.get("size") aspect_ratio: Final = mapped.get("aspect_ratio") or ( _SIZE_TO_ASPECT_RATIO.get(str(size), "1:1") if size else None ) n: Final = mapped.get("n") resolution: Final = mapped.get("resolution") - return { - **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), - **({"n": int(n)} if n is not None else {}), - **({"resolution": resolution} if resolution is not None else {}), + return { # mutable-ok: provider JSON body and base-class dict signature + **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **({"n": int(n)} if n is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **({"resolution": resolution} if resolution is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } def use_multipart_form_data(self) -> bool: @@ -81,7 +81,7 @@ class XAIImageEditConfig(BaseImageEditConfig): self, model: str, api_base: str | None, - litellm_params: dict, + litellm_params: dict, # mutable-ok: provider JSON body and base-class dict signature ) -> str: from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth @@ -98,19 +98,19 @@ class XAIImageEditConfig(BaseImageEditConfig): def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: provider JSON body and base-class dict signature model: str, api_key: str | None = None, - litellm_params: dict | None = None, + litellm_params: dict | None = None, # mutable-ok: provider JSON body and base-class dict signature api_base: str | None = None, - ) -> dict: + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature from litellm.llms.xai.oauth import ( XAIOAuthAuthenticator, XAIOAuthError, should_use_xai_oauth, ) - params: Final = litellm_params or {} + params: Final = litellm_params or {} # mutable-ok: provider JSON body and base-class dict signature dynamic_api_key: Final = XAIModelInfo.get_api_key(api_key) if should_use_xai_oauth(params) and not dynamic_api_key: try: @@ -141,10 +141,10 @@ class XAIImageEditConfig(BaseImageEditConfig): model: str, prompt: str | None, image: FileTypes | None, - image_edit_optional_request_params: dict, + image_edit_optional_request_params: dict, # mutable-ok: provider JSON body and base-class dict signature litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> tuple[dict, RequestFiles]: + headers: dict, # mutable-ok: provider JSON body and base-class dict signature + ) -> tuple[dict, RequestFiles]: # mutable-ok: provider JSON body and base-class dict signature if image is None: raise ValueError("xAI image edit requires at least one reference image.") @@ -153,18 +153,18 @@ class XAIImageEditConfig(BaseImageEditConfig): raise ValueError("xAI image edit requires at least one reference image.") n: Final = image_edit_optional_request_params.get("n") - request: Final[dict[str, object]] = { + request: Final[dict[str, object]] = { # mutable-ok: provider JSON body and base-class dict signature "model": XAIModelInfo.get_base_model(model) or model, - **({"prompt": prompt} if prompt is not None else {}), - **({"image": image_payloads[0]} if len(image_payloads) == 1 else {"images": list(image_payloads)}), - **{ + **({"prompt": prompt} if prompt is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **({"image": image_payloads[0]} if len(image_payloads) == 1 else {"images": list(image_payloads)}), # mutable-ok: provider JSON body and base-class dict signature + **{ # mutable-ok: provider JSON body and base-class dict signature key: image_edit_optional_request_params[key] for key in ("aspect_ratio", "resolution") if image_edit_optional_request_params.get(key) is not None }, - **({"n": int(n)} if n is not None else {}), + **({"n": int(n)} if n is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } - return request, [] + return request, [] # mutable-ok: provider JSON body and base-class dict signature def transform_image_edit_response( self, @@ -195,25 +195,25 @@ class XAIImageEditConfig(BaseImageEditConfig): status_code=raw_response.status_code, headers=raw_response.headers, ) - return ImageResponse(data=list(images)) + return ImageResponse(data=list(images)) # mutable-ok: provider JSON body and base-class dict signature - def _as_image_list(self, image: FileTypes | list[FileTypes]) -> tuple[FileTypes, ...]: + def _as_image_list(self, image: FileTypes | list[FileTypes]) -> tuple[FileTypes, ...]: # mutable-ok: provider JSON body and base-class dict signature if isinstance(image, list): return tuple(item for item in image if item is not None) return (image,) - def _to_image_url(self, image: FileTypes) -> dict[str, str]: + def _to_image_url(self, image: FileTypes) -> dict[str, str]: # mutable-ok: provider JSON body and base-class dict signature if isinstance(image, str): - return {"url": image} + return {"url": image} # mutable-ok: provider JSON body and base-class dict signature if isinstance(image, dict): if image.get("url"): - return {"url": str(image["url"])} + return {"url": str(image["url"])} # mutable-ok: provider JSON body and base-class dict signature if image.get("file_id"): - return {"file_id": str(image["file_id"])} + return {"file_id": str(image["file_id"])} # mutable-ok: provider JSON body and base-class dict signature mime: Final = ImageEditRequestUtils.get_image_content_type(image) encoded: Final = base64.b64encode(self._read_all_bytes(image)).decode("utf-8") - return {"url": f"data:{mime};base64,{encoded}"} + return {"url": f"data:{mime};base64,{encoded}"} # mutable-ok: provider JSON body and base-class dict signature def _read_all_bytes(self, image: FileTypes) -> bytes: if isinstance(image, bytes): diff --git a/litellm/llms/xai/image_generation/__init__.py b/litellm/llms/xai/image_generation/__init__.py index cc249fdf9ff..c1b24b609c3 100644 --- a/litellm/llms/xai/image_generation/__init__.py +++ b/litellm/llms/xai/image_generation/__init__.py @@ -4,7 +4,7 @@ from litellm.llms.base_llm.image_generation.transformation import ( from .transformation import XAIImageGenerationConfig -__all__ = ["XAIImageGenerationConfig", "get_xai_image_generation_config"] +__all__ = ["XAIImageGenerationConfig", "get_xai_image_generation_config"] # mutable-ok: provider JSON body and base-class dict signature def get_xai_image_generation_config(model: str) -> BaseImageGenerationConfig: diff --git a/litellm/llms/xai/image_generation/transformation.py b/litellm/llms/xai/image_generation/transformation.py index 327f7891ae2..2db407eb450 100644 --- a/litellm/llms/xai/image_generation/transformation.py +++ b/litellm/llms/xai/image_generation/transformation.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -_SIZE_TO_ASPECT_RATIO: Final = { +_SIZE_TO_ASPECT_RATIO: Final = { # mutable-ok: provider JSON body and base-class dict signature "1024x1024": "1:1", "1792x1024": "16:9", "1024x1792": "9:16", @@ -35,16 +35,16 @@ _XAI_NATIVE_PARAMS: Final = frozenset({"aspect_ratio", "n"}) class XAIImageGenerationConfig(BaseImageGenerationConfig): - def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: - return ["n", "response_format", "size", "user"] + def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: provider JSON body and base-class dict signature + return ["n", "response_format", "size", "user"] # mutable-ok: provider JSON body and base-class dict signature def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict, # mutable-ok: provider JSON body and base-class dict signature + optional_params: dict, # mutable-ok: provider JSON body and base-class dict signature model: str, drop_params: bool, - ) -> dict: + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature supported_params: Final = frozenset(self.get_supported_openai_params(model)) allowed: Final = supported_params | _XAI_NATIVE_PARAMS unknown: Final = tuple(key for key in non_default_params if key not in optional_params and key not in allowed) @@ -55,15 +55,15 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): "Set drop_params=True to drop unsupported parameters." ) - merged: Final = {**optional_params, **{k: v for k, v in non_default_params.items() if k in allowed}} + merged: Final = {**optional_params, **{k: v for k, v in non_default_params.items() if k in allowed}} # mutable-ok: provider JSON body and base-class dict signature size: Final = merged.get("size") aspect_ratio: Final = merged.get("aspect_ratio") or ( _SIZE_TO_ASPECT_RATIO.get(str(size), "1:1") if size else None ) n: Final = merged.get("n") - return { - **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), - **({"n": int(n)} if n is not None else {}), + return { # mutable-ok: provider JSON body and base-class dict signature + **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **({"n": int(n)} if n is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } def get_complete_url( @@ -71,8 +71,8 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict, # mutable-ok: provider JSON body and base-class dict signature + litellm_params: dict, # mutable-ok: provider JSON body and base-class dict signature stream: bool | None = None, ) -> str: from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth @@ -89,14 +89,14 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: provider JSON body and base-class dict signature model: str, - messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + messages: list[AllMessageValues], # mutable-ok: provider JSON body and base-class dict signature + optional_params: dict, # mutable-ok: provider JSON body and base-class dict signature + litellm_params: dict, # mutable-ok: provider JSON body and base-class dict signature api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature from litellm.llms.xai.oauth import ( XAIOAuthAuthenticator, XAIOAuthError, @@ -133,20 +133,20 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): self, model: str, prompt: str, - optional_params: dict, - litellm_params: dict, - headers: dict, - ) -> dict: + optional_params: dict, # mutable-ok: provider JSON body and base-class dict signature + litellm_params: dict, # mutable-ok: provider JSON body and base-class dict signature + headers: dict, # mutable-ok: provider JSON body and base-class dict signature + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature n: Final = optional_params.get("n") - return { + return { # mutable-ok: provider JSON body and base-class dict signature "model": XAIModelInfo.get_base_model(model) or model, "prompt": prompt, **( - {"aspect_ratio": optional_params["aspect_ratio"]} + {"aspect_ratio": optional_params["aspect_ratio"]} # mutable-ok: provider JSON body and base-class dict signature if optional_params.get("aspect_ratio") is not None - else {} + else {} # mutable-ok: provider JSON body and base-class dict signature ), - **({"n": int(n)} if n is not None else {}), + **({"n": int(n)} if n is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } def transform_image_generation_response( @@ -155,9 +155,9 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): raw_response: httpx.Response, model_response: ImageResponse, logging_obj: "LiteLLMLoggingObj", - request_data: dict, - optional_params: dict, - litellm_params: dict, + request_data: dict, # mutable-ok: provider JSON body and base-class dict signature + optional_params: dict, # mutable-ok: provider JSON body and base-class dict signature + litellm_params: dict, # mutable-ok: provider JSON body and base-class dict signature encoding: "tiktoken.Encoding | None", api_key: str | None = None, json_mode: bool | None = None, @@ -174,7 +174,7 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): logging_obj.post_call( input=request_data.get("prompt", ""), api_key=api_key, - additional_args={"complete_input_dict": request_data}, + additional_args={"complete_input_dict": request_data}, # mutable-ok: provider JSON body and base-class dict signature original_response=response_data, ) @@ -192,5 +192,5 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): status_code=raw_response.status_code, headers=raw_response.headers, ) - model_response.data = list(images) + model_response.data = list(images) # mutable-ok: provider JSON body and base-class dict signature return model_response diff --git a/litellm/llms/xai/videos/__init__.py b/litellm/llms/xai/videos/__init__.py index 1e9330ffba1..45650657059 100644 --- a/litellm/llms/xai/videos/__init__.py +++ b/litellm/llms/xai/videos/__init__.py @@ -1,3 +1,3 @@ from .transformation import XAIVideoConfig -__all__ = ["XAIVideoConfig"] +__all__ = ["XAIVideoConfig"] # mutable-ok: provider JSON body and base-class dict signature diff --git a/litellm/llms/xai/videos/transformation.py b/litellm/llms/xai/videos/transformation.py index 2755fe6660d..039dce3a9fa 100644 --- a/litellm/llms/xai/videos/transformation.py +++ b/litellm/llms/xai/videos/transformation.py @@ -27,7 +27,7 @@ from litellm.types.videos.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -_SIZE_TO_ASPECT_RATIO: Final = { +_SIZE_TO_ASPECT_RATIO: Final = { # mutable-ok: provider JSON body and base-class dict signature "1024x1024": "1:1", "1792x1024": "16:9", "1024x1792": "9:16", @@ -45,7 +45,7 @@ def _duration_from_seconds(seconds: object) -> int: return 6 -_STATUS_MAP: Final = { +_STATUS_MAP: Final = { # mutable-ok: provider JSON body and base-class dict signature "done": "completed", "completed": "completed", "succeeded": "completed", @@ -58,8 +58,8 @@ _STATUS_MAP: Final = { class XAIVideoConfig(BaseVideoConfig): - def get_supported_openai_params(self, model: str) -> list: - return [ + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: provider JSON body and base-class dict signature + return [ # mutable-ok: provider JSON body and base-class dict signature "model", "prompt", "input_reference", @@ -74,29 +74,29 @@ class XAIVideoConfig(BaseVideoConfig): video_create_optional_params: VideoCreateOptionalRequestParams, model: str, drop_params: bool, - ) -> dict: - incoming: Final = dict(video_create_optional_params) + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature + incoming: Final = dict(video_create_optional_params) # mutable-ok: provider JSON body and base-class dict signature size: Final = incoming.get("size") - return { - **{ + return { # mutable-ok: provider JSON body and base-class dict signature + **{ # mutable-ok: provider JSON body and base-class dict signature key: value for key, value in incoming.items() - if key not in {"seconds", "size", "input_reference", "user", "extra_headers", "model"} + if key not in {"seconds", "size", "input_reference", "user", "extra_headers", "model"} # mutable-ok: provider JSON body and base-class dict signature }, **( - {"duration": _duration_from_seconds(incoming.get("seconds"))} + {"duration": _duration_from_seconds(incoming.get("seconds"))} # mutable-ok: provider JSON body and base-class dict signature if "seconds" in incoming and "duration" not in incoming - else {} + else {} # mutable-ok: provider JSON body and base-class dict signature ), **( - {"aspect_ratio": incoming.get("aspect_ratio") or _SIZE_TO_ASPECT_RATIO.get(str(size), "16:9")} + {"aspect_ratio": incoming.get("aspect_ratio") or _SIZE_TO_ASPECT_RATIO.get(str(size), "16:9")} # mutable-ok: provider JSON body and base-class dict signature if size and "aspect_ratio" not in incoming - else {} + else {} # mutable-ok: provider JSON body and base-class dict signature ), **( - {"image": incoming.get("image") or incoming.get("input_reference")} + {"image": incoming.get("image") or incoming.get("input_reference")} # mutable-ok: provider JSON body and base-class dict signature if incoming.get("input_reference") and "image" not in incoming - else {} + else {} # mutable-ok: provider JSON body and base-class dict signature ), } @@ -104,12 +104,12 @@ class XAIVideoConfig(BaseVideoConfig): self, api_base: str | None, api_key: str | None, - litellm_params: GenericLiteLLMParams | dict | None, + litellm_params: GenericLiteLLMParams | dict | None, # mutable-ok: provider JSON body and base-class dict signature ) -> str: from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth params: Final = ( - litellm_params.model_dump() if isinstance(litellm_params, GenericLiteLLMParams) else (litellm_params or {}) + litellm_params.model_dump() if isinstance(litellm_params, GenericLiteLLMParams) else (litellm_params or {}) # mutable-ok: provider JSON body and base-class dict signature ) if should_use_xai_oauth(params) and not XAIModelInfo.get_api_key(api_key): return XAIOAuthAuthenticator().get_api_base().rstrip("/") @@ -131,18 +131,18 @@ class XAIVideoConfig(BaseVideoConfig): def validate_environment( self, - headers: dict, + headers: dict, # mutable-ok: provider JSON body and base-class dict signature model: str, api_key: str | None = None, litellm_params: GenericLiteLLMParams | None = None, - ) -> dict: + ) -> dict: # mutable-ok: provider JSON body and base-class dict signature from litellm.llms.xai.oauth import ( XAIOAuthAuthenticator, XAIOAuthError, should_use_xai_oauth, ) - params: Final = litellm_params.model_dump() if litellm_params is not None else {} + params: Final = litellm_params.model_dump() if litellm_params is not None else {} # mutable-ok: provider JSON body and base-class dict signature resolved_api_key: Final = api_key or (litellm_params.api_key if litellm_params else None) dynamic_api_key: Final = XAIModelInfo.get_api_key(resolved_api_key) if should_use_xai_oauth(params) and not dynamic_api_key: @@ -174,7 +174,7 @@ class XAIVideoConfig(BaseVideoConfig): self, model: str, api_base: str | None, - litellm_params: dict, + litellm_params: dict, # mutable-ok: provider JSON body and base-class dict signature ) -> str: resolved: Final = self._resolve_api_base( api_base=api_base, @@ -190,11 +190,11 @@ class XAIVideoConfig(BaseVideoConfig): model: str, prompt: str, api_base: str, - video_create_optional_request_params: dict, + video_create_optional_request_params: dict, # mutable-ok: provider JSON body and base-class dict signature litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> tuple[dict, RequestFiles, str]: - copied: Final = { + headers: dict, # mutable-ok: provider JSON body and base-class dict signature + ) -> tuple[dict, RequestFiles, str]: # mutable-ok: provider JSON body and base-class dict signature + copied: Final = { # mutable-ok: provider JSON body and base-class dict signature key: video_create_optional_request_params[key] for key in ( "image", @@ -207,13 +207,13 @@ class XAIVideoConfig(BaseVideoConfig): if video_create_optional_request_params.get(key) is not None } return ( - { + { # mutable-ok: provider JSON body and base-class dict signature "model": XAIModelInfo.get_base_model(model) or model, - **({"prompt": prompt} if prompt else {}), + **({"prompt": prompt} if prompt else {}), # mutable-ok: provider JSON body and base-class dict signature **copied, - **({"duration": 6} if "duration" not in copied else {}), + **({"duration": 6} if "duration" not in copied else {}), # mutable-ok: provider JSON body and base-class dict signature }, - [], + [], # mutable-ok: provider JSON body and base-class dict signature api_base, ) @@ -223,14 +223,14 @@ class XAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, - request_data: dict | None = None, + request_data: dict | None = None, # mutable-ok: provider JSON body and base-class dict signature ) -> VideoObject: response_data: Final = raw_response.json() request_id: Final = response_data.get("request_id") or response_data.get("id") if not request_id: raise ValueError(f"xAI video generation response missing request_id: {response_data}") - usage: Final = response_data.get("usage") or {} + usage: Final = response_data.get("usage") or {} # mutable-ok: provider JSON body and base-class dict signature video_obj: Final = VideoObject( id=str(request_id), object="video", @@ -241,7 +241,7 @@ class XAIVideoConfig(BaseVideoConfig): ) if custom_llm_provider: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) - video_obj.usage = usage if isinstance(usage, dict) else {} + video_obj.usage = usage if isinstance(usage, dict) else {} # mutable-ok: provider JSON body and base-class dict signature video_obj._hidden_params["video_url"] = None return video_obj @@ -257,9 +257,9 @@ class XAIVideoConfig(BaseVideoConfig): video_id: str, api_base: str, litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> tuple[str, dict]: - return self._video_resource_url(api_base, video_id), {} + headers: dict, # mutable-ok: provider JSON body and base-class dict signature + ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature + return self._video_resource_url(api_base, video_id), {} # mutable-ok: provider JSON body and base-class dict signature def transform_video_status_retrieve_response( self, @@ -270,7 +270,7 @@ class XAIVideoConfig(BaseVideoConfig): response_data: Final = raw_response.json() status_raw: Final = str(response_data.get("status") or "processing").lower() status: Final = _STATUS_MAP.get(status_raw, status_raw) - video_meta: Final = response_data.get("video") or {} + video_meta: Final = response_data.get("video") or {} # mutable-ok: provider JSON body and base-class dict signature video_url: Final = video_meta.get("url") if isinstance(video_meta, dict) else None seconds: Final = ( str(video_meta.get("duration")) @@ -291,7 +291,7 @@ class XAIVideoConfig(BaseVideoConfig): model=response_data.get("model"), progress=response_data.get("progress"), seconds=seconds, - usage=response_data.get("usage") if isinstance(response_data.get("usage"), dict) else {}, + usage=response_data.get("usage") if isinstance(response_data.get("usage"), dict) else {}, # mutable-ok: provider JSON body and base-class dict signature ) video_obj._hidden_params["video_url"] = video_url if custom_llm_provider and video_obj.id and video_obj.id != "unknown": @@ -303,10 +303,10 @@ class XAIVideoConfig(BaseVideoConfig): video_id: str, api_base: str, litellm_params: GenericLiteLLMParams, - headers: dict, + headers: dict, # mutable-ok: provider JSON body and base-class dict signature variant: str | None = None, - ) -> tuple[str, dict]: - return self._video_resource_url(api_base, video_id), {} + ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature + return self._video_resource_url(api_base, video_id), {} # mutable-ok: provider JSON body and base-class dict signature def _video_cdn_url(self, raw_response: httpx.Response) -> str | None: content_type: Final = (raw_response.headers.get("content-type") or "").lower() @@ -315,7 +315,7 @@ class XAIVideoConfig(BaseVideoConfig): payload: Final = raw_response.json() if not isinstance(payload, dict): return None - video_meta: Final = payload.get("video") or {} + video_meta: Final = payload.get("video") or {} # mutable-ok: provider JSON body and base-class dict signature url: Final = video_meta.get("url") if isinstance(video_meta, dict) else None if isinstance(url, str) and url: return url @@ -355,9 +355,9 @@ class XAIVideoConfig(BaseVideoConfig): prompt: str, api_base: str, litellm_params: GenericLiteLLMParams, - headers: dict, - extra_body: dict[str, object] | None = None, - ) -> tuple[str, dict]: + headers: dict, # mutable-ok: provider JSON body and base-class dict signature + extra_body: dict[str, object] | None = None, # mutable-ok: provider JSON body and base-class dict signature + ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature raise NotImplementedError("Video remix is not supported by xAI Imagine API") def transform_video_remix_response( @@ -372,12 +372,12 @@ class XAIVideoConfig(BaseVideoConfig): self, api_base: str, litellm_params: GenericLiteLLMParams, - headers: dict, + headers: dict, # mutable-ok: provider JSON body and base-class dict signature after: str | None = None, limit: int | None = None, order: str | None = None, - extra_query: dict[str, object] | None = None, - ) -> tuple[str, dict]: + extra_query: dict[str, object] | None = None, # mutable-ok: provider JSON body and base-class dict signature + ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature raise NotImplementedError("Video listing is not supported by xAI Imagine API") def transform_video_list_response( @@ -385,7 +385,7 @@ class XAIVideoConfig(BaseVideoConfig): raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str | None = None, - ) -> dict[str, str]: + ) -> dict[str, str]: # mutable-ok: provider JSON body and base-class dict signature raise NotImplementedError("Video listing is not supported by xAI Imagine API") def transform_video_delete_request( @@ -393,8 +393,8 @@ class XAIVideoConfig(BaseVideoConfig): video_id: str, api_base: str, litellm_params: GenericLiteLLMParams, - headers: dict, - ) -> tuple[str, dict]: + headers: dict, # mutable-ok: provider JSON body and base-class dict signature + ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature raise NotImplementedError("Video delete is not supported by xAI Imagine API") def transform_video_delete_response( diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index a892cc1ab46..615e821270d 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -54,13 +54,13 @@ async def uploadfile_to_bytesio(upload: UploadFile) -> io.BytesIO: async def batch_to_bytesio( uploads: Sequence[UploadFile] | None, -) -> list[io.BytesIO] | None: +) -> list[io.BytesIO] | None: # mutable-ok: provider JSON body and base-class dict signature """ Convert a sequence of UploadFiles to a list of BytesIO buffers, or None. """ if not uploads: return None - return [await uploadfile_to_bytesio(u) for u in uploads] + return [await uploadfile_to_bytesio(u) for u in uploads] # mutable-ok: provider JSON body and base-class dict signature def _is_image_reference_string(value: str) -> bool: @@ -95,10 +95,10 @@ async def _normalize_image_values(values: tuple[object, ...], field: str) -> obj coerced: Final = tuple([await _coerce_image_part(value, field) for value in values]) if len(coerced) == 1 and isinstance(coerced[0], str): return coerced[0] - return list(coerced) + return list(coerced) # mutable-ok: provider JSON body and base-class dict signature -def _json_image_values(data: dict[str, object], field: str) -> tuple[object, ...]: +def _json_image_values(data: dict[str, object], field: str) -> tuple[object, ...]: # mutable-ok: provider JSON body and base-class dict signature if field not in data: return () raw: Final = data[field] @@ -108,30 +108,30 @@ def _json_image_values(data: dict[str, object], field: str) -> tuple[object, ... async def _normalized_image_edit_fields( - values_by_field: dict[str, tuple[object, ...]], -) -> dict[str, object]: + values_by_field: dict[str, tuple[object, ...]], # mutable-ok: provider JSON body and base-class dict signature +) -> dict[str, object]: # mutable-ok: provider JSON body and base-class dict signature image: Final = await _normalize_image_values(values_by_field["image"], "image") mask: Final = await _normalize_image_values(values_by_field["mask"], "mask") - return { - **({"image": image} if image is not None else {}), - **({"mask": mask} if mask is not None else {}), + return { # mutable-ok: provider JSON body and base-class dict signature + **({"image": image} if image is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **({"mask": mask} if mask is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } async def _image_edit_assets_from_request( request: Request, - data: dict[str, object], -) -> dict[str, object]: + data: dict[str, object], # mutable-ok: provider JSON body and base-class dict signature +) -> dict[str, object]: # mutable-ok: provider JSON body and base-class dict signature form: Final = await request.form() if _is_form_content_type(request.headers.get("content-type", "")) else None if form is None: - return { - **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, + return { # mutable-ok: provider JSON body and base-class dict signature + **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, # mutable-ok: provider JSON body and base-class dict signature **await _normalized_image_edit_fields( - {field: _json_image_values(data, field) for field, _alias in _IMAGE_EDIT_FILE_FIELDS} + {field: _json_image_values(data, field) for field, _alias in _IMAGE_EDIT_FILE_FIELDS} # mutable-ok: provider JSON body and base-class dict signature ), } - form_values: Final = { + form_values: Final = { # mutable-ok: provider JSON body and base-class dict signature name: _form_field_values(form, name) for field, alias in _IMAGE_EDIT_FILE_FIELDS for name in (field, alias) } conflicts: Final = tuple( @@ -142,31 +142,31 @@ async def _image_edit_assets_from_request( status_code=422, detail=f"Cannot specify both '{conflicts[0]}' and '{conflicts[0]}[]'", ) - return { - **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, + return { # mutable-ok: provider JSON body and base-class dict signature + **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, # mutable-ok: provider JSON body and base-class dict signature **await _normalized_image_edit_fields( - {field: form_values[field] or form_values[alias] for field, alias in _IMAGE_EDIT_FILE_FIELDS} + {field: form_values[field] or form_values[alias] for field, alias in _IMAGE_EDIT_FILE_FIELDS} # mutable-ok: provider JSON body and base-class dict signature ), } @router.post( "/v1/images/generations", - dependencies=[Depends(user_api_key_auth)], + dependencies=[Depends(user_api_key_auth)], # mutable-ok: provider JSON body and base-class dict signature response_class=ORJSONResponse, - tags=["images"], + tags=["images"], # mutable-ok: provider JSON body and base-class dict signature ) @router.post( "/images/generations", - dependencies=[Depends(user_api_key_auth)], + dependencies=[Depends(user_api_key_auth)], # mutable-ok: provider JSON body and base-class dict signature response_class=ORJSONResponse, - tags=["images"], + tags=["images"], # mutable-ok: provider JSON body and base-class dict signature ) @router.post( "/openai/deployments/{model:path}/images/generations", - dependencies=[Depends(user_api_key_auth)], + dependencies=[Depends(user_api_key_auth)], # mutable-ok: provider JSON body and base-class dict signature response_class=ORJSONResponse, - tags=["images"], + tags=["images"], # mutable-ok: provider JSON body and base-class dict signature ) # azure compatible endpoint async def image_generation( request: Request, @@ -185,7 +185,7 @@ async def image_generation( version, ) - data = {} + data = {} # mutable-ok: provider JSON body and base-class dict signature try: # Use orjson to parse JSON data, orjson speeds up requests significantly body: Final = await request.body() @@ -227,7 +227,7 @@ async def image_generation( "role": "user", "content": prompt_value, } - data["messages"] = [user_message] + data["messages"] = [user_message] # mutable-ok: provider JSON body and base-class dict signature data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=data, call_type="image_generation" ) @@ -257,7 +257,7 @@ async def image_generation( ) ### RESPONSE HEADERS ### - hidden_params: Final = getattr(response, "_hidden_params", {}) or {} + hidden_params: Final = getattr(response, "_hidden_params", {}) or {} # mutable-ok: provider JSON body and base-class dict signature model_id: Final = hidden_params.get("model_id", None) or "" cache_key: Final = hidden_params.get("cache_key", None) or "" api_base: Final = hidden_params.get("api_base", None) or "" @@ -284,7 +284,7 @@ async def image_generation( data=data, user_api_key_dict=user_api_key_dict, response=response, - request_headers=dict(request.headers), + request_headers=dict(request.headers), # mutable-ok: provider JSON body and base-class dict signature ) if callback_headers: fastapi_response.headers.update(callback_headers) @@ -316,19 +316,19 @@ async def image_generation( @router.post( "/v1/images/edits", - dependencies=[Depends(user_api_key_auth)], - tags=["images"], + dependencies=[Depends(user_api_key_auth)], # mutable-ok: provider JSON body and base-class dict signature + tags=["images"], # mutable-ok: provider JSON body and base-class dict signature ) @router.post( "/images/edits", - dependencies=[Depends(user_api_key_auth)], - tags=["images"], + dependencies=[Depends(user_api_key_auth)], # mutable-ok: provider JSON body and base-class dict signature + tags=["images"], # mutable-ok: provider JSON body and base-class dict signature ) @router.post( "/openai/deployments/{model:path}/images/edits", - dependencies=[Depends(user_api_key_auth)], + dependencies=[Depends(user_api_key_auth)], # mutable-ok: provider JSON body and base-class dict signature response_class=ORJSONResponse, - tags=["images"], + tags=["images"], # mutable-ok: provider JSON body and base-class dict signature ) # azure compatible endpoint async def image_edit_api( request: Request, @@ -364,16 +364,16 @@ async def image_edit_api( version, ) - parsed_body: Final = dict( + parsed_body: Final = dict( # mutable-ok: provider JSON body and base-class dict signature coerce_numeric_form_fields( parsed_body=await _read_request_body(request=request), numeric_fields=IMAGE_EDIT_NUMERIC_FORM_FIELDS, ) ) with_assets: Final = await _image_edit_assets_from_request(request, parsed_body) - data: Final = { + data: Final = { # mutable-ok: provider JSON body and base-class dict signature **with_assets, - **({} if "prompt" in with_assets else {"prompt": None}), + **({} if "prompt" in with_assets else {"prompt": None}), # mutable-ok: provider JSON body and base-class dict signature "model": ( model or general_settings.get("image_generation_model", None) or user_model or with_assets.get("model") ), From 0f8e5279a7b910bc1b8c25162363c0ffc077f553 Mon Sep 17 00:00:00 2001 From: hx <1367557521@qq.com> Date: Wed, 23 Sep 2026 09:44:16 +0800 Subject: [PATCH 9/9] fix(proxy): format xai transforms and bind video downloads to the creating key --- litellm/llms/xai/image_edit/transformation.py | 40 ++++++++--- litellm/llms/xai/image_generation/__init__.py | 5 +- .../xai/image_generation/transformation.py | 21 ++++-- litellm/llms/xai/videos/transformation.py | 70 ++++++++++++++----- litellm/proxy/image_endpoints/endpoints.py | 36 +++++++--- litellm/proxy/video_endpoints/endpoints.py | 22 +++++- litellm/proxy/video_endpoints/utils.py | 33 ++++++++- litellm/types/videos/main.py | 1 + litellm/types/videos/utils.py | 10 ++- .../proxy/video_endpoints/test_utils.py | 32 +++++---- 10 files changed, 211 insertions(+), 59 deletions(-) diff --git a/litellm/llms/xai/image_edit/transformation.py b/litellm/llms/xai/image_edit/transformation.py index a109f1051b2..79b8b953d94 100644 --- a/litellm/llms/xai/image_edit/transformation.py +++ b/litellm/llms/xai/image_edit/transformation.py @@ -41,7 +41,9 @@ def _read_seekable(image: BytesIO | BufferedReader) -> bytes: class XAIImageEditConfig(BaseImageEditConfig): - def get_supported_openai_params(self, model: str) -> list: # mutable-ok: provider JSON body and base-class dict signature + def get_supported_openai_params( + self, model: str + ) -> list: # mutable-ok: provider JSON body and base-class dict signature return ["n", "response_format", "size", "user"] # mutable-ok: provider JSON body and base-class dict signature def map_openai_params( @@ -52,7 +54,9 @@ class XAIImageEditConfig(BaseImageEditConfig): ) -> dict: # mutable-ok: provider JSON body and base-class dict signature supported: Final = frozenset(self.get_supported_openai_params(model)) allowed: Final = supported | _XAI_NATIVE_PARAMS - incoming: Final = dict(image_edit_optional_params) # mutable-ok: provider JSON body and base-class dict signature + incoming: Final = dict( + image_edit_optional_params + ) # mutable-ok: provider JSON body and base-class dict signature unknown: Final = tuple(key for key in incoming if key not in allowed) if unknown and not drop_params: raise ValueError( @@ -61,7 +65,9 @@ class XAIImageEditConfig(BaseImageEditConfig): "Set drop_params=True to drop unsupported parameters." ) - mapped: Final = {key: value for key, value in incoming.items() if key in allowed} # mutable-ok: provider JSON body and base-class dict signature + mapped: Final = { + key: value for key, value in incoming.items() if key in allowed + } # mutable-ok: provider JSON body and base-class dict signature size: Final = mapped.get("size") aspect_ratio: Final = mapped.get("aspect_ratio") or ( _SIZE_TO_ASPECT_RATIO.get(str(size), "1:1") if size else None @@ -69,9 +75,13 @@ class XAIImageEditConfig(BaseImageEditConfig): n: Final = mapped.get("n") resolution: Final = mapped.get("resolution") return { # mutable-ok: provider JSON body and base-class dict signature - **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {} + ), # mutable-ok: provider JSON body and base-class dict signature **({"n": int(n)} if n is not None else {}), # mutable-ok: provider JSON body and base-class dict signature - **({"resolution": resolution} if resolution is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"resolution": resolution} if resolution is not None else {} + ), # mutable-ok: provider JSON body and base-class dict signature } def use_multipart_form_data(self) -> bool: @@ -155,8 +165,12 @@ class XAIImageEditConfig(BaseImageEditConfig): n: Final = image_edit_optional_request_params.get("n") request: Final[dict[str, object]] = { # mutable-ok: provider JSON body and base-class dict signature "model": XAIModelInfo.get_base_model(model) or model, - **({"prompt": prompt} if prompt is not None else {}), # mutable-ok: provider JSON body and base-class dict signature - **({"image": image_payloads[0]} if len(image_payloads) == 1 else {"images": list(image_payloads)}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"prompt": prompt} if prompt is not None else {} + ), # mutable-ok: provider JSON body and base-class dict signature + **( + {"image": image_payloads[0]} if len(image_payloads) == 1 else {"images": list(image_payloads)} + ), # mutable-ok: provider JSON body and base-class dict signature **{ # mutable-ok: provider JSON body and base-class dict signature key: image_edit_optional_request_params[key] for key in ("aspect_ratio", "resolution") @@ -197,19 +211,25 @@ class XAIImageEditConfig(BaseImageEditConfig): ) return ImageResponse(data=list(images)) # mutable-ok: provider JSON body and base-class dict signature - def _as_image_list(self, image: FileTypes | list[FileTypes]) -> tuple[FileTypes, ...]: # mutable-ok: provider JSON body and base-class dict signature + def _as_image_list( + self, image: FileTypes | list[FileTypes] + ) -> tuple[FileTypes, ...]: # mutable-ok: provider JSON body and base-class dict signature if isinstance(image, list): return tuple(item for item in image if item is not None) return (image,) - def _to_image_url(self, image: FileTypes) -> dict[str, str]: # mutable-ok: provider JSON body and base-class dict signature + def _to_image_url( + self, image: FileTypes + ) -> dict[str, str]: # mutable-ok: provider JSON body and base-class dict signature if isinstance(image, str): return {"url": image} # mutable-ok: provider JSON body and base-class dict signature if isinstance(image, dict): if image.get("url"): return {"url": str(image["url"])} # mutable-ok: provider JSON body and base-class dict signature if image.get("file_id"): - return {"file_id": str(image["file_id"])} # mutable-ok: provider JSON body and base-class dict signature + return { + "file_id": str(image["file_id"]) + } # mutable-ok: provider JSON body and base-class dict signature mime: Final = ImageEditRequestUtils.get_image_content_type(image) encoded: Final = base64.b64encode(self._read_all_bytes(image)).decode("utf-8") diff --git a/litellm/llms/xai/image_generation/__init__.py b/litellm/llms/xai/image_generation/__init__.py index c1b24b609c3..a9207e9a404 100644 --- a/litellm/llms/xai/image_generation/__init__.py +++ b/litellm/llms/xai/image_generation/__init__.py @@ -4,7 +4,10 @@ from litellm.llms.base_llm.image_generation.transformation import ( from .transformation import XAIImageGenerationConfig -__all__ = ["XAIImageGenerationConfig", "get_xai_image_generation_config"] # mutable-ok: provider JSON body and base-class dict signature +__all__ = [ + "XAIImageGenerationConfig", + "get_xai_image_generation_config", +] # mutable-ok: provider JSON body and base-class dict signature def get_xai_image_generation_config(model: str) -> BaseImageGenerationConfig: diff --git a/litellm/llms/xai/image_generation/transformation.py b/litellm/llms/xai/image_generation/transformation.py index 2db407eb450..e5a35733dd2 100644 --- a/litellm/llms/xai/image_generation/transformation.py +++ b/litellm/llms/xai/image_generation/transformation.py @@ -35,7 +35,9 @@ _XAI_NATIVE_PARAMS: Final = frozenset({"aspect_ratio", "n"}) class XAIImageGenerationConfig(BaseImageGenerationConfig): - def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: provider JSON body and base-class dict signature + def get_supported_openai_params( + self, model: str + ) -> list[OpenAIImageGenerationOptionalParams]: # mutable-ok: provider JSON body and base-class dict signature return ["n", "response_format", "size", "user"] # mutable-ok: provider JSON body and base-class dict signature def map_openai_params( @@ -55,14 +57,19 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): "Set drop_params=True to drop unsupported parameters." ) - merged: Final = {**optional_params, **{k: v for k, v in non_default_params.items() if k in allowed}} # mutable-ok: provider JSON body and base-class dict signature + merged: Final = { + **optional_params, + **{k: v for k, v in non_default_params.items() if k in allowed}, + } # mutable-ok: provider JSON body and base-class dict signature size: Final = merged.get("size") aspect_ratio: Final = merged.get("aspect_ratio") or ( _SIZE_TO_ASPECT_RATIO.get(str(size), "1:1") if size else None ) n: Final = merged.get("n") return { # mutable-ok: provider JSON body and base-class dict signature - **({"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"aspect_ratio": aspect_ratio} if aspect_ratio is not None else {} + ), # mutable-ok: provider JSON body and base-class dict signature **({"n": int(n)} if n is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } @@ -142,7 +149,9 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): "model": XAIModelInfo.get_base_model(model) or model, "prompt": prompt, **( - {"aspect_ratio": optional_params["aspect_ratio"]} # mutable-ok: provider JSON body and base-class dict signature + { + "aspect_ratio": optional_params["aspect_ratio"] + } # mutable-ok: provider JSON body and base-class dict signature if optional_params.get("aspect_ratio") is not None else {} # mutable-ok: provider JSON body and base-class dict signature ), @@ -174,7 +183,9 @@ class XAIImageGenerationConfig(BaseImageGenerationConfig): logging_obj.post_call( input=request_data.get("prompt", ""), api_key=api_key, - additional_args={"complete_input_dict": request_data}, # mutable-ok: provider JSON body and base-class dict signature + additional_args={ + "complete_input_dict": request_data + }, # mutable-ok: provider JSON body and base-class dict signature original_response=response_data, ) diff --git a/litellm/llms/xai/videos/transformation.py b/litellm/llms/xai/videos/transformation.py index 039dce3a9fa..64e83c8c673 100644 --- a/litellm/llms/xai/videos/transformation.py +++ b/litellm/llms/xai/videos/transformation.py @@ -58,7 +58,9 @@ _STATUS_MAP: Final = { # mutable-ok: provider JSON body and base-class dict sig class XAIVideoConfig(BaseVideoConfig): - def get_supported_openai_params(self, model: str) -> list: # mutable-ok: provider JSON body and base-class dict signature + def get_supported_openai_params( + self, model: str + ) -> list: # mutable-ok: provider JSON body and base-class dict signature return [ # mutable-ok: provider JSON body and base-class dict signature "model", "prompt", @@ -75,26 +77,42 @@ class XAIVideoConfig(BaseVideoConfig): model: str, drop_params: bool, ) -> dict: # mutable-ok: provider JSON body and base-class dict signature - incoming: Final = dict(video_create_optional_params) # mutable-ok: provider JSON body and base-class dict signature + incoming: Final = dict( + video_create_optional_params + ) # mutable-ok: provider JSON body and base-class dict signature size: Final = incoming.get("size") return { # mutable-ok: provider JSON body and base-class dict signature **{ # mutable-ok: provider JSON body and base-class dict signature key: value for key, value in incoming.items() - if key not in {"seconds", "size", "input_reference", "user", "extra_headers", "model"} # mutable-ok: provider JSON body and base-class dict signature + if key + not in { + "seconds", + "size", + "input_reference", + "user", + "extra_headers", + "model", + } # mutable-ok: provider JSON body and base-class dict signature }, **( - {"duration": _duration_from_seconds(incoming.get("seconds"))} # mutable-ok: provider JSON body and base-class dict signature + { + "duration": _duration_from_seconds(incoming.get("seconds")) + } # mutable-ok: provider JSON body and base-class dict signature if "seconds" in incoming and "duration" not in incoming else {} # mutable-ok: provider JSON body and base-class dict signature ), **( - {"aspect_ratio": incoming.get("aspect_ratio") or _SIZE_TO_ASPECT_RATIO.get(str(size), "16:9")} # mutable-ok: provider JSON body and base-class dict signature + { + "aspect_ratio": incoming.get("aspect_ratio") or _SIZE_TO_ASPECT_RATIO.get(str(size), "16:9") + } # mutable-ok: provider JSON body and base-class dict signature if size and "aspect_ratio" not in incoming else {} # mutable-ok: provider JSON body and base-class dict signature ), **( - {"image": incoming.get("image") or incoming.get("input_reference")} # mutable-ok: provider JSON body and base-class dict signature + { + "image": incoming.get("image") or incoming.get("input_reference") + } # mutable-ok: provider JSON body and base-class dict signature if incoming.get("input_reference") and "image" not in incoming else {} # mutable-ok: provider JSON body and base-class dict signature ), @@ -104,12 +122,16 @@ class XAIVideoConfig(BaseVideoConfig): self, api_base: str | None, api_key: str | None, - litellm_params: GenericLiteLLMParams | dict | None, # mutable-ok: provider JSON body and base-class dict signature + litellm_params: GenericLiteLLMParams + | dict + | None, # mutable-ok: provider JSON body and base-class dict signature ) -> str: from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth params: Final = ( - litellm_params.model_dump() if isinstance(litellm_params, GenericLiteLLMParams) else (litellm_params or {}) # mutable-ok: provider JSON body and base-class dict signature + litellm_params.model_dump() + if isinstance(litellm_params, GenericLiteLLMParams) + else (litellm_params or {}) # mutable-ok: provider JSON body and base-class dict signature ) if should_use_xai_oauth(params) and not XAIModelInfo.get_api_key(api_key): return XAIOAuthAuthenticator().get_api_base().rstrip("/") @@ -142,7 +164,9 @@ class XAIVideoConfig(BaseVideoConfig): should_use_xai_oauth, ) - params: Final = litellm_params.model_dump() if litellm_params is not None else {} # mutable-ok: provider JSON body and base-class dict signature + params: Final = ( + litellm_params.model_dump() if litellm_params is not None else {} + ) # mutable-ok: provider JSON body and base-class dict signature resolved_api_key: Final = api_key or (litellm_params.api_key if litellm_params else None) dynamic_api_key: Final = XAIModelInfo.get_api_key(resolved_api_key) if should_use_xai_oauth(params) and not dynamic_api_key: @@ -209,9 +233,13 @@ class XAIVideoConfig(BaseVideoConfig): return ( { # mutable-ok: provider JSON body and base-class dict signature "model": XAIModelInfo.get_base_model(model) or model, - **({"prompt": prompt} if prompt else {}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"prompt": prompt} if prompt else {} + ), # mutable-ok: provider JSON body and base-class dict signature **copied, - **({"duration": 6} if "duration" not in copied else {}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"duration": 6} if "duration" not in copied else {} + ), # mutable-ok: provider JSON body and base-class dict signature }, [], # mutable-ok: provider JSON body and base-class dict signature api_base, @@ -241,7 +269,9 @@ class XAIVideoConfig(BaseVideoConfig): ) if custom_llm_provider: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model) - video_obj.usage = usage if isinstance(usage, dict) else {} # mutable-ok: provider JSON body and base-class dict signature + video_obj.usage = ( + usage if isinstance(usage, dict) else {} + ) # mutable-ok: provider JSON body and base-class dict signature video_obj._hidden_params["video_url"] = None return video_obj @@ -259,7 +289,9 @@ class XAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, # mutable-ok: provider JSON body and base-class dict signature ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature - return self._video_resource_url(api_base, video_id), {} # mutable-ok: provider JSON body and base-class dict signature + return self._video_resource_url( + api_base, video_id + ), {} # mutable-ok: provider JSON body and base-class dict signature def transform_video_status_retrieve_response( self, @@ -270,7 +302,9 @@ class XAIVideoConfig(BaseVideoConfig): response_data: Final = raw_response.json() status_raw: Final = str(response_data.get("status") or "processing").lower() status: Final = _STATUS_MAP.get(status_raw, status_raw) - video_meta: Final = response_data.get("video") or {} # mutable-ok: provider JSON body and base-class dict signature + video_meta: Final = ( + response_data.get("video") or {} + ) # mutable-ok: provider JSON body and base-class dict signature video_url: Final = video_meta.get("url") if isinstance(video_meta, dict) else None seconds: Final = ( str(video_meta.get("duration")) @@ -291,7 +325,9 @@ class XAIVideoConfig(BaseVideoConfig): model=response_data.get("model"), progress=response_data.get("progress"), seconds=seconds, - usage=response_data.get("usage") if isinstance(response_data.get("usage"), dict) else {}, # mutable-ok: provider JSON body and base-class dict signature + usage=response_data.get("usage") + if isinstance(response_data.get("usage"), dict) + else {}, # mutable-ok: provider JSON body and base-class dict signature ) video_obj._hidden_params["video_url"] = video_url if custom_llm_provider and video_obj.id and video_obj.id != "unknown": @@ -306,7 +342,9 @@ class XAIVideoConfig(BaseVideoConfig): headers: dict, # mutable-ok: provider JSON body and base-class dict signature variant: str | None = None, ) -> tuple[str, dict]: # mutable-ok: provider JSON body and base-class dict signature - return self._video_resource_url(api_base, video_id), {} # mutable-ok: provider JSON body and base-class dict signature + return self._video_resource_url( + api_base, video_id + ), {} # mutable-ok: provider JSON body and base-class dict signature def _video_cdn_url(self, raw_response: httpx.Response) -> str | None: content_type: Final = (raw_response.headers.get("content-type") or "").lower() diff --git a/litellm/proxy/image_endpoints/endpoints.py b/litellm/proxy/image_endpoints/endpoints.py index 615e821270d..6d65748387f 100644 --- a/litellm/proxy/image_endpoints/endpoints.py +++ b/litellm/proxy/image_endpoints/endpoints.py @@ -60,7 +60,9 @@ async def batch_to_bytesio( """ if not uploads: return None - return [await uploadfile_to_bytesio(u) for u in uploads] # mutable-ok: provider JSON body and base-class dict signature + return [ + await uploadfile_to_bytesio(u) for u in uploads + ] # mutable-ok: provider JSON body and base-class dict signature def _is_image_reference_string(value: str) -> bool: @@ -98,7 +100,9 @@ async def _normalize_image_values(values: tuple[object, ...], field: str) -> obj return list(coerced) # mutable-ok: provider JSON body and base-class dict signature -def _json_image_values(data: dict[str, object], field: str) -> tuple[object, ...]: # mutable-ok: provider JSON body and base-class dict signature +def _json_image_values( + data: dict[str, object], field: str +) -> tuple[object, ...]: # mutable-ok: provider JSON body and base-class dict signature if field not in data: return () raw: Final = data[field] @@ -113,7 +117,9 @@ async def _normalized_image_edit_fields( image: Final = await _normalize_image_values(values_by_field["image"], "image") mask: Final = await _normalize_image_values(values_by_field["mask"], "mask") return { # mutable-ok: provider JSON body and base-class dict signature - **({"image": image} if image is not None else {}), # mutable-ok: provider JSON body and base-class dict signature + **( + {"image": image} if image is not None else {} + ), # mutable-ok: provider JSON body and base-class dict signature **({"mask": mask} if mask is not None else {}), # mutable-ok: provider JSON body and base-class dict signature } @@ -125,9 +131,13 @@ async def _image_edit_assets_from_request( form: Final = await request.form() if _is_form_content_type(request.headers.get("content-type", "")) else None if form is None: return { # mutable-ok: provider JSON body and base-class dict signature - **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, # mutable-ok: provider JSON body and base-class dict signature + **{ + key: value for key, value in data.items() if key not in {"image[]", "mask[]"} + }, # mutable-ok: provider JSON body and base-class dict signature **await _normalized_image_edit_fields( - {field: _json_image_values(data, field) for field, _alias in _IMAGE_EDIT_FILE_FIELDS} # mutable-ok: provider JSON body and base-class dict signature + { + field: _json_image_values(data, field) for field, _alias in _IMAGE_EDIT_FILE_FIELDS + } # mutable-ok: provider JSON body and base-class dict signature ), } @@ -143,9 +153,13 @@ async def _image_edit_assets_from_request( detail=f"Cannot specify both '{conflicts[0]}' and '{conflicts[0]}[]'", ) return { # mutable-ok: provider JSON body and base-class dict signature - **{key: value for key, value in data.items() if key not in {"image[]", "mask[]"}}, # mutable-ok: provider JSON body and base-class dict signature + **{ + key: value for key, value in data.items() if key not in {"image[]", "mask[]"} + }, # mutable-ok: provider JSON body and base-class dict signature **await _normalized_image_edit_fields( - {field: form_values[field] or form_values[alias] for field, alias in _IMAGE_EDIT_FILE_FIELDS} # mutable-ok: provider JSON body and base-class dict signature + { + field: form_values[field] or form_values[alias] for field, alias in _IMAGE_EDIT_FILE_FIELDS + } # mutable-ok: provider JSON body and base-class dict signature ), } @@ -257,7 +271,9 @@ async def image_generation( ) ### RESPONSE HEADERS ### - hidden_params: Final = getattr(response, "_hidden_params", {}) or {} # mutable-ok: provider JSON body and base-class dict signature + hidden_params: Final = ( + getattr(response, "_hidden_params", {}) or {} + ) # mutable-ok: provider JSON body and base-class dict signature model_id: Final = hidden_params.get("model_id", None) or "" cache_key: Final = hidden_params.get("cache_key", None) or "" api_base: Final = hidden_params.get("api_base", None) or "" @@ -373,7 +389,9 @@ async def image_edit_api( with_assets: Final = await _image_edit_assets_from_request(request, parsed_body) data: Final = { # mutable-ok: provider JSON body and base-class dict signature **with_assets, - **({} if "prompt" in with_assets else {"prompt": None}), # mutable-ok: provider JSON body and base-class dict signature + **( + {} if "prompt" in with_assets else {"prompt": None} + ), # mutable-ok: provider JSON body and base-class dict signature "model": ( model or general_settings.get("image_generation_model", None) or user_model or with_assets.get("model") ), diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 6da6301aac5..84fec0e33ee 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -17,11 +17,14 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( ) from litellm.proxy.image_endpoints.endpoints import batch_to_bytesio from litellm.proxy.video_endpoints.utils import ( + assert_video_owner, encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, infer_video_provider_from_model, resolve_video_request_model, + stamp_video_owner, + video_owner_from_key, video_reference_to_id, ) from litellm.types.videos.utils import ( @@ -91,7 +94,7 @@ async def video_generation( # Process request using ProxyBaseLLMRequestProcessing processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - return await processor.base_process_llm_request( + result = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -109,6 +112,9 @@ async def video_generation( user_api_base=user_api_base, version=version, ) + return _stamp_generated_video_owner( + result, video_owner_from_key(user_api_key_dict.token, user_api_key_dict.api_key) + ) except Exception as e: raise await processor._handle_llm_api_exception( e=e, @@ -118,6 +124,16 @@ async def video_generation( ) +def _stamp_generated_video_owner(result: object, owner: str | None) -> object: + video_id: Final = getattr(result, "id", None) + if isinstance(video_id, str): + result.id = stamp_video_owner(video_id, owner) # mutable-ok: response object id is rewritten before return + return result + if isinstance(result, dict) and isinstance(result.get("id"), str): + result["id"] = stamp_video_owner(result["id"], owner) # mutable-ok: JSON response body + return result + + @router.get( "/v1/videos", dependencies=[Depends(user_api_key_auth)], @@ -247,6 +263,8 @@ async def video_status( version, ) + assert_video_owner(video_id, video_owner_from_key(user_api_key_dict.token, user_api_key_dict.api_key)) + # Create data with video_id data: Final[dict[str, object]] = {"video_id": video_id} @@ -348,6 +366,8 @@ async def video_content( version, ) + assert_video_owner(video_id, video_owner_from_key(user_api_key_dict.token, user_api_key_dict.api_key)) + # Create data with video_id data: Final[dict[str, object]] = {"video_id": video_id} diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py index 9795e925863..baa2aea81c3 100644 --- a/litellm/proxy/video_endpoints/utils.py +++ b/litellm/proxy/video_endpoints/utils.py @@ -3,13 +3,44 @@ from typing import Final, Protocol import orjson -from litellm.types.videos.utils import encode_character_id_with_provider +from litellm.proxy._types import ProxyException +from litellm.types.videos.utils import ( + decode_video_id_with_provider, + encode_character_id_with_provider, + encode_video_id_with_provider, +) class VideoModelIdResolver(Protocol): def resolve_model_name_from_model_id(self, model_id: str | None) -> str | None: ... +def video_owner_from_key(token: str | None, api_key: str | None) -> str | None: + return token or api_key + + +def assert_video_owner(video_id: str, owner: str | None) -> None: + recorded: Final = decode_video_id_with_provider(video_id).get("owner") + if recorded and recorded != owner: + raise ProxyException( + message="Video does not belong to this API key", + type="permission_error", + param="video_id", + code=403, + ) + + +def stamp_video_owner(video_id: str, owner: str | None) -> str: + if not owner: + return video_id + decoded: Final = decode_video_id_with_provider(video_id) + provider: Final = decoded.get("custom_llm_provider") + raw_id: Final = decoded.get("video_id") + if not provider or not raw_id or decoded.get("owner"): + return video_id + return encode_video_id_with_provider(raw_id, provider, decoded.get("model_id"), owner) + + def infer_video_provider_from_model(model: str | None) -> str | None: if not isinstance(model, str) or not model: return None diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index f4369fd95af..010bcfd9ddd 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -102,6 +102,7 @@ class DecodedVideoId(TypedDict, total=False): custom_llm_provider: str | None model_id: str | None video_id: str + owner: str | None class CharacterObject(BaseModel): diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index b23b2269543..e07f71b3240 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -35,7 +35,9 @@ def _add_base64_padding(value: str) -> str: return value -def encode_video_id_with_provider(video_id: str, provider: str, model_id: str | None = None) -> str: +def encode_video_id_with_provider( + video_id: str, provider: str, model_id: str | None = None, owner: str | None = None +) -> str: """Encode provider and model_id into video_id using base64.""" if not provider or not video_id: return video_id @@ -50,6 +52,8 @@ def encode_video_id_with_provider(video_id: str, provider: str, model_id: str | # ID is not encoded (even if it starts with video_), so encode it assembled_id = str(SpecialEnums.LITELLM_MANAGED_VIDEO_COMPLETE_STR.value).format(provider, model_id or "", video_id) + if owner: + assembled_id = f"{assembled_id};owner:{owner}" base64_encoded_id: Final[str] = base64.b64encode(assembled_id.encode("utf-8")).decode("utf-8") @@ -89,20 +93,24 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: custom_llm_provider = None model_id = None decoded_video_id = encoded_video_id + owner = None if len(parts) >= 3: custom_llm_provider_part: Final = parts[0] model_id_part: Final = parts[1] video_id_part: Final = parts[2] + owner_part: Final = next((part for part in parts[3:] if part.startswith("owner:")), None) custom_llm_provider = custom_llm_provider_part.replace("litellm:custom_llm_provider:", "") model_id = model_id_part.replace("model_id:", "") decoded_video_id = video_id_part.replace("video_id:", "") + owner = owner_part.removeprefix("owner:") if owner_part else None return DecodedVideoId( custom_llm_provider=custom_llm_provider, model_id=model_id, video_id=decoded_video_id, + owner=owner, ) except Exception as e: verbose_logger.debug("Error decoding video_id '%s': %s", encoded_video_id, e) diff --git a/tests/test_litellm/proxy/video_endpoints/test_utils.py b/tests/test_litellm/proxy/video_endpoints/test_utils.py index af8af6cdcf2..3fb523944aa 100644 --- a/tests/test_litellm/proxy/video_endpoints/test_utils.py +++ b/tests/test_litellm/proxy/video_endpoints/test_utils.py @@ -13,21 +13,23 @@ is encode_character_id_with_provider, which runs for real; encoding assertions are checked by the genuine decode round-trip. """ - import pytest - +from litellm.proxy._types import ProxyException from litellm.proxy.video_endpoints.utils import ( + assert_video_owner, encode_character_id_in_response, extract_model_from_target_model_names, get_custom_provider_from_data, infer_video_provider_from_model, resolve_video_request_model, + stamp_video_owner, video_reference_to_id, ) from litellm.types.videos.utils import ( decode_character_id_with_provider, encode_character_id_with_provider, + encode_video_id_with_provider, ) # =========================================================================== # @@ -166,12 +168,7 @@ def test_provider__falsy_top_level_falls_through_to_extra_body(falsy): def test_provider__from_extra_body_dict(): - assert ( - get_custom_provider_from_data( - {"extra_body": {"custom_llm_provider": "bedrock"}} - ) - == "bedrock" - ) + assert get_custom_provider_from_data({"extra_body": {"custom_llm_provider": "bedrock"}}) == "bedrock" def test_provider__from_extra_body_json_string(): @@ -189,10 +186,7 @@ def test_provider__json_string_parsing_to_non_dict_is_none(): def test_provider__extra_body_provider_not_a_string_is_none(): - assert ( - get_custom_provider_from_data({"extra_body": {"custom_llm_provider": 123}}) - is None - ) + assert get_custom_provider_from_data({"extra_body": {"custom_llm_provider": 123}}) is None @pytest.mark.parametrize( @@ -224,9 +218,7 @@ def test_encode__dict_with_id_mutates_in_place_and_preserves_other_keys(): assert out is response # same dict, mutated in place assert out["object"] == "character" and out["name"] == "hero" - assert out["id"] == encode_character_id_with_provider( - "char_raw", "azure", "model-1" - ) + assert out["id"] == encode_character_id_with_provider("char_raw", "azure", "model-1") decoded = decode_character_id_with_provider(out["id"]) assert decoded["custom_llm_provider"] == "azure" assert decoded["model_id"] == "model-1" @@ -264,6 +256,16 @@ def test_encode__object_non_str_or_empty_id_unchanged(bad_id): assert resp.id == bad_id # untouched +def test_stamp_and_assert_video_owner_round_trip(): + encoded = encode_video_id_with_provider("req_1", "xai", "grok-imagine-video") + stamped = stamp_video_owner(encoded, "key-hash") + assert stamped != encoded + assert_video_owner(stamped, "key-hash") + with pytest.raises(ProxyException): + assert_video_owner(stamped, "other-key") + assert_video_owner(encoded, "other-key") + + def test_encode__object_without_id_attr_returned_unchanged(): resp = _Resp() out = encode_character_id_in_response(resp, "azure", "model-1")