diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 2201a63363d..6fb6f69a9e8 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -11,6 +11,7 @@ from litellm.types.videos.main import VideoCreateOptionalRequestParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.videos.main import CharacterObject as _CharacterObject from litellm.types.videos.main import VideoObject as _VideoObject from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -18,10 +19,12 @@ if TYPE_CHECKING: LiteLLMLoggingObj = _LiteLLMLoggingObj BaseLLMException = _BaseLLMException VideoObject = _VideoObject + CharacterObject = _CharacterObject else: LiteLLMLoggingObj = Any BaseLLMException = Any VideoObject = Any + CharacterObject = Any class BaseVideoConfig(ABC): @@ -265,6 +268,110 @@ class BaseVideoConfig(ABC): ) -> VideoObject: pass + @abstractmethod + def transform_video_create_character_request( + self, + name: str, + video: Any, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, list]: + """ + Transform the video create character request into a URL and files list (multipart). + + Returns: + Tuple[str, list]: (url, files_list) for the multipart POST request + """ + pass + + @abstractmethod + def transform_video_create_character_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CharacterObject: + pass + + @abstractmethod + def transform_video_get_character_request( + self, + character_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video get character request into a URL and params. + + Returns: + Tuple[str, Dict]: (url, params) for the GET request + """ + pass + + @abstractmethod + def transform_video_get_character_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CharacterObject: + pass + + @abstractmethod + def transform_video_edit_request( + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Transform the video edit request into a URL and JSON data. + + Returns: + Tuple[str, Dict]: (url, data) for the POST request + """ + pass + + @abstractmethod + def transform_video_edit_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + pass + + @abstractmethod + def transform_video_extension_request( + self, + prompt: str, + video_id: str, + seconds: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Transform the video extension request into a URL and JSON data. + + Returns: + Tuple[str, Dict]: (url, data) for the POST request + """ + pass + + @abstractmethod + def transform_video_extension_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + pass + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4394343c8e3..28f3b24a95a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6114,6 +6114,606 @@ class BaseLLMHTTPHandler: provider_config=video_remix_provider_config, ) + def video_create_character_handler( + self, + name: str, + video: Any, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_create_character_handler( + name=name, + video=video, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, files_list = video_provider_config.transform_video_create_character_request( + name=name, + video=video, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=name, + api_key="", + additional_args={ + "complete_input_dict": {"name": name}, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + files=files_list, + timeout=timeout, + ) + return video_provider_config.transform_video_create_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_create_character_handler( + self, + name: str, + video: Any, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, files_list = video_provider_config.transform_video_create_character_request( + name=name, + video=video, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=name, + api_key="", + additional_args={ + "complete_input_dict": {"name": name}, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + files=files_list, + timeout=timeout, + ) + return video_provider_config.transform_video_create_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + def video_get_character_handler( + self, + character_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_get_character_handler( + character_id=character_id, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, params = video_provider_config.transform_video_get_character_request( + character_id=character_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=character_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params + ) + return video_provider_config.transform_video_get_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_get_character_handler( + self, + character_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, params = video_provider_config.transform_video_get_character_request( + character_id=character_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=character_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params + ) + return video_provider_config.transform_video_get_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + def video_edit_handler( + self, + prompt: str, + video_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_edit_handler( + prompt=prompt, + video_id=video_id, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_edit_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_edit_handler( + self, + prompt: str, + video_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_edit_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + def video_extension_handler( + self, + prompt: str, + video_id: str, + seconds: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_extension_handler( + prompt=prompt, + video_id=video_id, + seconds=seconds, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_extension_request( + prompt=prompt, + video_id=video_id, + seconds=seconds, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_extension_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_extension_handler( + self, + prompt: str, + video_id: str, + seconds: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_extension_request( + prompt=prompt, + video_id=video_id, + seconds=seconds, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_extension_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + def video_list_handler( self, after: Optional[str], diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index c16b20fe579..0798472310e 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -1,29 +1,30 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union import base64 +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union import httpx from httpx._types import RequestFiles -from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject -from litellm.types.router import GenericLiteLLMParams -from litellm.secret_managers.main import get_secret_str -from litellm.types.videos.utils import ( - encode_video_id_with_provider, - extract_original_video_id, -) -from litellm.images.utils import ImageEditRequestUtils import litellm +from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.gemini import ( GeminiLongRunningOperationResponse, GeminiVideoGenerationInstance, GeminiVideoGenerationParameters, GeminiVideoGenerationRequest, ) -from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS -from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import ( + encode_video_id_with_provider, + extract_original_video_id, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -524,6 +525,30 @@ class GeminiVideoConfig(BaseVideoConfig): """Video delete is not supported.""" raise NotImplementedError("Video delete is not supported by Google Veo.") + def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + raise NotImplementedError("video create character is not supported for Gemini") + + def transform_video_create_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video create character is not supported for Gemini") + + def transform_video_get_character_request(self, character_id, api_base, litellm_params, headers): + raise NotImplementedError("video get character is not supported for Gemini") + + def transform_video_get_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video get character is not supported for Gemini") + + def transform_video_edit_request(self, prompt, video_id, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video edit is not supported for Gemini") + + def transform_video_edit_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video edit is not supported for Gemini") + + def transform_video_extension_request(self, prompt, video_id, seconds, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video extension is not supported for Gemini") + + def transform_video_extension_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video extension is not supported for Gemini") + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 0501a67fc16..61baa56949c 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -18,6 +18,7 @@ from litellm.types.videos.main import ( ) from litellm.types.videos.utils import ( encode_video_id_with_provider, + extract_original_character_id, extract_original_video_id, ) @@ -51,6 +52,7 @@ class OpenAIVideoConfig(BaseVideoConfig): "input_reference", "seconds", "size", + "characters", "user", "extra_headers", ] @@ -126,6 +128,7 @@ class OpenAIVideoConfig(BaseVideoConfig): model=model, prompt=prompt, **video_create_optional_request_params ) request_dict = cast(Dict, video_create_request) + request_dict = self._decode_character_ids_in_create_video_request(request_dict) # Handle input_reference parameter if provided _input_reference = video_create_optional_request_params.get("input_reference") @@ -143,6 +146,35 @@ class OpenAIVideoConfig(BaseVideoConfig): ) return data_without_files, files_list, api_base + def _decode_character_ids_in_create_video_request(self, request_dict: Dict) -> Dict: + """ + Decode LiteLLM-managed encoded character ids for provider requests. + + OpenAI expects character ids like `char_...`. If a caller sends + `character_`, convert it back to the + original provider id before forwarding upstream. + """ + raw_characters = request_dict.get("characters") + if not isinstance(raw_characters, list): + return request_dict + + decoded_characters: List[Any] = [] + for character in raw_characters: + if not isinstance(character, dict): + decoded_characters.append(character) + continue + + character_id = character.get("id") + if isinstance(character_id, str): + decoded_character = dict(character) + decoded_character["id"] = extract_original_character_id(character_id) + decoded_characters.append(decoded_character) + else: + decoded_characters.append(character) + + request_dict["characters"] = decoded_characters + return request_dict + def transform_video_create_response( self, model: str, diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 3fc656a92bd..2c29c2e21ee 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -592,6 +592,30 @@ class RunwayMLVideoConfig(BaseVideoConfig): return video_obj + def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + raise NotImplementedError("video create character is not supported for RunwayML") + + def transform_video_create_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video create character is not supported for RunwayML") + + def transform_video_get_character_request(self, character_id, api_base, litellm_params, headers): + raise NotImplementedError("video get character is not supported for RunwayML") + + def transform_video_get_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video get character is not supported for RunwayML") + + def transform_video_edit_request(self, prompt, video_id, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video edit is not supported for RunwayML") + + def transform_video_edit_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video edit is not supported for RunwayML") + + def transform_video_extension_request(self, prompt, video_id, seconds, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video extension is not supported for RunwayML") + + def transform_video_extension_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video extension is not supported for RunwayML") + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index e61f2f46ec8..07b3d6faf70 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -624,6 +624,30 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): """Video delete is not supported.""" raise NotImplementedError("Video delete is not supported by Vertex AI Veo.") + def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + raise NotImplementedError("video create character is not supported for Vertex AI") + + def transform_video_create_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video create character is not supported for Vertex AI") + + def transform_video_get_character_request(self, character_id, api_base, litellm_params, headers): + raise NotImplementedError("video get character is not supported for Vertex AI") + + def transform_video_get_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video get character is not supported for Vertex AI") + + def transform_video_edit_request(self, prompt, video_id, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video edit is not supported for Vertex AI") + + def transform_video_edit_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video edit is not supported for Vertex AI") + + def transform_video_extension_request(self, prompt, video_id, seconds, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video extension is not supported for Vertex AI") + + def transform_video_extension_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video extension is not supported for Vertex AI") + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a9e9d519f6f..72765aab7da 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -599,6 +599,10 @@ class ProxyBaseLLMRequestProcessing: "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_container", "alist_containers", "aingest", @@ -850,6 +854,10 @@ class ProxyBaseLLMRequestProcessing: "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_container", "alist_containers", "aingest", diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 6e02d28b383..a8169ca6d08 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -54,6 +54,10 @@ ROUTE_ENDPOINT_MAPPING = { "avideo_status": "/videos/{video_id}", "avideo_content": "/videos/{video_id}/content", "avideo_remix": "/videos/{video_id}/remix", + "avideo_create_character": "/videos/characters", + "avideo_get_character": "/videos/characters/{character_id}", + "avideo_edit": "/videos/edits", + "avideo_extension": "/videos/extensions", "acreate_realtime_client_secret": "/realtime/client_secrets", "arealtime_calls": "/realtime/calls", "acreate_container": "/containers", @@ -201,6 +205,10 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_container", "alist_containers", "aretrieve_container", @@ -370,6 +378,10 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "avector_store_file_list", "avector_store_file_retrieve", "avector_store_file_content", @@ -449,6 +461,8 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_edit", + "avideo_extension", ]: # Video endpoints: If model is provided (e.g., from decoded video_id), try router first try: diff --git a/litellm/router.py b/litellm/router.py index 0fd4ba80c74..f34368172ac 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1076,12 +1076,20 @@ class Router: """Initialize video endpoints.""" from litellm.videos import ( avideo_content, + avideo_create_character, + avideo_edit, + avideo_extension, avideo_generation, + avideo_get_character, avideo_list, avideo_remix, avideo_status, video_content, + video_create_character, + video_edit, + video_extension, video_generation, + video_get_character, video_list, video_remix, video_status, @@ -1111,6 +1119,26 @@ class Router: avideo_remix, call_type="avideo_remix" ) self.video_remix = self.factory_function(video_remix, call_type="video_remix") + self.avideo_create_character = self.factory_function( + avideo_create_character, call_type="avideo_create_character" + ) + self.video_create_character = self.factory_function( + video_create_character, call_type="video_create_character" + ) + self.avideo_get_character = self.factory_function( + avideo_get_character, call_type="avideo_get_character" + ) + self.video_get_character = self.factory_function( + video_get_character, call_type="video_get_character" + ) + self.avideo_edit = self.factory_function(avideo_edit, call_type="avideo_edit") + self.video_edit = self.factory_function(video_edit, call_type="video_edit") + self.avideo_extension = self.factory_function( + avideo_extension, call_type="avideo_extension" + ) + self.video_extension = self.factory_function( + video_extension, call_type="video_extension" + ) def _initialize_container_endpoints(self): """Initialize container endpoints.""" @@ -4828,6 +4856,14 @@ class Router: "video_content", "avideo_remix", "video_remix", + "avideo_create_character", + "video_create_character", + "avideo_get_character", + "video_get_character", + "avideo_edit", + "video_edit", + "avideo_extension", + "video_extension", "acreate_container", "create_container", "alist_containers", @@ -4995,6 +5031,10 @@ class Router: "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_skill", "alist_skills", "aget_skill", diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 661cdd87099..b65db466b9f 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -1,4 +1,5 @@ import asyncio +import io import json import os import sys @@ -174,6 +175,34 @@ class TestVideoGeneration: assert files == [] assert returned_api_base == "https://api.openai.com/v1/videos" + def test_video_generation_request_decodes_encoded_character_ids(self): + """Encoded character IDs should be decoded before upstream create-video call.""" + from litellm.types.videos.utils import encode_character_id_with_provider + + config = OpenAIVideoConfig() + encoded_character_id = encode_character_id_with_provider( + character_id="char_123", + provider="openai", + model_id="sora-2", + ) + + data, files, returned_api_base = config.transform_video_create_request( + model="sora-2", + prompt="Test video prompt", + api_base="https://api.openai.com/v1/videos", + video_create_optional_request_params={ + "seconds": "8", + "size": "720x1280", + "characters": [{"id": encoded_character_id}], + }, + litellm_params=MagicMock(), + headers={}, + ) + + assert data["characters"] == [{"id": "char_123"}] + assert files == [] + assert returned_api_base == "https://api.openai.com/v1/videos" + def test_video_generation_response_transformation(self): """Test video generation response transformation.""" config = OpenAIVideoConfig() @@ -1623,3 +1652,516 @@ def test_video_remix_handler_prefers_explicit_api_key(): if __name__ == "__main__": pytest.main([__file__]) + + +# ===== Tests for new video endpoints (characters, edits, extensions) ===== + + +class TestVideoCreateCharacter: + """Tests for video_create_character / avideo_create_character.""" + + def test_video_create_character_transform_request(self): + """Verify multipart form construction for POST /videos/characters.""" + config = OpenAIVideoConfig() + fake_video = b"fake_video_bytes" + + url, files_list = config.transform_video_create_character_request( + name="hero", + video=fake_video, + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/characters" + # Should have (name field) + (video file field) = 2 entries + assert len(files_list) == 2 + field_names = [f[0] for f in files_list] + assert "name" in field_names + assert "video" in field_names + + def test_video_create_character_sets_video_mimetype(self): + """Ensure character video upload is sent as video/mp4.""" + config = OpenAIVideoConfig() + fake_video = io.BytesIO(b"....ftyp....video-bytes") + fake_video.name = "character.mp4" + + _, files_list = config.transform_video_create_character_request( + name="hero", + video=fake_video, + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + video_parts = [f for f in files_list if f[0] == "video"] + assert len(video_parts) == 1 + video_tuple = video_parts[0][1] + assert video_tuple[0] == "character.mp4" + assert video_tuple[2] == "video/mp4" + + def test_video_create_character_transform_response(self): + """Verify CharacterObject is returned from response.""" + from litellm.types.videos.main import CharacterObject + + config = OpenAIVideoConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "char_abc123", + "object": "character", + "created_at": 1712697600, + "name": "hero", + } + + result = config.transform_video_create_character_response( + raw_response=mock_response, + logging_obj=MagicMock(), + ) + + assert isinstance(result, CharacterObject) + assert result.id == "char_abc123" + assert result.name == "hero" + + def test_video_create_character_mock_response(self): + """video_create_character returns CharacterObject on mock_response.""" + from litellm.types.videos.main import CharacterObject + from litellm.videos.main import video_create_character + + response = video_create_character( + name="hero", + video=b"fake", + mock_response={ + "id": "char_abc", + "object": "character", + "created_at": 1712697600, + "name": "hero", + }, + ) + assert isinstance(response, CharacterObject) + assert response.id == "char_abc" + + +class TestVideoGetCharacter: + """Tests for video_get_character / avideo_get_character.""" + + def test_video_get_character_transform_request(self): + """Verify URL construction for GET /videos/characters/{character_id}.""" + config = OpenAIVideoConfig() + + url, params = config.transform_video_get_character_request( + character_id="char_xyz", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/characters/char_xyz" + assert params == {} + + def test_video_get_character_transform_response(self): + """Verify CharacterObject is returned from GET response.""" + from litellm.types.videos.main import CharacterObject + + config = OpenAIVideoConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "char_xyz", + "object": "character", + "created_at": 1712697600, + "name": "villain", + } + + result = config.transform_video_get_character_response( + raw_response=mock_response, + logging_obj=MagicMock(), + ) + + assert isinstance(result, CharacterObject) + assert result.id == "char_xyz" + assert result.name == "villain" + + def test_video_get_character_mock_response(self): + """video_get_character returns CharacterObject on mock_response.""" + from litellm.types.videos.main import CharacterObject + from litellm.videos.main import video_get_character + + response = video_get_character( + character_id="char_xyz", + mock_response={ + "id": "char_xyz", + "object": "character", + "created_at": 1712697600, + "name": "villain", + }, + ) + assert isinstance(response, CharacterObject) + assert response.id == "char_xyz" + + +class TestVideoEdit: + """Tests for video_edit / avideo_edit.""" + + def test_video_edit_transform_request(self): + """Verify JSON body with video.id for POST /videos/edits.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_edit_request( + prompt="make it brighter", + video_id="video_abc123", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/edits" + assert data["prompt"] == "make it brighter" + assert data["video"]["id"] == "video_abc123" + + def test_video_edit_transform_request_with_extra_body(self): + """Extra body params are merged into request data.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_edit_request( + prompt="darken it", + video_id="video_abc123", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + extra_body={"resolution": "1080p"}, + ) + + assert data["resolution"] == "1080p" + + def test_video_edit_mock_response(self): + """video_edit returns VideoObject on mock_response.""" + from litellm.videos.main import video_edit + + response = video_edit( + video_id="video_abc123", + prompt="make it brighter", + mock_response={ + "id": "video_edit_001", + "object": "video", + "status": "queued", + "created_at": 1712697600, + }, + ) + assert isinstance(response, VideoObject) + assert response.id == "video_edit_001" + + def test_video_edit_strips_encoded_provider_from_video_id(self): + """Provider-encoded video IDs are decoded before sending to API.""" + from litellm.types.videos.utils import encode_video_id_with_provider + config = OpenAIVideoConfig() + + encoded_id = encode_video_id_with_provider("raw_video_id", "openai", None) + url, data = config.transform_video_edit_request( + prompt="test", + video_id=encoded_id, + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + # The video.id in the request body should be the raw ID, not the encoded one + assert data["video"]["id"] == "raw_video_id" + + +class TestVideoExtension: + """Tests for video_extension / avideo_extension.""" + + def test_video_extension_transform_request(self): + """Verify JSON body with video.id + seconds for POST /videos/extensions.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_extension_request( + prompt="continue the scene", + video_id="video_abc123", + seconds="5", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/extensions" + assert data["prompt"] == "continue the scene" + assert data["seconds"] == "5" + assert data["video"]["id"] == "video_abc123" + + def test_video_extension_transform_request_with_extra_body(self): + """Extra body params are merged into request data.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_extension_request( + prompt="extend", + video_id="video_abc123", + seconds="10", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + extra_body={"model": "sora-2"}, + ) + + assert data["model"] == "sora-2" + + def test_video_extension_mock_response(self): + """video_extension returns VideoObject on mock_response.""" + from litellm.videos.main import video_extension + + response = video_extension( + video_id="video_abc123", + prompt="continue the scene", + seconds="5", + mock_response={ + "id": "video_ext_001", + "object": "video", + "status": "queued", + "created_at": 1712697600, + }, + ) + assert isinstance(response, VideoObject) + assert response.id == "video_ext_001" + + def test_video_extension_strips_encoded_provider_from_video_id(self): + """Provider-encoded video IDs are decoded before sending to API.""" + from litellm.types.videos.utils import encode_video_id_with_provider + config = OpenAIVideoConfig() + + encoded_id = encode_video_id_with_provider("raw_video_id", "openai", None) + url, data = config.transform_video_extension_request( + prompt="extend", + video_id=encoded_id, + seconds="5", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert data["video"]["id"] == "raw_video_id" + + +@pytest.fixture +def video_proxy_test_client(): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.video_endpoints.endpoints import router as video_router + + app = FastAPI() + app.include_router(video_router) + app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + return TestClient(app) + + +def test_character_id_encode_decode_roundtrip(): + from litellm.types.videos.utils import ( + decode_character_id_with_provider, + encode_character_id_with_provider, + ) + + encoded = encode_character_id_with_provider( + character_id="char_raw_123", + provider="vertex_ai", + model_id="veo-2.0-generate-001", + ) + decoded = decode_character_id_with_provider(encoded) + + assert decoded["character_id"] == "char_raw_123" + assert decoded["custom_llm_provider"] == "vertex_ai" + assert decoded["model_id"] == "veo-2.0-generate-001" + + +def test_character_id_decode_handles_missing_base64_padding(): + from litellm.types.videos.utils import ( + decode_character_id_with_provider, + encode_character_id_with_provider, + ) + + encoded = encode_character_id_with_provider( + character_id="id", + provider="openai", + model_id="gpt-4o", + ) + encoded_without_padding = encoded.rstrip("=") + decoded = decode_character_id_with_provider(encoded_without_padding) + + assert decoded["character_id"] == "id" + assert decoded["custom_llm_provider"] == "openai" + assert decoded["model_id"] == "gpt-4o" + + +def test_video_create_character_target_model_names_returns_encoded_id(video_proxy_test_client): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.videos.utils import decode_character_id_with_provider + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "char_upstream_123", + "object": "character", + "created_at": 1712697600, + "name": "hero", + } + + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + "/v1/videos/characters", + headers={"Authorization": "Bearer sk-1234"}, + files={"video": ("character.mp4", b"fake-video", "video/mp4")}, + data={ + "name": "hero", + "target_model_names": "vertex-ai-sora-2", + "extra_body": json.dumps({"custom_llm_provider": "vertex_ai"}), + }, + ) + + assert response.status_code == 200, response.text + response_json = response.json() + decoded = decode_character_id_with_provider(response_json["id"]) + assert decoded["character_id"] == "char_upstream_123" + assert decoded["custom_llm_provider"] == "vertex_ai" + assert decoded["model_id"] == "vertex-ai-sora-2" + assert captured_data["model"] == "vertex-ai-sora-2" + assert captured_data["custom_llm_provider"] == "vertex_ai" + + +def test_video_get_character_accepts_encoded_character_id(video_proxy_test_client): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.videos.utils import ( + decode_character_id_with_provider, + encode_character_id_with_provider, + ) + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "char_upstream_123", + "object": "character", + "created_at": 1712697600, + "name": "hero", + } + + encoded_character_id = encode_character_id_with_provider( + character_id="char_upstream_123", + provider="vertex_ai", + model_id="veo-2.0-generate-001", + ) + mock_router = MagicMock() + mock_router.resolve_model_name_from_model_id.return_value = "vertex-ai-sora-2" + + with patch("litellm.proxy.proxy_server.llm_router", mock_router): + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.get( + f"/v1/videos/characters/{encoded_character_id}", + headers={"Authorization": "Bearer sk-1234"}, + ) + + assert response.status_code == 200, response.text + assert captured_data["character_id"] == "char_upstream_123" + assert captured_data["custom_llm_provider"] == "vertex_ai" + assert captured_data["model"] == "vertex-ai-sora-2" + response_decoded = decode_character_id_with_provider(response.json()["id"]) + assert response_decoded["character_id"] == "char_upstream_123" + assert response_decoded["custom_llm_provider"] == "vertex_ai" + assert response_decoded["model_id"] == "veo-2.0-generate-001" + + +@pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) +def test_edit_and_extension_support_custom_provider_from_extra_body( + video_proxy_test_client, endpoint +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "video_resp_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + } + + payload = { + "prompt": "test", + "video": {"id": "video_raw_123"}, + "extra_body": {"custom_llm_provider": "vertex_ai"}, + } + if endpoint.endswith("extensions"): + payload["seconds"] = "4" + + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + endpoint, + headers={"Authorization": "Bearer sk-1234"}, + json=payload, + ) + + assert response.status_code == 200, response.text + assert captured_data["custom_llm_provider"] == "vertex_ai" + + +@pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) +def test_edit_and_extension_route_with_encoded_video_ids( + video_proxy_test_client, endpoint +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.videos.utils import encode_video_id_with_provider + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "video_resp_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + } + + encoded_video_id = encode_video_id_with_provider( + video_id="video_raw_123", + provider="vertex_ai", + model_id="veo-2.0-generate-001", + ) + payload = {"prompt": "test", "video": {"id": encoded_video_id}} + if endpoint.endswith("extensions"): + payload["seconds"] = "4" + + mock_router = MagicMock() + mock_router.resolve_model_name_from_model_id.return_value = "vertex-ai-sora-2" + + with patch("litellm.proxy.proxy_server.llm_router", mock_router): + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + endpoint, + headers={"Authorization": "Bearer sk-1234"}, + json=payload, + ) + + assert response.status_code == 200, response.text + assert captured_data["video_id"] == encoded_video_id + assert captured_data["custom_llm_provider"] == "vertex_ai" + assert captured_data["model"] == "vertex-ai-sora-2"