mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-25 01:02:15 +00:00
feat(fal_ai): add Seedance video generation via fal queue API
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry
parent
f08b787685
commit
5f54f87d98
6 changed files with 992 additions and 0 deletions
3
litellm/llms/fal_ai/videos/__init__.py
Normal file
3
litellm/llms/fal_ai/videos/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from litellm.llms.fal_ai.videos.transformation import FalAIVideoConfig
|
||||
|
||||
__all__ = ("FalAIVideoConfig",)
|
||||
512
litellm/llms/fal_ai/videos/transformation.py
Normal file
512
litellm/llms/fal_ai/videos/transformation.py
Normal file
|
|
@ -0,0 +1,512 @@
|
|||
import math
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from httpx._types import FileContent, RequestFiles
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
_get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # shared HTTP factory is private
|
||||
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # shared HTTP factory lacks typed params
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.videos.main import (
|
||||
CharacterObject,
|
||||
VideoCreateOptionalRequestParams,
|
||||
VideoObject,
|
||||
)
|
||||
from litellm.types.videos.utils import (
|
||||
decode_video_id_with_provider,
|
||||
encode_video_id_with_provider,
|
||||
)
|
||||
|
||||
|
||||
class FalAIVideoError(BaseLLMException):
|
||||
pass
|
||||
|
||||
|
||||
_ALLOWED_ASPECT_RATIOS: Final[frozenset[str]] = frozenset({"auto", "16:9", "9:16", "1:1", "4:3", "3:4", "21:9"})
|
||||
_ALLOWED_RESOLUTIONS: Final[frozenset[str]] = frozenset({"480p", "720p", "1080p", "4k"})
|
||||
_RESOLUTION_TIERS: Final[tuple[tuple[int, str], ...]] = (
|
||||
(480, "480p"),
|
||||
(720, "720p"),
|
||||
(1080, "1080p"),
|
||||
)
|
||||
_FAL_AI_PROVIDER: Final[str] = LlmProviders.FAL_AI.value
|
||||
|
||||
|
||||
def _queue_request_base_path(model: str) -> str:
|
||||
segments: Final[tuple[str, ...]] = tuple(model.split("/"))
|
||||
segment_count: Final[int] = 3 if segments and segments[0] in frozenset(("workflows", "comfy")) else 2
|
||||
return "/".join(segments[:segment_count])
|
||||
|
||||
|
||||
def _duration_value(value: object) -> str | None:
|
||||
if isinstance(value, str) and value == "auto":
|
||||
return value
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
return str(int(float(value)))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _resolution_for_height(height: int) -> str:
|
||||
return next((resolution for threshold, resolution in _RESOLUTION_TIERS if height <= threshold), "4k")
|
||||
|
||||
|
||||
def _size_params(size: object) -> Mapping[str, str]:
|
||||
if not isinstance(size, str):
|
||||
return MappingProxyType({})
|
||||
if size in _ALLOWED_RESOLUTIONS:
|
||||
return MappingProxyType({"resolution": size})
|
||||
if size.count("x") != 1:
|
||||
return MappingProxyType({})
|
||||
width_text, height_text = size.split("x")
|
||||
if not (width_text.isdigit() and height_text.isdigit()):
|
||||
return MappingProxyType({})
|
||||
width: Final[int] = int(width_text)
|
||||
height: Final[int] = int(height_text)
|
||||
if width <= 0 or height <= 0:
|
||||
return MappingProxyType({})
|
||||
reduced_gcd: Final[int] = math.gcd(width, height)
|
||||
aspect_ratio: Final[str] = f"{width // reduced_gcd}:{height // reduced_gcd}"
|
||||
resolution: Final[str] = _resolution_for_height(height)
|
||||
if aspect_ratio in _ALLOWED_ASPECT_RATIOS:
|
||||
return MappingProxyType({"resolution": resolution, "aspect_ratio": aspect_ratio})
|
||||
return MappingProxyType({"resolution": resolution})
|
||||
|
||||
|
||||
def _numeric_duration(value: object) -> float | None:
|
||||
duration: Final[str | None] = _duration_value(value)
|
||||
if duration is None or duration == "auto":
|
||||
return None
|
||||
return float(duration)
|
||||
|
||||
|
||||
def _response_data(raw_response: httpx.Response) -> Mapping[str, object]:
|
||||
return TypeAdapter(Mapping[str, object]).validate_python(raw_response.json())
|
||||
|
||||
|
||||
def _response_string(response_data: Mapping[str, object], key: str, default: str = "") -> str:
|
||||
value: Final[object] = response_data.get(key)
|
||||
return value if isinstance(value, str) else default
|
||||
|
||||
|
||||
class FalAIVideoConfig(BaseVideoConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: API contract requires a list
|
||||
return [ # mutable-ok: API contract requires a list
|
||||
"model",
|
||||
"prompt",
|
||||
"input_reference",
|
||||
"seconds",
|
||||
"size",
|
||||
"user",
|
||||
"extra_headers",
|
||||
]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
video_create_optional_params: VideoCreateOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict[str, object]: # mutable-ok: BaseVideoConfig requires a mutable mapping
|
||||
supported_params: Final[frozenset[str]] = frozenset(self.get_supported_openai_params(model))
|
||||
input_reference: Final[object] = video_create_optional_params.get("input_reference")
|
||||
input_reference_params: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({})
|
||||
if "input_reference" not in video_create_optional_params
|
||||
else (
|
||||
MappingProxyType({"image_url": input_reference})
|
||||
if isinstance(input_reference, str)
|
||||
else self._invalid_input_reference()
|
||||
)
|
||||
)
|
||||
duration_params: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({})
|
||||
if "seconds" not in video_create_optional_params
|
||||
else self._duration_params(video_create_optional_params["seconds"])
|
||||
)
|
||||
size_params: Final[Mapping[str, str]] = (
|
||||
self._size_params(video_create_optional_params["size"])
|
||||
if "size" in video_create_optional_params
|
||||
else MappingProxyType({})
|
||||
)
|
||||
user_params: Final[Mapping[str, str]] = (
|
||||
MappingProxyType({"end_user_id": user})
|
||||
if isinstance(user := video_create_optional_params.get("user"), str)
|
||||
else MappingProxyType({})
|
||||
)
|
||||
return dict( # mutable-ok: BaseVideoConfig requires a mutable mapping
|
||||
MappingProxyType(
|
||||
{
|
||||
**input_reference_params,
|
||||
**duration_params,
|
||||
**size_params,
|
||||
**user_params,
|
||||
**{ # mutable-ok: dynamic passthrough fields require a mapping
|
||||
key: value for key, value in video_create_optional_params.items() if key not in supported_params
|
||||
},
|
||||
}
|
||||
)
|
||||
) # mutable-ok: BaseVideoConfig requires a mutable mapping
|
||||
|
||||
@staticmethod
|
||||
def _invalid_input_reference() -> Mapping[str, str]:
|
||||
raise ValueError("fal.ai needs a public image URL for input_reference")
|
||||
|
||||
@staticmethod
|
||||
def _duration_params(seconds: object) -> Mapping[str, str]:
|
||||
duration: Final[str | None] = _duration_value(seconds)
|
||||
if duration is None:
|
||||
raise ValueError("fal.ai seconds must be a numeric value")
|
||||
return MappingProxyType({"duration": duration})
|
||||
|
||||
@staticmethod
|
||||
def _size_params(size: object) -> Mapping[str, str]:
|
||||
return _size_params(size)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
litellm_params: GenericLiteLLMParams | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
final_api_key: Final[str | None] = (
|
||||
api_key
|
||||
or (litellm_params.api_key if litellm_params is not None else None)
|
||||
or get_secret_str("FAL_AI_API_KEY")
|
||||
or get_secret_str("FAL_KEY")
|
||||
)
|
||||
if not final_api_key:
|
||||
raise ValueError("fal.ai API key is required")
|
||||
return dict( # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
MappingProxyType(
|
||||
{
|
||||
**headers,
|
||||
"Authorization": f"Key {final_api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
)
|
||||
) # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: dict[str, object], # mutable-ok: BaseVideoConfig requires mutable parameters
|
||||
) -> str:
|
||||
return (api_base or get_secret_str("FAL_AI_QUEUE_API_BASE") or "https://queue.fal.run").rstrip("/")
|
||||
|
||||
def transform_video_create_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str,
|
||||
api_base: str,
|
||||
video_create_optional_request_params: dict[ # mutable-ok: BaseVideoConfig requires mutable parameters
|
||||
str, object
|
||||
], # mutable-ok: BaseVideoConfig requires mutable parameters
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
) -> tuple[dict[str, object], RequestFiles, str]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
request_data: Final[dict[str, object]] = dict( # mutable-ok: HTTP JSON payload requires mutable data
|
||||
MappingProxyType(
|
||||
{
|
||||
"prompt": prompt,
|
||||
**{ # mutable-ok: dynamic request fields require a mapping
|
||||
key: value for key, value in video_create_optional_request_params.items() if key != "model"
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return request_data, [], f"{api_base.rstrip('/')}/{model}" # mutable-ok: HTTP files payload requires a list
|
||||
|
||||
def transform_video_create_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_data: Mapping[str, object] | None = None,
|
||||
) -> VideoObject:
|
||||
response_data: Final[Mapping[str, object]] = _response_data(raw_response)
|
||||
request_params: Final[Mapping[str, object]] = request_data or MappingProxyType({})
|
||||
request_id: Final[str] = _response_string(response_data, "request_id")
|
||||
provider: Final[str] = custom_llm_provider or _FAL_AI_PROVIDER
|
||||
duration: Final[float | None] = _numeric_duration(request_params.get("duration"))
|
||||
resolution: Final[object] = request_params.get("resolution")
|
||||
seconds: Final[str | None] = _duration_value(request_params["duration"]) if duration is not None else None
|
||||
size: Final[str | None] = resolution if isinstance(resolution, str) else None
|
||||
usage: Final[dict[str, object]] = dict( # mutable-ok: VideoObject requires a mutable usage mapping
|
||||
MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in (
|
||||
("duration_seconds", duration),
|
||||
("video_resolution", resolution if isinstance(resolution, str) else "720p"),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
)
|
||||
) # mutable-ok: VideoObject requires a mutable usage mapping
|
||||
video_object: Final[VideoObject] = VideoObject(
|
||||
id=encode_video_id_with_provider(request_id, provider, model),
|
||||
object="video",
|
||||
status="queued",
|
||||
created_at=int(time.time()),
|
||||
model=model,
|
||||
seconds=seconds,
|
||||
size=size,
|
||||
)
|
||||
video_object.usage = usage
|
||||
return video_object
|
||||
|
||||
def transform_video_status_retrieve_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
request_id, model_id = self._decode_video_id(video_id)
|
||||
encoded_request_id: Final[str] = encode_url_path_segment(request_id, field_name="video_id")
|
||||
return (
|
||||
f"{api_base.rstrip('/')}/{_queue_request_base_path(model_id)}/requests/{encoded_request_id}/status",
|
||||
{}, # mutable-ok: BaseVideoConfig requires a mutable mapping
|
||||
)
|
||||
|
||||
def transform_video_status_retrieve_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> VideoObject:
|
||||
response_data: Final[Mapping[str, object]] = _response_data(raw_response)
|
||||
raw_status: Final[str] = _response_string(response_data, "status", "IN_QUEUE")
|
||||
status: Final[str] = MappingProxyType(
|
||||
{
|
||||
"IN_QUEUE": "queued",
|
||||
"IN_PROGRESS": "in_progress",
|
||||
"COMPLETED": "completed",
|
||||
}
|
||||
).get(raw_status, "queued")
|
||||
error_value: Final[object] = response_data.get("error")
|
||||
error: Final[str | None] = error_value if isinstance(error_value, str) else None
|
||||
provider: Final[str] = custom_llm_provider or _FAL_AI_PROVIDER
|
||||
return VideoObject(
|
||||
id=encode_video_id_with_provider(_response_string(response_data, "request_id"), provider),
|
||||
object="video",
|
||||
status="failed" if error else status,
|
||||
created_at=0,
|
||||
error=(
|
||||
{"code": "fal_error", "message": error} if error else None # mutable-ok: VideoObject requires a dict
|
||||
), # mutable-ok: VideoObject requires a dict
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_video_id(video_id: str) -> tuple[str, str]:
|
||||
decoded: Final = decode_video_id_with_provider(video_id)
|
||||
request_id: Final[str] = decoded.get("video_id", video_id)
|
||||
model_id: Final[str | None] = decoded.get("model_id")
|
||||
if not model_id:
|
||||
raise ValueError("fal.ai video ids must be created through litellm with a model")
|
||||
return request_id, model_id
|
||||
|
||||
def transform_video_content_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
variant: str | None = None,
|
||||
) -> tuple[str, dict[str, str]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
request_id, model_id = self._decode_video_id(video_id)
|
||||
encoded_request_id: Final[str] = encode_url_path_segment(request_id, field_name="video_id")
|
||||
return (
|
||||
f"{api_base.rstrip('/')}/{_queue_request_base_path(model_id)}/requests/{encoded_request_id}",
|
||||
{}, # mutable-ok: BaseVideoConfig requires a mutable mapping
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_video_url(response_data: Mapping[str, object]) -> str:
|
||||
raw_video_data: Final[object] = response_data.get("video")
|
||||
video_data: Final[Mapping[str, object] | None] = (
|
||||
TypeAdapter(Mapping[str, object]).validate_python(raw_video_data)
|
||||
if isinstance(raw_video_data, Mapping)
|
||||
else None
|
||||
)
|
||||
if video_data is not None:
|
||||
video_url: Final[object] = video_data.get("url")
|
||||
if isinstance(video_url, str) and video_url:
|
||||
return video_url
|
||||
error_message: Final[str | None] = next(
|
||||
(value for key in ("error", "detail") if isinstance(value := response_data.get(key), str)),
|
||||
None,
|
||||
)
|
||||
if error_message:
|
||||
raise ValueError(f"fal.ai video result did not include a video URL: {error_message}")
|
||||
raise ValueError("fal.ai video result did not include a video URL")
|
||||
|
||||
def transform_video_content_response(self, raw_response: httpx.Response, logging_obj: object) -> bytes:
|
||||
video_url: Final[str] = self._extract_video_url(_response_data(raw_response))
|
||||
httpx_client: Final[HTTPHandler] = _get_httpx_client()
|
||||
video_response: Final[httpx.Response] = httpx_client.get( # pyright: ignore[reportUnknownMemberType] # HTTP handler stubs are untyped
|
||||
video_url
|
||||
)
|
||||
video_response.raise_for_status()
|
||||
return video_response.content
|
||||
|
||||
async def async_transform_video_content_response(self, raw_response: httpx.Response, logging_obj: object) -> bytes:
|
||||
video_url: Final[str] = self._extract_video_url(_response_data(raw_response))
|
||||
async_httpx_client: Final[AsyncHTTPHandler] = get_async_httpx_client(llm_provider=LlmProviders.FAL_AI)
|
||||
video_response: Final[httpx.Response] = await async_httpx_client.get( # pyright: ignore[reportUnknownMemberType] # HTTP handler stubs are untyped
|
||||
video_url
|
||||
)
|
||||
video_response.raise_for_status()
|
||||
return video_response.content
|
||||
|
||||
def transform_video_remix_request(
|
||||
self,
|
||||
video_id: str,
|
||||
prompt: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
raise NotImplementedError("video remix is not supported for fal.ai")
|
||||
|
||||
def transform_video_remix_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> VideoObject:
|
||||
raise NotImplementedError("video remix is not supported for fal.ai")
|
||||
|
||||
def transform_video_list_request(
|
||||
self,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
after: str | None = None,
|
||||
limit: int | None = None,
|
||||
order: str | None = None,
|
||||
extra_query: Mapping[str, object] | None = None,
|
||||
) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
raise NotImplementedError("video listing is not supported for fal.ai")
|
||||
|
||||
def transform_video_list_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
raise NotImplementedError("video listing is not supported for fal.ai")
|
||||
|
||||
def transform_video_delete_request(
|
||||
self,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
raise NotImplementedError("video delete is not supported for fal.ai")
|
||||
|
||||
def transform_video_delete_response(self, raw_response: httpx.Response, logging_obj: object) -> VideoObject:
|
||||
raise NotImplementedError("video delete is not supported for fal.ai")
|
||||
|
||||
def transform_video_create_character_request(
|
||||
self,
|
||||
name: str,
|
||||
video: object,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
) -> tuple[str, list[object]]: # mutable-ok: BaseVideoConfig requires mutable lists
|
||||
raise NotImplementedError("video character creation is not supported for fal.ai")
|
||||
|
||||
def transform_video_create_character_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
) -> CharacterObject:
|
||||
raise NotImplementedError("video character creation is not supported for fal.ai")
|
||||
|
||||
def transform_video_get_character_request(
|
||||
self,
|
||||
character_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
raise NotImplementedError("video character retrieval is not supported for fal.ai")
|
||||
|
||||
def transform_video_get_character_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
) -> CharacterObject:
|
||||
raise NotImplementedError("video character retrieval is not supported for fal.ai")
|
||||
|
||||
def transform_video_edit_request(
|
||||
self,
|
||||
prompt: str,
|
||||
video_id: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
video_file: FileContent | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
prefetched_source_data: Mapping[str, object] | None = None,
|
||||
) -> tuple[str, Mapping[str, object], RequestFiles | None]:
|
||||
raise NotImplementedError("video edit is not supported for fal.ai")
|
||||
|
||||
def transform_video_edit_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_data: Mapping[str, object] | None = None,
|
||||
) -> VideoObject:
|
||||
raise NotImplementedError("video edit is not supported for fal.ai")
|
||||
|
||||
def transform_video_extension_request(
|
||||
self,
|
||||
prompt: str,
|
||||
video_id: str,
|
||||
seconds: str,
|
||||
api_base: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict[str, str], # mutable-ok: BaseVideoConfig requires mutable headers
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig requires mutable mappings
|
||||
raise NotImplementedError("video extension is not supported for fal.ai")
|
||||
|
||||
def transform_video_extension_response(
|
||||
self,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: object,
|
||||
custom_llm_provider: str | None = None,
|
||||
) -> VideoObject:
|
||||
raise NotImplementedError("video extension is not supported for fal.ai")
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseLLMException requires mutable headers
|
||||
) -> BaseLLMException:
|
||||
return FalAIVideoError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
|
@ -22807,6 +22807,127 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.5/text-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.473,
|
||||
"output_cost_per_second_480p": 0.2205,
|
||||
"output_cost_per_second_720p": 0.473,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.5/text-to-video",
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.5/image-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.473,
|
||||
"output_cost_per_second_480p": 0.2205,
|
||||
"output_cost_per_second_720p": 0.473,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.5/image-to-video",
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.5/reference-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.473,
|
||||
"output_cost_per_second_480p": 0.2205,
|
||||
"output_cost_per_second_720p": 0.473,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.5/reference-to-video",
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.0/text-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.3034,
|
||||
"output_cost_per_second_480p": 0.1346,
|
||||
"output_cost_per_second_720p": 0.3034,
|
||||
"output_cost_per_second_1080p": 0.682,
|
||||
"output_cost_per_second_4k": 1.5552,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.0/text-to-video",
|
||||
"metadata": {
|
||||
"comment": "fal bills $0.014 per 1k tokens (480p/720p/1080p) and $0.008 per 1k tokens (4k) with tokens = h*w*seconds*24/1024; 480p and 4k rates derived from that formula at 854x480 and 3840x2160"
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.0/image-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.3034,
|
||||
"output_cost_per_second_480p": 0.1346,
|
||||
"output_cost_per_second_720p": 0.3034,
|
||||
"output_cost_per_second_1080p": 0.682,
|
||||
"output_cost_per_second_4k": 1.5552,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.0/image-to-video",
|
||||
"metadata": {
|
||||
"comment": "fal bills $0.014 per 1k tokens (480p/720p/1080p) and $0.008 per 1k tokens (4k) with tokens = h*w*seconds*24/1024; 480p and 4k rates derived from that formula at 854x480 and 3840x2160"
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.0/reference-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.3034,
|
||||
"output_cost_per_second_480p": 0.1346,
|
||||
"output_cost_per_second_720p": 0.3034,
|
||||
"output_cost_per_second_1080p": 0.682,
|
||||
"output_cost_per_second_4k": 1.5552,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.0/reference-to-video",
|
||||
"metadata": {
|
||||
"comment": "fal bills $0.014 per 1k tokens (480p/720p/1080p) and $0.008 per 1k tokens (4k) with tokens = h*w*seconds*24/1024; 480p and 4k rates derived from that formula at 854x480 and 3840x2160"
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/fal-ai/ideogram/v3": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "image_generation",
|
||||
|
|
|
|||
|
|
@ -9403,6 +9403,10 @@ class ProviderConfigManager:
|
|||
from litellm.llms.runwayml.videos.transformation import RunwayMLVideoConfig
|
||||
|
||||
return RunwayMLVideoConfig()
|
||||
elif LlmProviders.FAL_AI == provider:
|
||||
from litellm.llms.fal_ai.videos.transformation import FalAIVideoConfig
|
||||
|
||||
return FalAIVideoConfig()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.hosted_vllm.videos import get_hosted_vllm_video_config
|
||||
|
||||
|
|
|
|||
|
|
@ -22807,6 +22807,127 @@
|
|||
"/v1/images/generations"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.5/text-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.473,
|
||||
"output_cost_per_second_480p": 0.2205,
|
||||
"output_cost_per_second_720p": 0.473,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.5/text-to-video",
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.5/image-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.473,
|
||||
"output_cost_per_second_480p": 0.2205,
|
||||
"output_cost_per_second_720p": 0.473,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.5/image-to-video",
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.5/reference-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.473,
|
||||
"output_cost_per_second_480p": 0.2205,
|
||||
"output_cost_per_second_720p": 0.473,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.5/reference-to-video",
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.0/text-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.3034,
|
||||
"output_cost_per_second_480p": 0.1346,
|
||||
"output_cost_per_second_720p": 0.3034,
|
||||
"output_cost_per_second_1080p": 0.682,
|
||||
"output_cost_per_second_4k": 1.5552,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.0/text-to-video",
|
||||
"metadata": {
|
||||
"comment": "fal bills $0.014 per 1k tokens (480p/720p/1080p) and $0.008 per 1k tokens (4k) with tokens = h*w*seconds*24/1024; 480p and 4k rates derived from that formula at 854x480 and 3840x2160"
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.0/image-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.3034,
|
||||
"output_cost_per_second_480p": 0.1346,
|
||||
"output_cost_per_second_720p": 0.3034,
|
||||
"output_cost_per_second_1080p": 0.682,
|
||||
"output_cost_per_second_4k": 1.5552,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.0/image-to-video",
|
||||
"metadata": {
|
||||
"comment": "fal bills $0.014 per 1k tokens (480p/720p/1080p) and $0.008 per 1k tokens (4k) with tokens = h*w*seconds*24/1024; 480p and 4k rates derived from that formula at 854x480 and 3840x2160"
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/bytedance/seedance-2.0/reference-to-video": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "video_generation",
|
||||
"output_cost_per_second": 0.3034,
|
||||
"output_cost_per_second_480p": 0.1346,
|
||||
"output_cost_per_second_720p": 0.3034,
|
||||
"output_cost_per_second_1080p": 0.682,
|
||||
"output_cost_per_second_4k": 1.5552,
|
||||
"source": "https://fal.ai/models/bytedance/seedance-2.0/reference-to-video",
|
||||
"metadata": {
|
||||
"comment": "fal bills $0.014 per 1k tokens (480p/720p/1080p) and $0.008 per 1k tokens (4k) with tokens = h*w*seconds*24/1024; 480p and 4k rates derived from that formula at 854x480 and 3840x2160"
|
||||
},
|
||||
"supported_endpoints": [
|
||||
"/v1/videos"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
"image"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"video"
|
||||
]
|
||||
},
|
||||
"fal_ai/fal-ai/ideogram/v3": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"mode": "image_generation",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,231 @@
|
|||
from unittest.mock import Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
import litellm.llms.fal_ai.videos.transformation as fal_video_module
|
||||
from litellm.cost_calculator import default_video_cost_calculator
|
||||
from litellm.llms.fal_ai.videos.transformation import (
|
||||
FalAIVideoConfig,
|
||||
FalAIVideoError,
|
||||
_queue_request_base_path,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.videos.utils import decode_video_id_with_provider
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
MODEL = "bytedance/seedance-2.5/text-to-video"
|
||||
|
||||
|
||||
class TestFalAIVideoTransformation:
|
||||
def setup_method(self):
|
||||
self.config = FalAIVideoConfig()
|
||||
self.logging_obj = Mock()
|
||||
|
||||
def test_map_openai_params(self):
|
||||
mapped = self.config.map_openai_params(
|
||||
{
|
||||
"seconds": "5",
|
||||
"size": "1280x720",
|
||||
"input_reference": "https://example.com/image.png",
|
||||
"user": "user-123",
|
||||
"generate_audio": False,
|
||||
},
|
||||
MODEL,
|
||||
False,
|
||||
)
|
||||
|
||||
assert mapped == {
|
||||
"duration": "5",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
"image_url": "https://example.com/image.png",
|
||||
"end_user_id": "user-123",
|
||||
"generate_audio": False,
|
||||
}
|
||||
|
||||
assert self.config.map_openai_params({"size": "1080x1080"}, MODEL, False) == {
|
||||
"resolution": "1080p",
|
||||
"aspect_ratio": "1:1",
|
||||
}
|
||||
assert self.config.map_openai_params({"size": "720p"}, MODEL, False) == {"resolution": "720p"}
|
||||
|
||||
def test_map_openai_params_rejects_non_url_input_reference(self):
|
||||
with pytest.raises(ValueError, match="public image URL"):
|
||||
self.config.map_openai_params({"input_reference": b"image"}, MODEL, False)
|
||||
|
||||
def test_transform_video_create_request(self):
|
||||
body, files, url = self.config.transform_video_create_request(
|
||||
model=MODEL,
|
||||
prompt="A quiet ocean at sunrise",
|
||||
api_base="https://queue.fal.run",
|
||||
video_create_optional_request_params={
|
||||
"duration": "5",
|
||||
"resolution": "480p",
|
||||
"aspect_ratio": "16:9",
|
||||
"generate_audio": False,
|
||||
"model": MODEL,
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert url == f"https://queue.fal.run/{MODEL}"
|
||||
assert files == []
|
||||
assert body == {
|
||||
"prompt": "A quiet ocean at sunrise",
|
||||
"duration": "5",
|
||||
"resolution": "480p",
|
||||
"aspect_ratio": "16:9",
|
||||
"generate_audio": False,
|
||||
}
|
||||
assert "model" not in body
|
||||
|
||||
def test_transform_video_create_response_encodes_model_and_usage(self):
|
||||
response = Mock(spec=httpx.Response)
|
||||
response.json.return_value = {"request_id": "abc"}
|
||||
|
||||
video = self.config.transform_video_create_response(
|
||||
model=MODEL,
|
||||
raw_response=response,
|
||||
logging_obj=self.logging_obj,
|
||||
custom_llm_provider="fal_ai",
|
||||
request_data={"duration": "5", "resolution": "480p"},
|
||||
)
|
||||
|
||||
decoded = decode_video_id_with_provider(video.id)
|
||||
assert decoded["custom_llm_provider"] == "fal_ai"
|
||||
assert decoded["model_id"] == MODEL
|
||||
assert decoded["video_id"] == "abc"
|
||||
assert video.status == "queued"
|
||||
assert video.usage == {"duration_seconds": 5.0, "video_resolution": "480p"}
|
||||
|
||||
auto_video = self.config.transform_video_create_response(
|
||||
model=MODEL,
|
||||
raw_response=response,
|
||||
logging_obj=self.logging_obj,
|
||||
custom_llm_provider="fal_ai",
|
||||
request_data={"duration": "auto"},
|
||||
)
|
||||
assert auto_video.usage == {"video_resolution": "720p"}
|
||||
assert auto_video.seconds is None
|
||||
assert auto_video.size is None
|
||||
|
||||
def test_status_request_uses_queue_base_path(self):
|
||||
response = Mock(spec=httpx.Response)
|
||||
response.json.return_value = {"request_id": "abc"}
|
||||
video = self.config.transform_video_create_response(
|
||||
model=MODEL,
|
||||
raw_response=response,
|
||||
logging_obj=self.logging_obj,
|
||||
custom_llm_provider="fal_ai",
|
||||
request_data={},
|
||||
)
|
||||
|
||||
url, params = self.config.transform_video_status_retrieve_request(
|
||||
video_id=video.id,
|
||||
api_base="https://queue.fal.run",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
assert url == "https://queue.fal.run/bytedance/seedance-2.5/requests/abc/status"
|
||||
assert params == {}
|
||||
assert _queue_request_base_path("workflows/owner/app/x") == "workflows/owner/app"
|
||||
assert _queue_request_base_path("comfy/owner/app/x") == "comfy/owner/app"
|
||||
|
||||
def test_status_request_rejects_unencoded_video_id(self):
|
||||
with pytest.raises(ValueError, match="must be created through litellm"):
|
||||
self.config.transform_video_status_retrieve_request(
|
||||
video_id="abc",
|
||||
api_base="https://queue.fal.run",
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("response_data", "expected_status"),
|
||||
[
|
||||
({"request_id": "abc", "status": "IN_QUEUE"}, "queued"),
|
||||
({"request_id": "abc", "status": "IN_PROGRESS"}, "in_progress"),
|
||||
({"request_id": "abc", "status": "COMPLETED"}, "completed"),
|
||||
],
|
||||
)
|
||||
def test_status_response_mapping(self, response_data, expected_status):
|
||||
response = Mock(spec=httpx.Response)
|
||||
response.json.return_value = response_data
|
||||
|
||||
video = self.config.transform_video_status_retrieve_response(
|
||||
raw_response=response,
|
||||
logging_obj=self.logging_obj,
|
||||
custom_llm_provider="fal_ai",
|
||||
)
|
||||
|
||||
assert video.status == expected_status
|
||||
assert video.created_at == 0
|
||||
|
||||
def test_status_response_error(self):
|
||||
response = Mock(spec=httpx.Response)
|
||||
response.json.return_value = {
|
||||
"request_id": "abc",
|
||||
"status": "COMPLETED",
|
||||
"error": "generation failed",
|
||||
}
|
||||
|
||||
video = self.config.transform_video_status_retrieve_response(
|
||||
raw_response=response,
|
||||
logging_obj=self.logging_obj,
|
||||
custom_llm_provider="fal_ai",
|
||||
)
|
||||
|
||||
assert video.status == "failed"
|
||||
assert video.error == {"code": "fal_error", "message": "generation failed"}
|
||||
|
||||
def test_content_response_downloads_video_url(self, monkeypatch):
|
||||
content_response = httpx.Response(
|
||||
200,
|
||||
content=b"video-bytes",
|
||||
request=httpx.Request("GET", "https://cdn.example.com/video.mp4"),
|
||||
)
|
||||
|
||||
class FakeHTTPClient:
|
||||
def get(self, url):
|
||||
assert url == "https://cdn.example.com/video.mp4"
|
||||
return content_response
|
||||
|
||||
monkeypatch.setattr(fal_video_module, "_get_httpx_client", lambda: FakeHTTPClient())
|
||||
response = Mock(spec=httpx.Response)
|
||||
response.json.return_value = {"video": {"url": "https://cdn.example.com/video.mp4"}}
|
||||
|
||||
assert self.config.transform_video_content_response(response, self.logging_obj) == b"video-bytes"
|
||||
|
||||
def test_content_response_rejects_missing_video(self):
|
||||
response = Mock(spec=httpx.Response)
|
||||
response.json.return_value = {"error": "generation failed"}
|
||||
|
||||
with pytest.raises(ValueError, match="generation failed"):
|
||||
self.config.transform_video_content_response(response, self.logging_obj)
|
||||
|
||||
def test_provider_config_and_error_class(self):
|
||||
provider_config = ProviderConfigManager.get_provider_video_config(
|
||||
model=MODEL,
|
||||
provider=LlmProviders.FAL_AI,
|
||||
)
|
||||
assert isinstance(provider_config, FalAIVideoConfig)
|
||||
assert isinstance(self.config.get_error_class("bad key", 401, {}), FalAIVideoError)
|
||||
|
||||
def test_video_cost_uses_tiered_rows(self):
|
||||
rows = {
|
||||
model: row
|
||||
for model, row in litellm.model_cost.items()
|
||||
if row.get("litellm_provider") == "fal_ai" and row.get("mode") == "video_generation"
|
||||
}
|
||||
assert rows
|
||||
for model, row in rows.items():
|
||||
assert default_video_cost_calculator(model, 5, "fal_ai", video_resolution="480p") == (
|
||||
5 * row["output_cost_per_second_480p"]
|
||||
)
|
||||
assert default_video_cost_calculator(model, 5, "fal_ai", video_resolution="720p") == (
|
||||
5 * row["output_cost_per_second"]
|
||||
)
|
||||
Loading…
Add table
Reference in a new issue