mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
test(integration): cover fal Seedance video queue wire contract
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
85a6a8e206
commit
9a63e06c63
7 changed files with 76 additions and 109 deletions
|
|
@ -80,7 +80,6 @@
|
|||
- {id: llm.images_generations.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/image_generation/image_generation_handler.py", rationale: "Vertex Imagen"}
|
||||
- {id: llm.images_generations.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "bedrock/image_generation/image_handler.py", rationale: "Bedrock Titan Image"}
|
||||
- {id: llm.images_generations.black_forest_labs.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "black_forest_labs/image_generation/handler.py", rationale: "BFL Flux via OpenAI-compat"}
|
||||
- {id: llm.videos.fal_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: videos, route: fal_ai, capability: basic, streaming: nonstream, assertions: [works], source: "test_video_generation_e2e.py", rationale: "fal queue video create, poll, content download"}
|
||||
- {id: llm.audio_speech.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_audio_speech_e2e.py:22", rationale: "OpenAI TTS binary audio"}
|
||||
- {id: llm.audio_speech.openai.basic.stream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: openai, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:9043", rationale: "TTS streaming chunk generator"}
|
||||
- {id: llm.audio_speech.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.6 / LIT-4778", rationale: "TTS missing input/model, invalid voice, empty input rejected"}
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ LlmEndpoint = Literal[
|
|||
"vector_stores",
|
||||
"ocr",
|
||||
"bedrock_native",
|
||||
"videos",
|
||||
]
|
||||
|
||||
LlmRoute = Literal[
|
||||
|
|
@ -54,7 +53,6 @@ LlmRoute = Literal[
|
|||
"bedrock_converse",
|
||||
"bedrock_invoke",
|
||||
"cohere",
|
||||
"fal_ai",
|
||||
"gemini",
|
||||
"hosted_vllm",
|
||||
"openai",
|
||||
|
|
|
|||
|
|
@ -48,7 +48,6 @@ most likely to silently break and the one a mock can't prove works.
|
|||
|----------|---------------|-----------|------------|-------------|--------|
|
||||
| Chat | live (spend suite) | live (spend suite) | gap | live | partial |
|
||||
| Embeddings | live (spend suite) | n/a | n/a | live | covered |
|
||||
| Video | live (fal.ai Seedance) | n/a | n/a | - | partial |
|
||||
| Responses / image / audio / rerank / realtime | - | - | - | - | gap |
|
||||
|
||||
## This suite's files
|
||||
|
|
@ -62,7 +61,6 @@ most likely to silently break and the one a mock can't prove works.
|
|||
| `test_anthropic_passthrough_streaming_logs_cost` | anthropic native, stream, cost |
|
||||
| `test_anthropic_passthrough_tool_call_logs_cost` | anthropic native, tool call, cost |
|
||||
| `test_vertex_passthrough_via_managed_model_logs_cost` | vertex_ai native, non-stream, cost |
|
||||
| `test_fal_seedance_video_completes_and_downloads` | fal.ai Seedance video create, poll, and content download |
|
||||
|
||||
Vertex keeps the credential on the proxy like gemini/anthropic, but the deployment is
|
||||
added at runtime instead of declared in the gateway config: the test POSTs `/model/new`
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from dataclasses import dataclass
|
|||
from typing import Literal
|
||||
|
||||
from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS
|
||||
from e2e_http import BinaryStream, NoBody, Result, StreamingResponse
|
||||
from e2e_http import BinaryStream, Result, StreamingResponse
|
||||
from models import CacheControl, ChatMessage, LiteLLMParamsBody, RichMessage, TextBlock
|
||||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -26,8 +26,6 @@ __all__ = [
|
|||
"TextBlock",
|
||||
"TranscriptionForm",
|
||||
"TranscriptionResult",
|
||||
"VideoObject",
|
||||
"VideoRequest",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -129,13 +127,6 @@ class ImageRequest(BaseModel):
|
|||
size: str = "1024x1024"
|
||||
|
||||
|
||||
class VideoRequest(BaseModel):
|
||||
model: str
|
||||
prompt: str
|
||||
seconds: str = "4"
|
||||
size: str = "1280x720"
|
||||
|
||||
|
||||
class ImageEditForm(BaseModel):
|
||||
model: str
|
||||
prompt: str
|
||||
|
|
@ -276,12 +267,6 @@ class ImagesResult(BaseModel):
|
|||
data: list[ImageItem] = []
|
||||
|
||||
|
||||
class VideoObject(BaseModel):
|
||||
id: str
|
||||
status: str
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class TranscriptionResult(BaseModel):
|
||||
text: str = ""
|
||||
|
||||
|
|
@ -455,25 +440,6 @@ class EndpointsClient:
|
|||
"/v1/images/generations", key, ImageRequest(model=model, prompt=prompt)
|
||||
)
|
||||
|
||||
def videos(self, key: str, model: str, prompt: str) -> StreamingResponse:
|
||||
return self._send(
|
||||
"/v1/videos", key, VideoRequest(model=model, prompt=prompt)
|
||||
)
|
||||
|
||||
def video_status(self, key: str, video_id: str) -> Result[VideoObject]:
|
||||
return self.proxy.transport.get(
|
||||
f"/v1/videos/{video_id}",
|
||||
headers=self.proxy.transport.bearer(key),
|
||||
params=NoBody(),
|
||||
response_type=VideoObject,
|
||||
)
|
||||
|
||||
def video_content(self, key: str, video_id: str) -> StreamingResponse:
|
||||
return self.proxy.transport.download(
|
||||
f"/v1/videos/{video_id}/content",
|
||||
headers=self.proxy.transport.bearer(key),
|
||||
)
|
||||
|
||||
def image_edit(
|
||||
self, key: str, model: str, prompt: str, image: bytes, *, filename: str = "image.png"
|
||||
) -> Result[ImagesResult]:
|
||||
|
|
|
|||
|
|
@ -1,69 +0,0 @@
|
|||
"""Live e2e: POST /v1/videos creates a video and serves its content.
|
||||
|
||||
Registers a fal.ai Seedance deployment at runtime, polls the queued video until it
|
||||
completes, and asserts the generated content is returned as binary data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import require_successful_call, unwrap
|
||||
from endpoints_client import EndpointsClient, VideoObject
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
_POLL_INTERVAL_SECONDS: Final[float] = 5.0
|
||||
_POLL_TIMEOUT_SECONDS: Final[float] = 600.0
|
||||
|
||||
|
||||
def _wait_for_completion(
|
||||
endpoints_client: EndpointsClient, key: str, created: VideoObject
|
||||
) -> VideoObject:
|
||||
deadline = time.monotonic() + _POLL_TIMEOUT_SECONDS
|
||||
while time.monotonic() < deadline:
|
||||
status = unwrap(endpoints_client.video_status(key, created.id))
|
||||
assert status.id == created.id
|
||||
if status.status == "completed":
|
||||
return status
|
||||
if status.status == "failed":
|
||||
pytest.fail(f"fal.ai video generation failed: {status}")
|
||||
time.sleep(_POLL_INTERVAL_SECONDS)
|
||||
pytest.fail(f"fal.ai video {created.id!r} did not complete within {_POLL_TIMEOUT_SECONDS}s")
|
||||
|
||||
|
||||
class TestVideoGeneration:
|
||||
@pytest.mark.covers("llm.videos.fal_ai.basic.nonstream.works")
|
||||
def test_fal_seedance_video_completes_and_downloads(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-fal-video-{unique_marker()}"
|
||||
model_id = endpoints_client.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(
|
||||
model="fal_ai/bytedance/seedance-2.5/text-to-video",
|
||||
api_key="os.environ/FAL_AI_API_KEY",
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
result = endpoints_client.videos(
|
||||
key, model, "a red fox running through snow at dawn"
|
||||
)
|
||||
require_successful_call(result)
|
||||
created = VideoObject.model_validate_json(result.body)
|
||||
assert created.id
|
||||
assert created.model
|
||||
|
||||
_wait_for_completion(endpoints_client, key, created)
|
||||
|
||||
content = endpoints_client.video_content(key, created.id)
|
||||
require_successful_call(content)
|
||||
assert len(content.body) > 0
|
||||
assert not (content.content_type or "").startswith("application/json")
|
||||
|
|
@ -160,6 +160,9 @@
|
|||
"other.provider_wire.anthropic.tool_history_system_cache_and_internal_fields",
|
||||
"quota_management.spend_tracking.cache_tokens.disjoint_classes_use_explicit_rates"
|
||||
],
|
||||
"tests/integration/providers/test_fal_ai_video_wire.py::test_fal_video_create_status_and_content_follow_queue_wire_contract": [
|
||||
"other.provider_wire.fal_ai.video_queue_create_status_and_content_download"
|
||||
],
|
||||
"tests/integration/mcp/test_mcp_lifecycle.py::test_saved_headers_reach_real_mcp_tool_and_survive_unrelated_edit": [
|
||||
"mcp.call_tool.saved_headers.reach_actual_transport"
|
||||
],
|
||||
|
|
|
|||
72
tests/integration/providers/test_fal_ai_video_wire.py
Normal file
72
tests/integration/providers/test_fal_ai_video_wire.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
_MODEL: Final = "bytedance/seedance-2.5/text-to-video"
|
||||
_MP4: Final = b"\x00\x00\x00\x18ftypmp42" + uuid.uuid4().bytes * 4
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.fal_ai.video_queue_create_status_and_content_download")
|
||||
def test_fal_video_create_status_and_content_follow_queue_wire_contract(gateway: Gateway) -> None:
|
||||
request_id: Final = "fal-req-" + uuid.uuid4().hex
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.target == f"/files/{request_id}.mp4":
|
||||
assert request.method == "GET"
|
||||
return Reply(body=_MP4, content_type="video/mp4")
|
||||
assert request.headers["authorization"] == "Key synthetic-fal-key"
|
||||
if request.method == "POST":
|
||||
assert request.target == f"/{_MODEL}"
|
||||
assert json.loads(request.body) == {
|
||||
"prompt": "a cat playing volleyball on a beach",
|
||||
"duration": "4",
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
}
|
||||
return Reply(
|
||||
body=json.dumps({"status": "IN_QUEUE", "request_id": request_id, "queue_position": 0}).encode()
|
||||
)
|
||||
assert request.method == "GET"
|
||||
if request.target == f"/bytedance/seedance-2.5/requests/{request_id}/status":
|
||||
return Reply(body=json.dumps({"status": "COMPLETED", "request_id": request_id}).encode())
|
||||
assert request.target == f"/bytedance/seedance-2.5/requests/{request_id}"
|
||||
return Reply(body=json.dumps({"video": {"url": f"{wire_url}/files/{request_id}.mp4"}}).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
wire_url: Final = wire.url
|
||||
model: Final = scenario.model(
|
||||
model=f"fal_ai/{_MODEL}",
|
||||
api_base=wire.url,
|
||||
api_key="synthetic-fal-key",
|
||||
)
|
||||
created: Final = gateway.post(
|
||||
"/v1/videos",
|
||||
{
|
||||
"model": model,
|
||||
"prompt": "a cat playing volleyball on a beach",
|
||||
"seconds": "4",
|
||||
"size": "1280x720",
|
||||
},
|
||||
)
|
||||
assert created["status"] == "queued"
|
||||
video_id: Final = created["id"]
|
||||
assert isinstance(video_id, str) and video_id
|
||||
status: Final = gateway.get(f"/v1/videos/{video_id}")
|
||||
assert status["status"] == "completed"
|
||||
status_id_matches_created_id: Final = status["id"] == video_id
|
||||
sys.stdout.write(f"status_id_matches_created_id={status_id_matches_created_id}\n")
|
||||
content: Final = gateway.request("GET", f"/v1/videos/{video_id}/content")
|
||||
assert content.status_code == 200, content.text
|
||||
assert content.headers["content-type"].startswith("video/mp4")
|
||||
assert content.content == _MP4
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", f"/{_MODEL}"),
|
||||
("GET", f"/bytedance/seedance-2.5/requests/{request_id}/status"),
|
||||
("GET", f"/bytedance/seedance-2.5/requests/{request_id}"),
|
||||
("GET", f"/files/{request_id}.mp4"),
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue