Merge pull request #30782 from emerzon/litellm_veo_31_lite

feat(vertex-ai): add veo 3.1 lite model metadata

litellm/llms/vertex_ai/videos/transformation.py

@@ -8,12 +8,14 @@ Based on: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/model-refer
 import base64
 import time
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, TypedDict, cast
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, ClassVar, Final, TypedDict, cast
 
 import httpx
 from httpx._types import FileContent, RequestFiles
 from typing_extensions import ReadOnly
 
+import litellm
 from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS
 from litellm.images.utils import ImageEditRequestUtils
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
@@ -119,6 +121,23 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
     3. Extract video data (base64) from response
     """
 
+    _OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO: ClassVar[Mapping[str, str]] = MappingProxyType(
+        {
+            "1280x720": "16:9",
+            "1920x1080": "16:9",
+            "720x1280": "9:16",
+            "1080x1920": "9:16",
+        }
+    )
+    _OPENAI_VIDEO_SIZE_TO_RESOLUTION: ClassVar[Mapping[str, str]] = MappingProxyType(
+        {
+            "1280x720": "720p",
+            "1920x1080": "1080p",
+            "720x1280": "720p",
+            "1080x1920": "1080p",
+        }
+    )
+
     def __init__(self):
         BaseVideoConfig.__init__(self)
         VertexBase.__init__(self)
@@ -161,6 +180,9 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         - prompt → prompt (in instances)
         - input_reference → image (in instances)
         - size → aspectRatio (e.g., "1280x720" → "16:9")
+        - size → resolution for models with resolution-tier pricing when inferable
+          ("1280x720"/"720x1280" → "720p", "1920x1080"/"1080x1920" → "1080p");
+          skipped if ``resolution`` is already set
         - seconds → durationSeconds (defaults to 4 seconds if not provided)
         """
         mapped_params: Final[dict[str, object]] = {}
@@ -175,6 +197,9 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         if "parameters" in video_create_optional_params:
             mapped_params["parameters"] = video_create_optional_params["parameters"]
 
+        if "resolution" in video_create_optional_params:
+            mapped_params["resolution"] = video_create_optional_params["resolution"]
+
         # Map size to aspectRatio
         if "size" in video_create_optional_params:
             size: Final = video_create_optional_params["size"]
@@ -182,6 +207,15 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
                 aspect_ratio: Final = self._convert_size_to_aspect_ratio(size)
                 if aspect_ratio:
                     mapped_params["aspectRatio"] = aspect_ratio
+                nested_params: Final = video_create_optional_params.get("parameters")
+                has_resolution = "resolution" in mapped_params or (
+                    isinstance(nested_params, dict) and nested_params.get("resolution") is not None
+                )
+                supports_resolution = self._supports_resolution_inference(model)
+                if supports_resolution and not has_resolution:
+                    inferred_resolution = self._convert_size_to_resolution(size)
+                    if inferred_resolution is not None:
+                        mapped_params["resolution"] = inferred_resolution
 
         # Map seconds to durationSeconds, default to 4 seconds (matching OpenAI)
         if "seconds" in video_create_optional_params:
@@ -205,14 +239,16 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase):
         if not size:
             return None
 
-        aspect_ratio_map: Final = {
-            "1280x720": "16:9",
-            "1920x1080": "16:9",
-            "720x1280": "9:16",
-            "1080x1920": "9:16",
-        }
+        return self._OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO.get(size, "16:9")
 
-        return aspect_ratio_map.get(size, "16:9")
+    def _convert_size_to_resolution(self, size: str) -> str | None:
+        return self._OPENAI_VIDEO_SIZE_TO_RESOLUTION.get(size)
+
+    @staticmethod
+    def _supports_resolution_inference(model: str) -> bool:
+        model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}"
+        model_info: Final = litellm.model_cost.get(model_key)
+        return model_info is not None and model_info.get("output_cost_per_second_1080p") is not None
 
     def validate_environment(
         self,

litellm/model_prices_and_context_window_backup.json

@@ -43318,6 +43318,22 @@
             "video"
         ]
     },
+    "vertex_ai/veo-3.1-lite-generate-001": {
+        "litellm_provider": "vertex_ai-video-models",
+        "max_input_tokens": 1024,
+        "max_tokens": 1024,
+        "mode": "video_generation",
+        "output_cost_per_second": 0.05,
+        "output_cost_per_second_1080p": 0.08,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ]
+    },
     "voyage/rerank-2": {
         "input_cost_per_token": 5e-08,
         "litellm_provider": "voyage",

litellm/types/videos/main.py

@@ -2,7 +2,7 @@ from typing import Any, Literal
 
 from openai.types.audio.transcription_create_params import FileTypes
 from pydantic import BaseModel
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 
 class VideoObject(BaseModel):
@@ -76,6 +76,7 @@ class VideoCreateOptionalRequestParams(TypedDict, total=False):
     image: Any | None  # Image for image-to-video; dict with gcsUri/bytesBase64Encoded, or file-like object
     parameters: dict[str, Any] | None  # Provider-specific parameters block passed directly to the API
     model: str | None
+    resolution: ReadOnly[str | None]
     seconds: str | None
     size: str | None
     characters: list[dict[str, str]] | None

model_prices_and_context_window.json

@@ -43318,6 +43318,22 @@
             "video"
         ]
     },
+    "vertex_ai/veo-3.1-lite-generate-001": {
+        "litellm_provider": "vertex_ai-video-models",
+        "max_input_tokens": 1024,
+        "max_tokens": 1024,
+        "mode": "video_generation",
+        "output_cost_per_second": 0.05,
+        "output_cost_per_second_1080p": 0.08,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ]
+    },
     "voyage/rerank-2": {
         "input_cost_per_token": 5e-08,
         "litellm_provider": "voyage",

tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py

@@ -4,13 +4,17 @@ Tests for Vertex AI (Veo) video generation transformation.
 
 import base64
 import json
-import os
-from unittest.mock import MagicMock, Mock, patch
+from collections.abc import Mapping
+from pathlib import Path
+from typing import cast
+from unittest.mock import Mock, patch
 
 import httpx
 import pytest
 
 import litellm
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+from litellm.llms.openai.cost_calculation import video_generation_cost
 from litellm.llms.vertex_ai.videos.transformation import (
     VertexAIVideoConfig,
     _convert_image_to_vertex_format,
@@ -18,6 +22,21 @@ from litellm.llms.vertex_ai.videos.transformation import (
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.videos.main import VideoObject
 
+VEO_31_LITE_VERTEX_MODEL = "vertex_ai/veo-3.1-lite-generate-001"
+ROOT_MODEL_COST_PATH = (
+    Path(__file__).parents[5] / "model_prices_and_context_window.json"
+)
+BACKUP_MODEL_COST_PATH = (
+    Path(__file__).parents[5]
+    / "litellm"
+    / "model_prices_and_context_window_backup.json"
+)
+ModelCostMap = Mapping[str, Mapping[str, object]]
+
+
+def _load_model_cost_map(path: Path) -> ModelCostMap:
+    return cast(ModelCostMap, json.loads(path.read_text()))
+
 
 class TestVertexAIVideoConfig:
     """Test VertexAIVideoConfig transformation class."""
@@ -117,6 +136,56 @@ class TestVertexAIVideoConfig:
         # Should NOT include endpoint
         assert not url.endswith(":predictLongRunning")
 
+    def test_veo_31_lite_model_cost_entries_match_pricing(self):
+        for path in (ROOT_MODEL_COST_PATH, BACKUP_MODEL_COST_PATH):
+            model_cost = _load_model_cost_map(path)
+            info = model_cost.get(VEO_31_LITE_VERTEX_MODEL)
+
+            assert info is not None, f"{VEO_31_LITE_VERTEX_MODEL} missing from {path}"
+            assert info["litellm_provider"] == "vertex_ai-video-models"
+            assert info["mode"] == "video_generation"
+            assert info["max_input_tokens"] == 1024
+            assert info["output_cost_per_second"] == 0.05
+            assert info["output_cost_per_second_1080p"] == 0.08
+            assert info["supported_modalities"] == ["text", "image"]
+
+    def test_veo_31_lite_provider_routing_from_local_model_map(
+        self, monkeypatch: pytest.MonkeyPatch
+    ):
+        model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH)
+        vertex_video_models = {
+            model_name.removeprefix("vertex_ai/")
+            for model_name, info in model_cost.items()
+            if info.get("litellm_provider") == "vertex_ai-video-models"
+        }
+        monkeypatch.setattr(litellm, "vertex_ai_video_models", vertex_video_models)
+
+        model, custom_llm_provider, _, _ = get_llm_provider(
+            model="veo-3.1-lite-generate-001"
+        )
+
+        assert model == "veo-3.1-lite-generate-001"
+        assert custom_llm_provider == "vertex_ai"
+
+    def test_veo_31_lite_cost_uses_resolution_tiers(self):
+        model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH)
+        model_info = model_cost[VEO_31_LITE_VERTEX_MODEL]
+
+        assert video_generation_cost(
+            model=VEO_31_LITE_VERTEX_MODEL,
+            duration_seconds=10.0,
+            custom_llm_provider="vertex_ai",
+            model_info=dict(model_info),
+            video_resolution="720p",
+        ) == pytest.approx(0.5)
+        assert video_generation_cost(
+            model=VEO_31_LITE_VERTEX_MODEL,
+            duration_seconds=10.0,
+            custom_llm_provider="vertex_ai",
+            model_info=dict(model_info),
+            video_resolution="1080p",
+        ) == pytest.approx(0.8)
+
     def test_transform_video_create_request(self):
         """Test transformation of video creation request."""
         prompt = "A cat playing with a ball of yarn"
@@ -210,6 +279,95 @@ class TestVertexAIVideoConfig:
 
         assert mapped["durationSeconds"] == 8
         assert mapped["aspectRatio"] == "16:9"
+        assert "resolution" not in mapped
+
+    @pytest.mark.parametrize(
+        ("model", "size", "expected_resolution"),
+        (
+            (VEO_31_LITE_VERTEX_MODEL, "1280x720", "720p"),
+            (
+                VEO_31_LITE_VERTEX_MODEL.removeprefix("vertex_ai/"),
+                "1920x1080",
+                "1080p",
+            ),
+        ),
+    )
+    def test_map_openai_size_to_resolution_for_resolution_tier_model(
+        self,
+        model: str,
+        size: str,
+        expected_resolution: str,
+        monkeypatch: pytest.MonkeyPatch,
+    ):
+        model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH)
+        monkeypatch.setitem(
+            litellm.model_cost,
+            VEO_31_LITE_VERTEX_MODEL,
+            dict(model_cost[VEO_31_LITE_VERTEX_MODEL]),
+        )
+
+        mapped = self.config.map_openai_params(
+            video_create_optional_params={"size": size},
+            model=model,
+            drop_params=False,
+        )
+
+        assert mapped["aspectRatio"] == "16:9"
+        assert mapped["resolution"] == expected_resolution
+
+    def test_map_openai_size_does_not_infer_resolution_for_veo_2(self):
+        mapped = self.config.map_openai_params(
+            video_create_optional_params={"size": "1920x1080"},
+            model="vertex_ai/veo-2.0-generate-001",
+            drop_params=False,
+        )
+
+        assert mapped["aspectRatio"] == "16:9"
+        assert "resolution" not in mapped
+
+    def test_map_openai_size_does_not_infer_resolution_for_existing_veo_3(
+        self, monkeypatch: pytest.MonkeyPatch
+    ):
+        model = "veo-3.1-generate-001"
+        model_key = f"vertex_ai/{model}"
+        model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH)
+        monkeypatch.setitem(litellm.model_cost, model_key, dict(model_cost[model_key]))
+
+        mapped = self.config.map_openai_params(
+            video_create_optional_params={"size": "1920x1080"},
+            model=model,
+            drop_params=False,
+        )
+
+        assert mapped["aspectRatio"] == "16:9"
+        assert "resolution" not in mapped
+
+    def test_map_openai_size_does_not_override_provider_resolution(self):
+        mapped = self.config.map_openai_params(
+            video_create_optional_params={
+                "size": "1920x1080",
+                "parameters": {"resolution": "720p"},
+            },
+            model=VEO_31_LITE_VERTEX_MODEL,
+            drop_params=False,
+        )
+
+        assert mapped["aspectRatio"] == "16:9"
+        assert "resolution" not in mapped
+        assert mapped["parameters"] == {"resolution": "720p"}
+
+    def test_map_openai_size_does_not_override_direct_resolution(self):
+        mapped = self.config.map_openai_params(
+            video_create_optional_params={
+                "size": "1920x1080",
+                "resolution": "720p",
+            },
+            model=VEO_31_LITE_VERTEX_MODEL,
+            drop_params=False,
+        )
+
+        assert mapped["aspectRatio"] == "16:9"
+        assert mapped["resolution"] == "720p"
 
     def test_map_openai_params_default_duration(self):
         """Test that durationSeconds is omitted when not provided."""