feat(voyage): add multimodal embedding support for voyage-multimodal-3/3.5 (#24972)

Route voyage-multimodal-* models to the correct /v1/multimodalembeddings
endpoint and transform inputs to the Voyage multimodal format. Supports
text, image_url, image_base64, video_url, and video_base64 content types.

Fixes #24957

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

litellm/__init__.py

@@ -1604,6 +1604,9 @@ if TYPE_CHECKING:
     from .llms.voyage.embedding.transformation_contextual import (
         VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig,
     )
+    from .llms.voyage.embedding.transformation_multimodal import (
+        VoyageMultimodalEmbeddingConfig as VoyageMultimodalEmbeddingConfig,
+    )
     from .llms.infinity.embedding.transformation import (
         InfinityEmbeddingConfig as InfinityEmbeddingConfig,
     )

litellm/_lazy_imports_registry.py

@@ -220,6 +220,7 @@ LLM_CONFIG_NAMES = (
     "GenAIHubOrchestrationConfig",
     "VoyageEmbeddingConfig",
     "VoyageContextualEmbeddingConfig",
+    "VoyageMultimodalEmbeddingConfig",
     "InfinityEmbeddingConfig",
     "PerplexityEmbeddingConfig",
     "AzureAIStudioConfig",
@@ -884,6 +885,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
         ".llms.voyage.embedding.transformation_contextual",
         "VoyageContextualEmbeddingConfig",
     ),
+    "VoyageMultimodalEmbeddingConfig": (
+        ".llms.voyage.embedding.transformation_multimodal",
+        "VoyageMultimodalEmbeddingConfig",
+    ),
     "InfinityEmbeddingConfig": (
         ".llms.infinity.embedding.transformation",
         "InfinityEmbeddingConfig",

litellm/llms/voyage/embedding/transformation_multimodal.py

@@ -0,0 +1,225 @@
+"""
+Transformation logic for the Voyage multimodal embeddings API.
+
+Used for all multimodal embedding models in Voyage (voyage-multimodal-3, voyage-multimodal-3.5).
+
+Reference: https://docs.voyageai.com/reference/multimodal-embeddings-api
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
+from litellm.types.utils import EmbeddingResponse, Usage
+
+
+class VoyageMultimodalEmbeddingError(BaseLLMException):
+    def __init__(
+        self,
+        status_code: int,
+        message: str,
+        headers: Union[dict, httpx.Headers] = {},
+    ):
+        self.status_code = status_code
+        self.message = message
+        self.request = httpx.Request(
+            method="POST",
+            url="https://api.voyageai.com/v1/multimodalembeddings",
+        )
+        self.response = httpx.Response(status_code=status_code, request=self.request)
+        super().__init__(
+            status_code=status_code,
+            message=message,
+            headers=headers,
+        )
+
+
+class VoyageMultimodalEmbeddingConfig(BaseEmbeddingConfig):
+    """
+    Reference: https://docs.voyageai.com/reference/multimodal-embeddings-api
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        if api_base:
+            if not api_base.endswith("/multimodalembeddings"):
+                api_base = f"{api_base}/multimodalembeddings"
+            return api_base
+        return "https://api.voyageai.com/v1/multimodalembeddings"
+
+    def get_supported_openai_params(self, model: str) -> list:
+        return ["encoding_format", "dimensions", "input_type"]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        if "encoding_format" in non_default_params:
+            optional_params["output_encoding"] = non_default_params["encoding_format"]
+        if "dimensions" in non_default_params:
+            optional_params["output_dimension"] = non_default_params["dimensions"]
+        if "input_type" in non_default_params:
+            optional_params["input_type"] = non_default_params["input_type"]
+        return optional_params
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        if api_key is None:
+            api_key = (
+                get_secret_str("VOYAGE_API_KEY")
+                or get_secret_str("VOYAGE_AI_API_KEY")
+                or get_secret_str("VOYAGE_AI_TOKEN")
+            )
+        return {
+            "Authorization": f"Bearer {api_key}",
+        }
+
+    @staticmethod
+    def _convert_input_to_multimodal(
+        input: Any,
+    ) -> List[Dict[str, Any]]:
+        """Convert various input formats to Voyage multimodal input format.
+
+        Supports:
+        - str: single text -> [{"content": [{"type": "text", "text": "..."}]}]
+        - List[str]: multiple texts -> [{"content": [{"type": "text", "text": "..."}]}, ...]
+        - List[dict]: already structured content blocks (OpenAI-style or Voyage-native)
+        """
+        if isinstance(input, str):
+            return [{"content": [{"type": "text", "text": input}]}]
+
+        if isinstance(input, list) and len(input) > 0:
+            if isinstance(input[0], str):
+                return [{"content": [{"type": "text", "text": text}]} for text in input]
+
+            if isinstance(input[0], dict):
+                inputs = []
+                for item in input:
+                    if "content" in item:
+                        content = item["content"]
+                        converted = []
+                        for block in content:
+                            converted.append(_convert_content_block(block))
+                        inputs.append({"content": converted})
+                    else:
+                        inputs.append({"content": [_convert_content_block(item)]})
+                return inputs
+
+        return [{"content": [{"type": "text", "text": str(input)}]}]
+
+    def transform_embedding_request(
+        self,
+        model: str,
+        input: AllEmbeddingInputValues,
+        optional_params: dict,
+        headers: dict,
+    ) -> dict:
+        inputs = self._convert_input_to_multimodal(input)
+        return {
+            "inputs": inputs,
+            "model": model,
+            **optional_params,
+        }
+
+    def transform_embedding_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: EmbeddingResponse,
+        logging_obj: LiteLLMLoggingObj,
+        api_key: Optional[str] = None,
+        request_data: dict = {},
+        optional_params: dict = {},
+        litellm_params: dict = {},
+    ) -> EmbeddingResponse:
+        try:
+            raw_response_json = raw_response.json()
+        except Exception:
+            raise VoyageMultimodalEmbeddingError(
+                message=raw_response.text, status_code=raw_response.status_code
+            )
+
+        model_response.model = raw_response_json.get("model")
+        model_response.data = raw_response_json.get("data")
+        model_response.object = raw_response_json.get("object")
+
+        usage_data = raw_response_json.get("usage", {})
+        usage = Usage(
+            prompt_tokens=usage_data.get("total_tokens", 0),
+            total_tokens=usage_data.get("total_tokens", 0),
+        )
+        model_response.usage = usage
+        return model_response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: Union[dict, httpx.Headers],
+    ) -> BaseLLMException:
+        return VoyageMultimodalEmbeddingError(
+            message=error_message, status_code=status_code, headers=headers
+        )
+
+    @staticmethod
+    def is_multimodal_embeddings(model: str) -> bool:
+        return "multimodal" in model.lower()
+
+
+def _convert_content_block(block: dict) -> dict:
+    """Convert an OpenAI-style content block to Voyage multimodal format."""
+    block_type = block.get("type", "")
+
+    if block_type == "text":
+        return {"type": "text", "text": block.get("text", "")}
+
+    if block_type == "image_url":
+        url = block.get("image_url", "")
+        if isinstance(url, dict):
+            url = url.get("url", "")
+        if url.startswith("data:"):
+            return {"type": "image_base64", "image_base64": url}
+        return {"type": "image_url", "image_url": url}
+
+    if block_type == "image_base64":
+        return {
+            "type": "image_base64",
+            "image_base64": block.get("image_base64", ""),
+        }
+
+    if block_type == "video_url":
+        return {"type": "video_url", "video_url": block.get("video_url", "")}
+
+    if block_type == "video_base64":
+        return {
+            "type": "video_base64",
+            "video_base64": block.get("video_base64", ""),
+        }
+
+    return block

litellm/model_prices_and_context_window_backup.json

@@ -32263,7 +32263,17 @@
         "max_input_tokens": 32000,
         "max_tokens": 32000,
         "mode": "embedding",
-        "output_cost_per_token": 0.0
+        "output_cost_per_token": 0.0,
+        "supports_vision": true
+    },
+    "voyage/voyage-multimodal-3.5": {
+        "input_cost_per_token": 1.2e-07,
+        "litellm_provider": "voyage",
+        "max_input_tokens": 32000,
+        "max_tokens": 32000,
+        "mode": "embedding",
+        "output_cost_per_token": 0.0,
+        "supports_vision": true
     },
     "wandb/openai/gpt-oss-120b": {
         "max_tokens": 131072,

litellm/utils.py

@@ -3427,6 +3427,15 @@ def get_optional_params_embeddings(  # noqa: PLR0915
                     drop_params=drop_params if drop_params is not None else False,
                 )
             )
+        elif litellm.VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(model):
+            optional_params = (
+                litellm.VoyageMultimodalEmbeddingConfig().map_openai_params(
+                    non_default_params=non_default_params,
+                    optional_params={},
+                    model=model,
+                    drop_params=drop_params if drop_params is not None else False,
+                )
+            )
         else:
             optional_params = litellm.VoyageEmbeddingConfig().map_openai_params(
                 non_default_params=non_default_params,
@@ -8263,6 +8272,13 @@ class ProviderConfigManager:
             )
         ):
             return litellm.VoyageContextualEmbeddingConfig()
+        elif (
+            litellm.LlmProviders.VOYAGE == provider
+            and litellm.VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(
+                model
+            )
+        ):
+            return litellm.VoyageMultimodalEmbeddingConfig()
         elif litellm.LlmProviders.VOYAGE == provider:
             return litellm.VoyageEmbeddingConfig()
         elif litellm.LlmProviders.TRITON == provider:

tests/test_litellm/llms/voyage/embedding/test_voyage_multimodal_embedding.py

@@ -0,0 +1,418 @@
+"""
+Test cases for Voyage multimodal embedding configuration.
+
+Tests the VoyageMultimodalEmbeddingConfig class including model detection,
+URL generation, parameter mapping, input transformation, and response handling.
+"""
+
+import json
+import os
+import sys
+
+import httpx
+import pytest
+
+sys.path.insert(0, os.path.abspath("../../../../.."))
+
+from litellm.llms.voyage.embedding.transformation_multimodal import (
+    VoyageMultimodalEmbeddingConfig,
+    _convert_content_block,
+)
+from litellm.types.utils import EmbeddingResponse, Usage
+
+
+class TestIsMultimodalEmbeddings:
+    def test_multimodal_model_detected(self):
+        assert (
+            VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(
+                "voyage-multimodal-3"
+            )
+            is True
+        )
+
+    def test_multimodal_3_5_detected(self):
+        assert (
+            VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(
+                "voyage-multimodal-3.5"
+            )
+            is True
+        )
+
+    def test_case_insensitive(self):
+        assert (
+            VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(
+                "Voyage-Multimodal-3"
+            )
+            is True
+        )
+
+    def test_standard_model_not_detected(self):
+        assert (
+            VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings("voyage-3")
+            is False
+        )
+
+    def test_contextual_model_not_detected(self):
+        assert (
+            VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings("voyage-context-3")
+            is False
+        )
+
+
+class TestGetCompleteUrl:
+    def setup_method(self):
+        self.config = VoyageMultimodalEmbeddingConfig()
+
+    def test_default_url(self):
+        url = self.config.get_complete_url(
+            api_base=None,
+            api_key=None,
+            model="voyage-multimodal-3",
+            optional_params={},
+            litellm_params={},
+        )
+        assert url == "https://api.voyageai.com/v1/multimodalembeddings"
+
+    def test_custom_api_base(self):
+        url = self.config.get_complete_url(
+            api_base="https://custom.api.com/v1",
+            api_key=None,
+            model="voyage-multimodal-3",
+            optional_params={},
+            litellm_params={},
+        )
+        assert url == "https://custom.api.com/v1/multimodalembeddings"
+
+    def test_custom_api_base_already_has_path(self):
+        url = self.config.get_complete_url(
+            api_base="https://custom.api.com/v1/multimodalembeddings",
+            api_key=None,
+            model="voyage-multimodal-3",
+            optional_params={},
+            litellm_params={},
+        )
+        assert url == "https://custom.api.com/v1/multimodalembeddings"
+
+
+class TestGetSupportedOpenaiParams:
+    def test_supported_params(self):
+        config = VoyageMultimodalEmbeddingConfig()
+        params = config.get_supported_openai_params("voyage-multimodal-3")
+        assert "encoding_format" in params
+        assert "dimensions" in params
+        assert "input_type" in params
+
+
+class TestMapOpenaiParams:
+    def setup_method(self):
+        self.config = VoyageMultimodalEmbeddingConfig()
+
+    def test_encoding_format_mapped_to_output_encoding(self):
+        result = self.config.map_openai_params(
+            non_default_params={"encoding_format": "base64"},
+            optional_params={},
+            model="voyage-multimodal-3",
+            drop_params=False,
+        )
+        assert result == {"output_encoding": "base64"}
+
+    def test_dimensions_mapped_to_output_dimension(self):
+        result = self.config.map_openai_params(
+            non_default_params={"dimensions": 1024},
+            optional_params={},
+            model="voyage-multimodal-3",
+            drop_params=False,
+        )
+        assert result == {"output_dimension": 1024}
+
+    def test_input_type_passed_through(self):
+        result = self.config.map_openai_params(
+            non_default_params={"input_type": "query"},
+            optional_params={},
+            model="voyage-multimodal-3",
+            drop_params=False,
+        )
+        assert result == {"input_type": "query"}
+
+    def test_all_params_combined(self):
+        result = self.config.map_openai_params(
+            non_default_params={
+                "encoding_format": "base64",
+                "dimensions": 512,
+                "input_type": "document",
+            },
+            optional_params={},
+            model="voyage-multimodal-3",
+            drop_params=False,
+        )
+        assert result == {
+            "output_encoding": "base64",
+            "output_dimension": 512,
+            "input_type": "document",
+        }
+
+
+class TestConvertContentBlock:
+    def test_text_block(self):
+        result = _convert_content_block({"type": "text", "text": "hello"})
+        assert result == {"type": "text", "text": "hello"}
+
+    def test_image_url_block_string(self):
+        result = _convert_content_block(
+            {"type": "image_url", "image_url": "https://example.com/img.jpg"}
+        )
+        assert result == {
+            "type": "image_url",
+            "image_url": "https://example.com/img.jpg",
+        }
+
+    def test_image_url_block_openai_format(self):
+        result = _convert_content_block(
+            {"type": "image_url", "image_url": {"url": "https://example.com/img.jpg"}}
+        )
+        assert result == {
+            "type": "image_url",
+            "image_url": "https://example.com/img.jpg",
+        }
+
+    def test_image_url_block_base64_data_uri(self):
+        data_uri = "data:image/jpeg;base64,/9j/4AAQ..."
+        result = _convert_content_block({"type": "image_url", "image_url": data_uri})
+        assert result == {"type": "image_base64", "image_base64": data_uri}
+
+    def test_image_url_block_base64_data_uri_openai_format(self):
+        data_uri = "data:image/jpeg;base64,/9j/4AAQ..."
+        result = _convert_content_block(
+            {"type": "image_url", "image_url": {"url": data_uri}}
+        )
+        assert result == {"type": "image_base64", "image_base64": data_uri}
+
+    def test_image_base64_block(self):
+        result = _convert_content_block(
+            {"type": "image_base64", "image_base64": "data:image/png;base64,abc"}
+        )
+        assert result == {
+            "type": "image_base64",
+            "image_base64": "data:image/png;base64,abc",
+        }
+
+    def test_video_url_block(self):
+        result = _convert_content_block(
+            {"type": "video_url", "video_url": "https://example.com/video.mp4"}
+        )
+        assert result == {
+            "type": "video_url",
+            "video_url": "https://example.com/video.mp4",
+        }
+
+    def test_unknown_type_passed_through(self):
+        block = {"type": "unknown", "data": "something"}
+        result = _convert_content_block(block)
+        assert result == block
+
+
+class TestTransformEmbeddingRequest:
+    def setup_method(self):
+        self.config = VoyageMultimodalEmbeddingConfig()
+
+    def test_single_text_string(self):
+        result = self.config.transform_embedding_request(
+            model="voyage-multimodal-3",
+            input="Hello world",
+            optional_params={},
+            headers={},
+        )
+        assert result == {
+            "inputs": [{"content": [{"type": "text", "text": "Hello world"}]}],
+            "model": "voyage-multimodal-3",
+        }
+
+    def test_text_list(self):
+        result = self.config.transform_embedding_request(
+            model="voyage-multimodal-3",
+            input=["Hello", "World"],
+            optional_params={},
+            headers={},
+        )
+        assert result == {
+            "inputs": [
+                {"content": [{"type": "text", "text": "Hello"}]},
+                {"content": [{"type": "text", "text": "World"}]},
+            ],
+            "model": "voyage-multimodal-3",
+        }
+
+    def test_multimodal_input_with_content_blocks(self):
+        input_data = [
+            {
+                "content": [
+                    {"type": "text", "text": "A photo of a cat"},
+                    {"type": "image_url", "image_url": "https://example.com/cat.jpg"},
+                ]
+            }
+        ]
+        result = self.config.transform_embedding_request(
+            model="voyage-multimodal-3",
+            input=input_data,
+            optional_params={},
+            headers={},
+        )
+        assert result == {
+            "inputs": [
+                {
+                    "content": [
+                        {"type": "text", "text": "A photo of a cat"},
+                        {
+                            "type": "image_url",
+                            "image_url": "https://example.com/cat.jpg",
+                        },
+                    ]
+                }
+            ],
+            "model": "voyage-multimodal-3",
+        }
+
+    def test_openai_style_image_url_converted(self):
+        input_data = [
+            {
+                "content": [
+                    {"type": "text", "text": "Describe this"},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": "https://example.com/img.png"},
+                    },
+                ]
+            }
+        ]
+        result = self.config.transform_embedding_request(
+            model="voyage-multimodal-3",
+            input=input_data,
+            optional_params={},
+            headers={},
+        )
+        assert result["inputs"][0]["content"][1] == {
+            "type": "image_url",
+            "image_url": "https://example.com/img.png",
+        }
+
+    def test_optional_params_included(self):
+        result = self.config.transform_embedding_request(
+            model="voyage-multimodal-3",
+            input="test",
+            optional_params={"input_type": "query", "output_dimension": 512},
+            headers={},
+        )
+        assert result["input_type"] == "query"
+        assert result["output_dimension"] == 512
+
+
+class TestTransformEmbeddingResponse:
+    def setup_method(self):
+        self.config = VoyageMultimodalEmbeddingConfig()
+
+    def test_standard_response(self):
+        voyage_response = {
+            "data": [
+                {"object": "embedding", "embedding": [0.1, 0.2, 0.3], "index": 0},
+                {"object": "embedding", "embedding": [0.4, 0.5, 0.6], "index": 1},
+            ],
+            "object": "list",
+            "model": "voyage-multimodal-3",
+            "usage": {
+                "text_tokens": 5,
+                "image_pixels": 2000000,
+                "total_tokens": 1005,
+            },
+        }
+        mock_response = httpx.Response(
+            status_code=200,
+            content=json.dumps(voyage_response).encode("utf-8"),
+            headers={"content-type": "application/json"},
+        )
+
+        model_response = EmbeddingResponse()
+        result = self.config.transform_embedding_response(
+            model="voyage-multimodal-3",
+            raw_response=mock_response,
+            model_response=model_response,
+            logging_obj=None,
+            request_data={"inputs": []},
+        )
+
+        assert result.object == "list"
+        assert result.model == "voyage-multimodal-3"
+        assert len(result.data) == 2
+        assert result.data[0]["embedding"] == [0.1, 0.2, 0.3]
+        assert result.data[1]["embedding"] == [0.4, 0.5, 0.6]
+        assert isinstance(result.usage, Usage)
+        assert result.usage.total_tokens == 1005
+
+    def test_error_response(self):
+        mock_response = httpx.Response(
+            status_code=400,
+            content=b"not json",
+            headers={"content-type": "text/plain"},
+        )
+        model_response = EmbeddingResponse()
+        from litellm.llms.voyage.embedding.transformation_multimodal import (
+            VoyageMultimodalEmbeddingError,
+        )
+
+        with pytest.raises(VoyageMultimodalEmbeddingError):
+            self.config.transform_embedding_response(
+                model="voyage-multimodal-3",
+                raw_response=mock_response,
+                model_response=model_response,
+                logging_obj=None,
+                request_data={},
+            )
+
+
+class TestProviderConfigRouting:
+    def test_multimodal_model_returns_multimodal_config(self):
+        from litellm.utils import ProviderConfigManager
+        import litellm
+
+        config = ProviderConfigManager.get_provider_embedding_config(
+            model="voyage-multimodal-3",
+            provider=litellm.LlmProviders.VOYAGE,
+        )
+        assert isinstance(config, VoyageMultimodalEmbeddingConfig)
+
+    def test_multimodal_3_5_returns_multimodal_config(self):
+        from litellm.utils import ProviderConfigManager
+        import litellm
+
+        config = ProviderConfigManager.get_provider_embedding_config(
+            model="voyage-multimodal-3.5",
+            provider=litellm.LlmProviders.VOYAGE,
+        )
+        assert isinstance(config, VoyageMultimodalEmbeddingConfig)
+
+    def test_standard_model_returns_standard_config(self):
+        from litellm.utils import ProviderConfigManager
+        from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig
+        import litellm
+
+        config = ProviderConfigManager.get_provider_embedding_config(
+            model="voyage-3",
+            provider=litellm.LlmProviders.VOYAGE,
+        )
+        assert isinstance(config, VoyageEmbeddingConfig)
+
+    def test_contextual_model_returns_contextual_config(self):
+        from litellm.utils import ProviderConfigManager
+        from litellm.llms.voyage.embedding.transformation_contextual import (
+            VoyageContextualEmbeddingConfig,
+        )
+        import litellm
+
+        config = ProviderConfigManager.get_provider_embedding_config(
+            model="voyage-context-3",
+            provider=litellm.LlmProviders.VOYAGE,
+        )
+        assert isinstance(config, VoyageContextualEmbeddingConfig)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])