diff --git a/litellm/__init__.py b/litellm/__init__.py index d4418c661a3..45859b90dfe 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1604,6 +1604,9 @@ if TYPE_CHECKING: from .llms.voyage.embedding.transformation_contextual import ( VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig, ) + from .llms.voyage.embedding.transformation_multimodal import ( + VoyageMultimodalEmbeddingConfig as VoyageMultimodalEmbeddingConfig, + ) from .llms.infinity.embedding.transformation import ( InfinityEmbeddingConfig as InfinityEmbeddingConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 9164a3c8ae4..c6ffe0dac3f 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -220,6 +220,7 @@ LLM_CONFIG_NAMES = ( "GenAIHubOrchestrationConfig", "VoyageEmbeddingConfig", "VoyageContextualEmbeddingConfig", + "VoyageMultimodalEmbeddingConfig", "InfinityEmbeddingConfig", "PerplexityEmbeddingConfig", "AzureAIStudioConfig", @@ -884,6 +885,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.voyage.embedding.transformation_contextual", "VoyageContextualEmbeddingConfig", ), + "VoyageMultimodalEmbeddingConfig": ( + ".llms.voyage.embedding.transformation_multimodal", + "VoyageMultimodalEmbeddingConfig", + ), "InfinityEmbeddingConfig": ( ".llms.infinity.embedding.transformation", "InfinityEmbeddingConfig", diff --git a/litellm/llms/voyage/embedding/transformation_multimodal.py b/litellm/llms/voyage/embedding/transformation_multimodal.py new file mode 100644 index 00000000000..c57d2030e8b --- /dev/null +++ b/litellm/llms/voyage/embedding/transformation_multimodal.py @@ -0,0 +1,225 @@ +""" +Transformation logic for the Voyage multimodal embeddings API. + +Used for all multimodal embedding models in Voyage (voyage-multimodal-3, voyage-multimodal-3.5). + +Reference: https://docs.voyageai.com/reference/multimodal-embeddings-api +""" + +from typing import Any, Dict, List, Optional, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues +from litellm.types.utils import EmbeddingResponse, Usage + + +class VoyageMultimodalEmbeddingError(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: Union[dict, httpx.Headers] = {}, + ): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", + url="https://api.voyageai.com/v1/multimodalembeddings", + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + status_code=status_code, + message=message, + headers=headers, + ) + + +class VoyageMultimodalEmbeddingConfig(BaseEmbeddingConfig): + """ + Reference: https://docs.voyageai.com/reference/multimodal-embeddings-api + """ + + def __init__(self) -> None: + pass + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base: + if not api_base.endswith("/multimodalembeddings"): + api_base = f"{api_base}/multimodalembeddings" + return api_base + return "https://api.voyageai.com/v1/multimodalembeddings" + + def get_supported_openai_params(self, model: str) -> list: + return ["encoding_format", "dimensions", "input_type"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + if "encoding_format" in non_default_params: + optional_params["output_encoding"] = non_default_params["encoding_format"] + if "dimensions" in non_default_params: + optional_params["output_dimension"] = non_default_params["dimensions"] + if "input_type" in non_default_params: + optional_params["input_type"] = non_default_params["input_type"] + return optional_params + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + if api_key is None: + api_key = ( + get_secret_str("VOYAGE_API_KEY") + or get_secret_str("VOYAGE_AI_API_KEY") + or get_secret_str("VOYAGE_AI_TOKEN") + ) + return { + "Authorization": f"Bearer {api_key}", + } + + @staticmethod + def _convert_input_to_multimodal( + input: Any, + ) -> List[Dict[str, Any]]: + """Convert various input formats to Voyage multimodal input format. + + Supports: + - str: single text -> [{"content": [{"type": "text", "text": "..."}]}] + - List[str]: multiple texts -> [{"content": [{"type": "text", "text": "..."}]}, ...] + - List[dict]: already structured content blocks (OpenAI-style or Voyage-native) + """ + if isinstance(input, str): + return [{"content": [{"type": "text", "text": input}]}] + + if isinstance(input, list) and len(input) > 0: + if isinstance(input[0], str): + return [{"content": [{"type": "text", "text": text}]} for text in input] + + if isinstance(input[0], dict): + inputs = [] + for item in input: + if "content" in item: + content = item["content"] + converted = [] + for block in content: + converted.append(_convert_content_block(block)) + inputs.append({"content": converted}) + else: + inputs.append({"content": [_convert_content_block(item)]}) + return inputs + + return [{"content": [{"type": "text", "text": str(input)}]}] + + def transform_embedding_request( + self, + model: str, + input: AllEmbeddingInputValues, + optional_params: dict, + headers: dict, + ) -> dict: + inputs = self._convert_input_to_multimodal(input) + return { + "inputs": inputs, + "model": model, + **optional_params, + } + + def transform_embedding_response( + self, + model: str, + raw_response: httpx.Response, + model_response: EmbeddingResponse, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str] = None, + request_data: dict = {}, + optional_params: dict = {}, + litellm_params: dict = {}, + ) -> EmbeddingResponse: + try: + raw_response_json = raw_response.json() + except Exception: + raise VoyageMultimodalEmbeddingError( + message=raw_response.text, status_code=raw_response.status_code + ) + + model_response.model = raw_response_json.get("model") + model_response.data = raw_response_json.get("data") + model_response.object = raw_response_json.get("object") + + usage_data = raw_response_json.get("usage", {}) + usage = Usage( + prompt_tokens=usage_data.get("total_tokens", 0), + total_tokens=usage_data.get("total_tokens", 0), + ) + model_response.usage = usage + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + ) -> BaseLLMException: + return VoyageMultimodalEmbeddingError( + message=error_message, status_code=status_code, headers=headers + ) + + @staticmethod + def is_multimodal_embeddings(model: str) -> bool: + return "multimodal" in model.lower() + + +def _convert_content_block(block: dict) -> dict: + """Convert an OpenAI-style content block to Voyage multimodal format.""" + block_type = block.get("type", "") + + if block_type == "text": + return {"type": "text", "text": block.get("text", "")} + + if block_type == "image_url": + url = block.get("image_url", "") + if isinstance(url, dict): + url = url.get("url", "") + if url.startswith("data:"): + return {"type": "image_base64", "image_base64": url} + return {"type": "image_url", "image_url": url} + + if block_type == "image_base64": + return { + "type": "image_base64", + "image_base64": block.get("image_base64", ""), + } + + if block_type == "video_url": + return {"type": "video_url", "video_url": block.get("video_url", "")} + + if block_type == "video_base64": + return { + "type": "video_base64", + "video_base64": block.get("video_base64", ""), + } + + return block diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ebaecf049c..5f1b6dfec5f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -32263,7 +32263,17 @@ "max_input_tokens": 32000, "max_tokens": 32000, "mode": "embedding", - "output_cost_per_token": 0.0 + "output_cost_per_token": 0.0, + "supports_vision": true + }, + "voyage/voyage-multimodal-3.5": { + "input_cost_per_token": 1.2e-07, + "litellm_provider": "voyage", + "max_input_tokens": 32000, + "max_tokens": 32000, + "mode": "embedding", + "output_cost_per_token": 0.0, + "supports_vision": true }, "wandb/openai/gpt-oss-120b": { "max_tokens": 131072, diff --git a/litellm/utils.py b/litellm/utils.py index f902644e760..b07cb9c23fc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3427,6 +3427,15 @@ def get_optional_params_embeddings( # noqa: PLR0915 drop_params=drop_params if drop_params is not None else False, ) ) + elif litellm.VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings(model): + optional_params = ( + litellm.VoyageMultimodalEmbeddingConfig().map_openai_params( + non_default_params=non_default_params, + optional_params={}, + model=model, + drop_params=drop_params if drop_params is not None else False, + ) + ) else: optional_params = litellm.VoyageEmbeddingConfig().map_openai_params( non_default_params=non_default_params, @@ -8263,6 +8272,13 @@ class ProviderConfigManager: ) ): return litellm.VoyageContextualEmbeddingConfig() + elif ( + litellm.LlmProviders.VOYAGE == provider + and litellm.VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings( + model + ) + ): + return litellm.VoyageMultimodalEmbeddingConfig() elif litellm.LlmProviders.VOYAGE == provider: return litellm.VoyageEmbeddingConfig() elif litellm.LlmProviders.TRITON == provider: diff --git a/tests/test_litellm/llms/voyage/embedding/test_voyage_multimodal_embedding.py b/tests/test_litellm/llms/voyage/embedding/test_voyage_multimodal_embedding.py new file mode 100644 index 00000000000..3b5173db1ff --- /dev/null +++ b/tests/test_litellm/llms/voyage/embedding/test_voyage_multimodal_embedding.py @@ -0,0 +1,418 @@ +""" +Test cases for Voyage multimodal embedding configuration. + +Tests the VoyageMultimodalEmbeddingConfig class including model detection, +URL generation, parameter mapping, input transformation, and response handling. +""" + +import json +import os +import sys + +import httpx +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.voyage.embedding.transformation_multimodal import ( + VoyageMultimodalEmbeddingConfig, + _convert_content_block, +) +from litellm.types.utils import EmbeddingResponse, Usage + + +class TestIsMultimodalEmbeddings: + def test_multimodal_model_detected(self): + assert ( + VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings( + "voyage-multimodal-3" + ) + is True + ) + + def test_multimodal_3_5_detected(self): + assert ( + VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings( + "voyage-multimodal-3.5" + ) + is True + ) + + def test_case_insensitive(self): + assert ( + VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings( + "Voyage-Multimodal-3" + ) + is True + ) + + def test_standard_model_not_detected(self): + assert ( + VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings("voyage-3") + is False + ) + + def test_contextual_model_not_detected(self): + assert ( + VoyageMultimodalEmbeddingConfig.is_multimodal_embeddings("voyage-context-3") + is False + ) + + +class TestGetCompleteUrl: + def setup_method(self): + self.config = VoyageMultimodalEmbeddingConfig() + + def test_default_url(self): + url = self.config.get_complete_url( + api_base=None, + api_key=None, + model="voyage-multimodal-3", + optional_params={}, + litellm_params={}, + ) + assert url == "https://api.voyageai.com/v1/multimodalembeddings" + + def test_custom_api_base(self): + url = self.config.get_complete_url( + api_base="https://custom.api.com/v1", + api_key=None, + model="voyage-multimodal-3", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.api.com/v1/multimodalembeddings" + + def test_custom_api_base_already_has_path(self): + url = self.config.get_complete_url( + api_base="https://custom.api.com/v1/multimodalembeddings", + api_key=None, + model="voyage-multimodal-3", + optional_params={}, + litellm_params={}, + ) + assert url == "https://custom.api.com/v1/multimodalembeddings" + + +class TestGetSupportedOpenaiParams: + def test_supported_params(self): + config = VoyageMultimodalEmbeddingConfig() + params = config.get_supported_openai_params("voyage-multimodal-3") + assert "encoding_format" in params + assert "dimensions" in params + assert "input_type" in params + + +class TestMapOpenaiParams: + def setup_method(self): + self.config = VoyageMultimodalEmbeddingConfig() + + def test_encoding_format_mapped_to_output_encoding(self): + result = self.config.map_openai_params( + non_default_params={"encoding_format": "base64"}, + optional_params={}, + model="voyage-multimodal-3", + drop_params=False, + ) + assert result == {"output_encoding": "base64"} + + def test_dimensions_mapped_to_output_dimension(self): + result = self.config.map_openai_params( + non_default_params={"dimensions": 1024}, + optional_params={}, + model="voyage-multimodal-3", + drop_params=False, + ) + assert result == {"output_dimension": 1024} + + def test_input_type_passed_through(self): + result = self.config.map_openai_params( + non_default_params={"input_type": "query"}, + optional_params={}, + model="voyage-multimodal-3", + drop_params=False, + ) + assert result == {"input_type": "query"} + + def test_all_params_combined(self): + result = self.config.map_openai_params( + non_default_params={ + "encoding_format": "base64", + "dimensions": 512, + "input_type": "document", + }, + optional_params={}, + model="voyage-multimodal-3", + drop_params=False, + ) + assert result == { + "output_encoding": "base64", + "output_dimension": 512, + "input_type": "document", + } + + +class TestConvertContentBlock: + def test_text_block(self): + result = _convert_content_block({"type": "text", "text": "hello"}) + assert result == {"type": "text", "text": "hello"} + + def test_image_url_block_string(self): + result = _convert_content_block( + {"type": "image_url", "image_url": "https://example.com/img.jpg"} + ) + assert result == { + "type": "image_url", + "image_url": "https://example.com/img.jpg", + } + + def test_image_url_block_openai_format(self): + result = _convert_content_block( + {"type": "image_url", "image_url": {"url": "https://example.com/img.jpg"}} + ) + assert result == { + "type": "image_url", + "image_url": "https://example.com/img.jpg", + } + + def test_image_url_block_base64_data_uri(self): + data_uri = "data:image/jpeg;base64,/9j/4AAQ..." + result = _convert_content_block({"type": "image_url", "image_url": data_uri}) + assert result == {"type": "image_base64", "image_base64": data_uri} + + def test_image_url_block_base64_data_uri_openai_format(self): + data_uri = "data:image/jpeg;base64,/9j/4AAQ..." + result = _convert_content_block( + {"type": "image_url", "image_url": {"url": data_uri}} + ) + assert result == {"type": "image_base64", "image_base64": data_uri} + + def test_image_base64_block(self): + result = _convert_content_block( + {"type": "image_base64", "image_base64": "data:image/png;base64,abc"} + ) + assert result == { + "type": "image_base64", + "image_base64": "data:image/png;base64,abc", + } + + def test_video_url_block(self): + result = _convert_content_block( + {"type": "video_url", "video_url": "https://example.com/video.mp4"} + ) + assert result == { + "type": "video_url", + "video_url": "https://example.com/video.mp4", + } + + def test_unknown_type_passed_through(self): + block = {"type": "unknown", "data": "something"} + result = _convert_content_block(block) + assert result == block + + +class TestTransformEmbeddingRequest: + def setup_method(self): + self.config = VoyageMultimodalEmbeddingConfig() + + def test_single_text_string(self): + result = self.config.transform_embedding_request( + model="voyage-multimodal-3", + input="Hello world", + optional_params={}, + headers={}, + ) + assert result == { + "inputs": [{"content": [{"type": "text", "text": "Hello world"}]}], + "model": "voyage-multimodal-3", + } + + def test_text_list(self): + result = self.config.transform_embedding_request( + model="voyage-multimodal-3", + input=["Hello", "World"], + optional_params={}, + headers={}, + ) + assert result == { + "inputs": [ + {"content": [{"type": "text", "text": "Hello"}]}, + {"content": [{"type": "text", "text": "World"}]}, + ], + "model": "voyage-multimodal-3", + } + + def test_multimodal_input_with_content_blocks(self): + input_data = [ + { + "content": [ + {"type": "text", "text": "A photo of a cat"}, + {"type": "image_url", "image_url": "https://example.com/cat.jpg"}, + ] + } + ] + result = self.config.transform_embedding_request( + model="voyage-multimodal-3", + input=input_data, + optional_params={}, + headers={}, + ) + assert result == { + "inputs": [ + { + "content": [ + {"type": "text", "text": "A photo of a cat"}, + { + "type": "image_url", + "image_url": "https://example.com/cat.jpg", + }, + ] + } + ], + "model": "voyage-multimodal-3", + } + + def test_openai_style_image_url_converted(self): + input_data = [ + { + "content": [ + {"type": "text", "text": "Describe this"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/img.png"}, + }, + ] + } + ] + result = self.config.transform_embedding_request( + model="voyage-multimodal-3", + input=input_data, + optional_params={}, + headers={}, + ) + assert result["inputs"][0]["content"][1] == { + "type": "image_url", + "image_url": "https://example.com/img.png", + } + + def test_optional_params_included(self): + result = self.config.transform_embedding_request( + model="voyage-multimodal-3", + input="test", + optional_params={"input_type": "query", "output_dimension": 512}, + headers={}, + ) + assert result["input_type"] == "query" + assert result["output_dimension"] == 512 + + +class TestTransformEmbeddingResponse: + def setup_method(self): + self.config = VoyageMultimodalEmbeddingConfig() + + def test_standard_response(self): + voyage_response = { + "data": [ + {"object": "embedding", "embedding": [0.1, 0.2, 0.3], "index": 0}, + {"object": "embedding", "embedding": [0.4, 0.5, 0.6], "index": 1}, + ], + "object": "list", + "model": "voyage-multimodal-3", + "usage": { + "text_tokens": 5, + "image_pixels": 2000000, + "total_tokens": 1005, + }, + } + mock_response = httpx.Response( + status_code=200, + content=json.dumps(voyage_response).encode("utf-8"), + headers={"content-type": "application/json"}, + ) + + model_response = EmbeddingResponse() + result = self.config.transform_embedding_response( + model="voyage-multimodal-3", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={"inputs": []}, + ) + + assert result.object == "list" + assert result.model == "voyage-multimodal-3" + assert len(result.data) == 2 + assert result.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert result.data[1]["embedding"] == [0.4, 0.5, 0.6] + assert isinstance(result.usage, Usage) + assert result.usage.total_tokens == 1005 + + def test_error_response(self): + mock_response = httpx.Response( + status_code=400, + content=b"not json", + headers={"content-type": "text/plain"}, + ) + model_response = EmbeddingResponse() + from litellm.llms.voyage.embedding.transformation_multimodal import ( + VoyageMultimodalEmbeddingError, + ) + + with pytest.raises(VoyageMultimodalEmbeddingError): + self.config.transform_embedding_response( + model="voyage-multimodal-3", + raw_response=mock_response, + model_response=model_response, + logging_obj=None, + request_data={}, + ) + + +class TestProviderConfigRouting: + def test_multimodal_model_returns_multimodal_config(self): + from litellm.utils import ProviderConfigManager + import litellm + + config = ProviderConfigManager.get_provider_embedding_config( + model="voyage-multimodal-3", + provider=litellm.LlmProviders.VOYAGE, + ) + assert isinstance(config, VoyageMultimodalEmbeddingConfig) + + def test_multimodal_3_5_returns_multimodal_config(self): + from litellm.utils import ProviderConfigManager + import litellm + + config = ProviderConfigManager.get_provider_embedding_config( + model="voyage-multimodal-3.5", + provider=litellm.LlmProviders.VOYAGE, + ) + assert isinstance(config, VoyageMultimodalEmbeddingConfig) + + def test_standard_model_returns_standard_config(self): + from litellm.utils import ProviderConfigManager + from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig + import litellm + + config = ProviderConfigManager.get_provider_embedding_config( + model="voyage-3", + provider=litellm.LlmProviders.VOYAGE, + ) + assert isinstance(config, VoyageEmbeddingConfig) + + def test_contextual_model_returns_contextual_config(self): + from litellm.utils import ProviderConfigManager + from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, + ) + import litellm + + config = ProviderConfigManager.get_provider_embedding_config( + model="voyage-context-3", + provider=litellm.LlmProviders.VOYAGE, + ) + assert isinstance(config, VoyageContextualEmbeddingConfig) + + +if __name__ == "__main__": + pytest.main([__file__])