diff --git a/litellm/constants.py b/litellm/constants.py index cb8414870ad..8eba230baab 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1085,11 +1085,13 @@ dashscope_models: set = set( ] ) -nebius_embedding_models: List = [ - "BAAI/bge-en-icl", - "BAAI/bge-multilingual-gemma2", - "intfloat/e5-mistral-7b-instruct", -] +nebius_embedding_models: set = set( + [ + "BAAI/bge-en-icl", + "BAAI/bge-multilingual-gemma2", + "intfloat/e5-mistral-7b-instruct", + ] +) WANDB_MODELS: set = set( [ @@ -1120,45 +1122,47 @@ WANDB_MODELS: set = set( ] ) -modelscope_models: List = [ - # Qwen series models - "Qwen/Qwen3-0.6B", - "Qwen/Qwen3-1.7B", - "Qwen/Qwen3-4B", - "Qwen/Qwen3-8B", - "Qwen/Qwen3-14B", - "Qwen/Qwen3-30B-A3B", - "Qwen/Qwen3-32B", - "Qwen/Qwen3-235B-A22B", - "Qwen/Qwen3-235B-A22B-Instruct-2507", - "Qwen/Qwen3-235B-A22B-Thinking-2507", - "Qwen/Qwen3-30B-A3B-Thinking-2507", - "Qwen/Qwen3-Coder-30B-A3B-Instruct", - "Qwen/Qwen3-Coder-480B-A35B-Instruct", - "Qwen/Qwen3-Next-80B-A3B-Instruct", - "Qwen/Qwen3-Next-80B-A3B-Thinking", - "Qwen/Qwen3-VL-235B-A22B-Instruct", - "Qwen/Qwen3-VL-8B-Instruct", - "Qwen/Qwen3-VL-8B-Thinking", - "Qwen/Qwen3.5-122B-A10B", - "Qwen/Qwen3.5-27B", - "Qwen/Qwen3.5-35B-A3B", - "Qwen/Qwen3.5-397B-A17B", - "Qwen/QwQ-32B", - "Qwen/QwQ-32B-Preview", - "Qwen/QVQ-72B-Preview", - "Qwen/Qwen-Image-Edit", - # DeepSeek series models - "deepseek-ai/DeepSeek-R1-0528", - "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - "deepseek-ai/DeepSeek-V3.2", - "deepseek-ai/DeepSeek-V4-Flash", -] +modelscope_models: set = set( + [ + # Qwen series models + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3-1.7B", + "Qwen/Qwen3-4B", + "Qwen/Qwen3-8B", + "Qwen/Qwen3-14B", + "Qwen/Qwen3-30B-A3B", + "Qwen/Qwen3-32B", + "Qwen/Qwen3-235B-A22B", + "Qwen/Qwen3-235B-A22B-Instruct-2507", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "Qwen/Qwen3-30B-A3B-Thinking-2507", + "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Thinking", + "Qwen/Qwen3-VL-235B-A22B-Instruct", + "Qwen/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-8B-Thinking", + "Qwen/Qwen3.5-122B-A10B", + "Qwen/Qwen3.5-27B", + "Qwen/Qwen3.5-35B-A3B", + "Qwen/Qwen3.5-397B-A17B", + "Qwen/QwQ-32B", + "Qwen/QwQ-32B-Preview", + "Qwen/QVQ-72B-Preview", + "Qwen/Qwen-Image-Edit", + # DeepSeek series models + "deepseek-ai/DeepSeek-R1-0528", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + "deepseek-ai/DeepSeek-V3.2", + "deepseek-ai/DeepSeek-V4-Flash", + ] +) BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "cohere", diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py index 257959685e8..9f26bee5f54 100644 --- a/litellm/llms/modelscope/chat/transformation.py +++ b/litellm/llms/modelscope/chat/transformation.py @@ -4,15 +4,20 @@ Translates from OpenAI's `/v1/chat/completions` to ModelScope's `/v1/chat/comple from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - handle_messages_with_content_list_to_str_conversion, -) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from ...openai.chat.gpt_transformation import OpenAIGPTConfig +def _has_non_text_content(message: AllMessageValues) -> bool: + """Check if a message has non-text content items (e.g. image_url).""" + content = message.get("content") + if not isinstance(content, list): + return False + return any(item.get("type") != "text" for item in content) + + class ModelScopeChatConfig(OpenAIGPTConfig): DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1" @@ -33,9 +38,20 @@ class ModelScopeChatConfig(OpenAIGPTConfig): self, messages: List[AllMessageValues], model: str, is_async: bool = False ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: """ - ModelScope does not support content in list format. + Flatten text-only content lists to strings for ModelScope. + + Messages with non-text content (e.g. image_url for vision models) + are kept as lists so the parent class can normalize them properly. """ - messages = handle_messages_with_content_list_to_str_conversion(messages) + for message in messages: + if _has_non_text_content(message): + continue + content = message.get("content") + if isinstance(content, list): + message["content"] = "".join( + item.get("text") or "" for item in content + ) + if is_async: return super()._transform_messages( messages=messages, model=model, is_async=True diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py index 2716a8057ef..6b848b20ee9 100644 --- a/litellm/llms/modelscope/image_generation/transformation.py +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -176,7 +177,6 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): error_message=f"Error parsing ModelScope response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, - model=model, ) # Check for errors in response @@ -188,7 +188,6 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): error_message=f"ModelScope error: {error_msg}", status_code=raw_response.status_code, headers=raw_response.headers, - model=model, ) # Extract images from response @@ -211,8 +210,7 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): error_message: str, status_code: int, headers: Union[dict, httpx.Headers], - model: Optional[str] = None, - ) -> Exception: + ) -> BaseLLMException: """Return the appropriate error class for ModelScope.""" from litellm.exceptions import ( AuthenticationError, @@ -221,26 +219,26 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): ) if status_code == 400: - return BadRequestError( + return BadRequestError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) elif status_code == 401: - return AuthenticationError( + return AuthenticationError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) elif status_code >= 500: - return InternalServerError( + return InternalServerError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) else: - return BadRequestError( + return BadRequestError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) diff --git a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py index 89c521533a1..2767deae176 100644 --- a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py +++ b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py @@ -5,6 +5,7 @@ These tests validate the ModelScopeChatConfig class which extends OpenAIGPTConfi ModelScope is an OpenAI-compatible provider with minor customizations. """ +import json import os import sys @@ -12,12 +13,18 @@ sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +from unittest.mock import patch + +import httpx import pytest +import respx import litellm from litellm import completion from litellm.llms.modelscope.chat.transformation import ModelScopeChatConfig +DEFAULT_MODEL = "Qwen/Qwen3.5-35B-A3B" + class TestModelScopeConfig: """Test class for ModelScope functionality""" @@ -28,55 +35,40 @@ class TestModelScopeConfig: headers = {} api_key = "fake-modelscope-key" - # Call validate_environment without specifying api_base result = config.validate_environment( headers=headers, - model="Qwen/Qwen3-8B", + model=DEFAULT_MODEL, messages=[{"role": "user", "content": "Hey"}], optional_params={}, litellm_params={}, api_key=api_key, - api_base=None, # Not providing api_base + api_base=None, ) - # Verify headers are still set correctly assert result["Authorization"] == f"Bearer {api_key}" assert result["Content-Type"] == "application/json" - # We can't directly test the api_base value here since validate_environment - # only returns the headers, but we can verify it doesn't raise an exception - # which would happen if api_base handling was incorrect - @pytest.mark.respx() def test_modelscope_completion_mock(self, respx_mock): - """ - Mock test for ModelScope completion using the model format from docs. - This test mocks the actual HTTP request to test the integration properly. - """ + """Mock test for basic ModelScope completion.""" - litellm.disable_aiohttp_transport = ( - True # since this uses respx, we need to set use_aiohttp_transport to False - ) + litellm.disable_aiohttp_transport = True - # Set up environment variables for the test api_key = "fake-modelscope-key" api_base = "https://api-inference.modelscope.cn/v1" - model = "modelscope/Qwen/Qwen3-8B" # Use modelscope/ prefix to specify provider - model_name = "Qwen3-8B" # The actual model name without provider prefix - # Mock the HTTP request to the ModelScope API respx_mock.post(f"{api_base}/chat/completions").respond( json={ "id": "chatcmpl-123", "object": "chat.completion", "created": 1677652288, - "model": model_name, + "model": DEFAULT_MODEL, "choices": [ { "index": 0, "message": { "role": "assistant", - "content": '```python\nprint("Hey from LiteLLM!")\n```\n\nThis simple Python code prints a greeting message from LiteLLM.', + "content": '```python\nprint("Hey from LiteLLM!")\n```', }, "finish_reason": "stop", } @@ -90,9 +82,8 @@ class TestModelScopeConfig: status_code=200, ) - # Make the actual API call through LiteLLM response = completion( - model=model, + model=f"modelscope/{DEFAULT_MODEL}", messages=[ {"role": "user", "content": "write code for saying hey from LiteLLM"} ], @@ -100,15 +91,304 @@ class TestModelScopeConfig: api_base=api_base, ) - # Verify response structure assert response is not None - assert hasattr(response, "choices") - assert len(response.choices) > 0 - assert hasattr(response.choices[0], "message") - assert hasattr(response.choices[0].message, "content") assert response.choices[0].message.content is not None - - # Check for specific content in the response assert "```python" in response.choices[0].message.content - assert "Hey from LiteLLM" in response.choices[0].message.content + # ── _transform_messages tests ────────────────────────────────────── + + def test_transform_messages_flattens_text_content_list(self): + """Content lists containing only text items should be flattened to a string.""" + config = ModelScopeChatConfig() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ], + } + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hello world" + + def test_transform_messages_preserves_multimodal_content_list(self): + """Content lists with image_url should be preserved as lists for vision models.""" + config = ModelScopeChatConfig() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/img.png"}}, + ], + } + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert isinstance(result[0]["content"], list) + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][1]["type"] == "image_url" + + def test_transform_messages_string_content_unchanged(self): + """Messages with string content should pass through unchanged.""" + config = ModelScopeChatConfig() + messages = [{"role": "user", "content": "Hello"}] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hello" + + def test_transform_messages_multi_turn(self): + """Multi-turn conversations should be handled correctly.""" + config = ModelScopeChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Tell me more"}, + ], + }, + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hi" + assert result[1]["content"] == "Hello!" + assert result[2]["content"] == "Tell me more" + + def test_transform_messages_multimodal_multi_turn(self): + """Multi-turn with mixed text-only and multimodal messages.""" + config = ModelScopeChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "https://example.com/photo.jpg"}}, + ], + }, + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hi" + assert result[1]["content"] == "Hello!" + # Multimodal message should keep list format + assert isinstance(result[2]["content"], list) + assert result[2]["content"][1]["type"] == "image_url" + + # ── get_complete_url tests ───────────────────────────────────────── + + def test_get_complete_url_default(self): + """Default api_base should append /chat/completions.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base=None, + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://api-inference.modelscope.cn/v1/chat/completions" + + def test_get_complete_url_custom_base(self): + """Custom api_base should append /chat/completions.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base="https://custom.modelscope.cn/v1", + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://custom.modelscope.cn/v1/chat/completions" + + def test_get_complete_url_already_has_endpoint(self): + """api_base already ending in /chat/completions should not be doubled.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base="https://api-inference.modelscope.cn/v1/chat/completions", + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://api-inference.modelscope.cn/v1/chat/completions" + assert url.count("/chat/completions") == 1 + + # ── _get_openai_compatible_provider_info tests ───────────────────── + + def test_get_provider_info_with_explicit_api_base(self): + """Explicit api_base and api_key should be returned as-is.""" + config = ModelScopeChatConfig() + + api_base, api_key = config._get_openai_compatible_provider_info( + api_base="https://custom.example.com/v1", + api_key="my-key", + ) + + assert api_base == "https://custom.example.com/v1" + assert api_key == "my-key" + + def test_get_provider_info_default_fallback(self): + """When no api_base or env var is set, DEFAULT_BASE_URL should be used.""" + config = ModelScopeChatConfig() + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("MODELSCOPE_API_BASE", None) + os.environ.pop("MODELSCOPE_API_KEY", None) + + api_base, api_key = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None, + ) + + assert api_base == "https://api-inference.modelscope.cn/v1" + assert api_key is None + + def test_get_provider_info_env_var_fallback(self): + """MODELSCOPE_API_BASE env var should be used when api_base is not provided.""" + config = ModelScopeChatConfig() + + with patch.dict( + os.environ, + {"MODELSCOPE_API_BASE": "https://env.modelscope.cn/v1"}, + ): + api_base, _ = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None, + ) + + assert api_base == "https://env.modelscope.cn/v1" + + # ── Mock HTTP tests ──────────────────────────────────────────────── + + @pytest.mark.respx() + def test_completion_with_text_content_list(self, respx_mock): + """Verify that text-only content list messages are flattened before sending.""" + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + captured_request = {} + + def capture_request(request): + captured_request["body"] = request.content + return httpx.Response( + 200, + json={ + "id": "chatcmpl-456", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Sure!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ) + + respx_mock.post(f"{api_base}/chat/completions").mock(side_effect=capture_request) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ], + } + ], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "Sure!" + + body = json.loads(captured_request["body"]) + assert isinstance(body["messages"][0]["content"], str) + assert body["messages"][0]["content"] == "Hello world" + + @pytest.mark.respx() + def test_completion_with_multimodal_messages(self, respx_mock): + """Verify that multimodal messages (text + image_url) are sent as content lists.""" + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + captured_request = {} + + def capture_request(request): + captured_request["body"] = request.content + return httpx.Response( + 200, + json={ + "id": "chatcmpl-789", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "A cat sitting on a couch.", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 8, "total_tokens": 108}, + }, + ) + + respx_mock.post(f"{api_base}/chat/completions").mock(side_effect=capture_request) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.jpg"}, + }, + ], + } + ], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "A cat sitting on a couch." + + body = json.loads(captured_request["body"]) + msg = body["messages"][0] + # Multimodal content should remain as a list + assert isinstance(msg["content"], list) + assert len(msg["content"]) == 2 + assert msg["content"][0] == {"type": "text", "text": "What is in this image?"} + assert msg["content"][1]["type"] == "image_url" + assert msg["content"][1]["image_url"]["url"] == "https://example.com/cat.jpg" diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py index 69d50237f72..7f00f53c451 100644 --- a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -415,7 +415,6 @@ class TestModelScopeImageGenerationTransformation: error_message="Bad request", status_code=400, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, BadRequestError) @@ -428,7 +427,6 @@ class TestModelScopeImageGenerationTransformation: error_message="Invalid API key", status_code=401, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, AuthenticationError) @@ -441,7 +439,6 @@ class TestModelScopeImageGenerationTransformation: error_message="Internal server error", status_code=500, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, InternalServerError) @@ -454,7 +451,6 @@ class TestModelScopeImageGenerationTransformation: error_message="Some error", status_code=404, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, BadRequestError)