From bbc68e21f92f017853859d76fb43ca5e4641a2f2 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:48:38 -0700 Subject: [PATCH] feat(dashscope): add image edits and text-to-speech (#48998) * feat(dashscope): add image edits and text-to-speech * fix(dashscope): bill reported speech characters and input images, follow the chat region for edits, keep the call timeout on the audio download * fix(dashscope): bill deployment image prices as main does and route edits to the host speech uses A deployment that sets its own input_cost_per_image, output_cost_per_image or input_cost_per_pixel bills through the default image calculator again, so an override is never read as a per-input-image price and pixel pricing on qwen-image-3.0-pro no longer raises. Edits now resolve the caller's api_base or DASHSCOPE_API_BASE through the same API-root helper speech uses, on any host, with DASHSCOPE_API_BASE_IMAGE still winning. An image response whose usage is null or not an object keeps its images instead of raising. * fix(images): bill a deployment's own prices on image edits through the proxy The proxy builds the logging object before the router picks a deployment, so image_edit never handed the deployment's price fields to its cost calculation and billed the cost-map price instead. Pass the custom pricing params with the pre-call update, as the Responses API does. * fix(images): bill per-pixel image prices at a DashScope W*H size DashScope's mapped size reads 1328*1328, which the cost size reader rejected, so a deployment's input_cost_per_pixel billed every DashScope generation or edit at the 1024x1024 default. * fix(dashscope): bill image prices by their meaning, report the rendered size, prefer the deployment edit base, fetch audio over https * fix(dashscope): read the rendered image size from output_width and output_height * test(images): bill a fal edit at its deployment's own image price * fix(dashscope): keep the qwen-image-3.0 input-image fee out of the shared cost map row Released proxies read the cost map from main and bill input_cost_per_image as the flat price of each DashScope image, so the row's $0.003 input fee billed every qwen-image-3.0 image there at a tenth of its $0.03 price. A deployment can still set input_cost_per_image to bill DashScope's input images --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 10 +- litellm/images/main.py | 3 +- .../litellm_core_utils/llm_cost_calc/utils.py | 24 +- .../base_llm/text_to_speech/transformation.py | 12 + litellm/llms/custom_httpx/llm_http_handler.py | 2 +- litellm/llms/dashscope/common_utils.py | 22 +- litellm/llms/dashscope/image_edit/__init__.py | 0 .../dashscope/image_edit/transformation.py | 160 +++++++ .../image_generation/cost_calculator.py | 19 + .../image_generation/transformation.py | 94 ++-- .../llms/dashscope/text_to_speech/__init__.py | 0 .../text_to_speech/transformation.py | 297 +++++++++++++ litellm/main.py | 9 +- ...odel_prices_and_context_window_backup.json | 67 ++- .../provider_endpoints_support_backup.json | 4 +- litellm/utils.py | 57 ++- model_prices_and_context_window.json | 67 ++- provider_endpoints_support.json | 4 +- ...st_dashscope_image_edit_and_speech_wire.py | 259 +++++++++++ tests/unit/images/test_image_edit.py | 119 +++++ ...est_dashscope_image_edit_transformation.py | 358 +++++++++++++++ ...dashscope_text_to_speech_transformation.py | 415 ++++++++++++++++++ .../test_text_to_speech_transformation.py | 38 ++ .../test_get_complete_url_trailing_slash.py | 1 + tests/unit/test_dashscope_image_generation.py | 61 ++- 25 files changed, 2009 insertions(+), 93 deletions(-) create mode 100644 litellm/llms/dashscope/image_edit/__init__.py create mode 100644 litellm/llms/dashscope/image_edit/transformation.py create mode 100644 litellm/llms/dashscope/image_generation/cost_calculator.py create mode 100644 litellm/llms/dashscope/text_to_speech/__init__.py create mode 100644 litellm/llms/dashscope/text_to_speech/transformation.py create mode 100644 tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py create mode 100644 tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py create mode 100644 tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a4a9a5bee39..defe1e63b86 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1887,7 +1887,10 @@ def completion_cost( video_resolution=video_resolution, ) elif call_type in _SPEECH_CALL_TYPES: - prompt_characters = litellm.utils.count_characters(text=prompt) + prompt_characters = _speech_billed_characters( + completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None, + prompt, + ) elif call_type in _TRANSCRIPTION_CALL_TYPES: # Check _hidden_params first (duration stored there to # avoid polluting the response body), then fall back to @@ -2366,6 +2369,11 @@ def response_cost_calculator( raise e +def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int: + reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None + return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt) + + def _video_input_image_cost( model: str, custom_llm_provider: str | None, diff --git a/litellm/images/main.py b/litellm/images/main.py index 4aad5210028..165543aadba 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -53,7 +53,7 @@ from litellm.main import ( from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.llms.openai import ImageGenerationRequestQuality -from litellm.types.router import GenericLiteLLMParams +from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams from litellm.types.utils import ( LITELLM_IMAGE_VARIATION_PROVIDERS, LlmProviders, @@ -1010,6 +1010,7 @@ def image_edit( optional_params=dict(image_edit_request_params), litellm_params={ **image_edit_request_params, + **litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True), "litellm_call_id": litellm_call_id, "model_info": model_info, "vertex_location": litellm_params.vertex_location, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..404dfab0a6d 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = ( "output_cost_per_image_token", ) + +def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool: + return pricing is not None and ( + "input_cost_per_pixel" in pricing + or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing) + ) + + # Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per # request in the cost-calc path, so the f-strings are built once here instead # of being rebuilt for every model_info key on every call. Longest-first so a @@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None: return value if isinstance(value, int) else None -_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+") +_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+") def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None: @@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None: value: Final = _requested_image_param(optional_params, "size") - return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None + return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None def get_web_search_requests(server_tool_use: object) -> int | None: @@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol): def _model_priced_image_cost_calculator( - custom_llm_provider: str | None, model: str + custom_llm_provider: str | None, model: str, pricing: ModelInfo | None ) -> _ModelPricedImageCostCalculator | None: match custom_llm_provider: case litellm.LlmProviders.RECRAFT.value: @@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator( from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None + case litellm.LlmProviders.DASHSCOPE.value: + if _deployment_prices_images_without_an_output_price(pricing): + return None + from litellm.llms.dashscope.image_generation.cost_calculator import ( + cost_calculator as dashscope_image_cost_calculator, + ) + + return dashscope_image_cost_calculator case _: return None @@ -2178,7 +2194,7 @@ class CostCalculatorUtils: ) resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0) pricing: Final = deployment_pricing(model_info) - model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model) + model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing) if model_priced_calculator is not None: return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing) diff --git a/litellm/llms/base_llm/text_to_speech/transformation.py b/litellm/llms/base_llm/text_to_speech/transformation.py index be49098b62b..6df83966bf8 100644 --- a/litellm/llms/base_llm/text_to_speech/transformation.py +++ b/litellm/llms/base_llm/text_to_speech/transformation.py @@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC): Transform provider response to standard format """ + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + return self.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=logging_obj, + ) + def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException: from ..chat.transformation import BaseLLMException diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1f25235a651..c92315e53b3 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -12849,7 +12849,7 @@ class BaseLLMHTTPHandler: provider_config=text_to_speech_provider_config, ) - speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response( + speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response( model=model, raw_response=response, logging_obj=logging_obj, diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 5ad2aa17a3a..d3e1fb8eada 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic" DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1" DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1" DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks" -DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com" _STANDARD_DASHSCOPE_HOSTS: Final = frozenset( @@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool: return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/") +def dashscope_api_root(api_base: str) -> str: + endpoints: Final = resolve_dashscope_endpoints(api_base) + if endpoints is not None: + return endpoints.origin + trimmed: Final = api_base.rstrip("/") + if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH): + return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH) + root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH) + return root if separator else trimmed + + +def is_aliyun_host(host: str | None) -> bool: + return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com")) + + def resolve_dashscope_openai_api_base( api_base: str | None, env_var: str = "DASHSCOPE_API_BASE", @@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None: if api_base is None: return None parsed: Final = urlparse(api_base) - host: Final = parsed.hostname or "" - on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com") - if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: + if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH: return None return f"{parsed.scheme}://{parsed.netloc}" diff --git a/litellm/llms/dashscope/image_edit/__init__.py b/litellm/llms/dashscope/image_edit/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/image_edit/transformation.py b/litellm/llms/dashscope/image_edit/transformation.py new file mode 100644 index 00000000000..0d34017a321 --- /dev/null +++ b/litellm/llms/dashscope/image_edit/transformation.py @@ -0,0 +1,160 @@ +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +import httpx +from httpx._types import RequestFiles +from pydantic import TypeAdapter +from typing_extensions import override + +import litellm +from litellm.images.utils import image_to_data_url +from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_CHAT_COMPATIBLE_PATH, + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + dashscope_api_root, + is_dashscope_native_endpoint_url, + resolve_dashscope_endpoints, +) +from litellm.llms.dashscope.image_generation.transformation import ( + DEFAULT_API_BASE, + OPENAI_TO_DASHSCOPE_SIZE, + DashScopeImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.images.main import ImageEditOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +MAX_INPUT_IMAGES: Final = 3 +DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed") + +_IMAGES: Final = TypeAdapter(tuple[object, ...]) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + +def _deployment_edit_url(api_base: str) -> str: + derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and ( + resolve_dashscope_endpoints(api_base) is not None + or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH) + ) + return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base + + +class DashScopeImageEditConfig(BaseImageEditConfig): + def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None: + super().__init__() + self._generation_config: Final = generation_config or DashScopeImageGenerationConfig() + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + return list(DASHSCOPE_IMAGE_EDIT_PARAMS) + + @override + def map_openai_params( + self, + image_edit_optional_params: ImageEditOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: mirrors override contract + return { + key: _dashscope_size(value) if key == "size" else value + for key, value in image_edit_optional_params.items() + if key in DASHSCOPE_IMAGE_EDIT_PARAMS + } + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return self._generation_config.validate_environment( + headers=dict(headers), + model=model, + messages=[], + optional_params={}, + litellm_params=dict(litellm_params or {}), + api_key=api_key, + api_base=api_base, + ) + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + if api_base is not None: + return _deployment_edit_url(api_base) + env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE") + derived: Final = ( + f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None + ) + return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE + + @override + def use_multipart_form_data(self) -> bool: + return False + + @override + def transform_image_edit_request( + self, + model: str, + prompt: str | None, + image: object, + image_edit_optional_request_params: Mapping[str, object], + litellm_params: GenericLiteLLMParams, + headers: Mapping[str, str], + ) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract + candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,) + images: Final = tuple(candidate for candidate in candidates if candidate is not None) + if len(images) > MAX_INPUT_IMAGES: + raise litellm.BadRequestError( + message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}", + model=model, + llm_provider="dashscope", + ) + prompt_content: Final = ({"text": prompt},) if prompt is not None else () + return { + "model": model, + "input": { + "messages": [ + { + "role": "user", + "content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content], + } + ] + }, + "parameters": dict(image_edit_optional_request_params), + }, () + + @override + def transform_image_edit_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> ImageResponse: + return self._generation_config.transform_image_generation_response( + model=model, + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) diff --git a/litellm/llms/dashscope/image_generation/cost_calculator.py b/litellm/llms/dashscope/image_generation/cost_calculator.py new file mode 100644 index 00000000000..f699ef7e1c5 --- /dev/null +++ b/litellm/llms/dashscope/image_generation/cost_calculator.py @@ -0,0 +1,19 @@ +from typing import Final + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info +from litellm.types.utils import ImageResponse, ModelInfo + + +def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float: + prices: Final = resolve_image_model_info( + model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info + ) + output_cost_per_image: Final = prices.get("output_cost_per_image") + if output_cost_per_image is None: + raise ValueError(f"No output_cost_per_image found for dashscope/{model}") + input_images: Final = image_response.hidden_params.get("billable_input_image_count") + input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * ( + input_images if isinstance(input_images, int) else 0 + ) + return output_cost_per_image * len(image_response.data or ()) + input_image_cost diff --git a/litellm/llms/dashscope/image_generation/transformation.py b/litellm/llms/dashscope/image_generation/transformation.py index 2f528b5a2c3..d1b5a7b40e7 100644 --- a/litellm/llms/dashscope/image_generation/transformation.py +++ b/litellm/llms/dashscope/image_generation/transformation.py @@ -23,7 +23,7 @@ Response format: } """ -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final import httpx @@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object) DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" # Maps OpenAI size strings (WxH) to DashScope size strings (W*H) -OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { +OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = { "256x256": "256*256", "512x512": "512*512", "1024x1024": "1024*1024", @@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = { } +def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]: + contents: Final = _JSON_OBJECTS.validate_python(message.get("content", [])) + return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url) + + +def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]: + output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) + choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) + for choice in choices: + yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {}))) + + +def _dashscope_size(size: object) -> object: + if not isinstance(size, str): + return size + return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*")) + + class DashScopeImageGenerationConfig(BaseImageGenerationConfig): """ Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro, @@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def map_openai_params( self, - non_default_params: dict, - optional_params: dict, + non_default_params: dict[str, object], + optional_params: dict[str, object], model: str, drop_params: bool, - ) -> dict: + ) -> dict[str, object]: supported_params: Final = self.get_supported_openai_params(model) - mapped: Final[dict] = {} - for k, v in non_default_params.items(): - if k in optional_params: - continue - if k not in supported_params: - continue - if k == "size": - # Convert "WxH" → "W*H" - mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*")) - else: - mapped[k] = v - return mapped + requested: Final = { + k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params + } + return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()} def _resolve_api_key(self, api_key: str | None) -> str: resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY") @@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): api_base: str | None, api_key: str | None, model: str, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], stream: bool | None = None, ) -> str: image_env_api_base: Final = self._image_api_base_env() @@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}" headers["Content-Type"] = "application/json" return headers @@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): raw_response: httpx.Response, model_response: ImageResponse, logging_obj: LiteLLMLoggingObj, - request_data: dict, - optional_params: dict, - litellm_params: dict, + request_data: dict[str, object], + optional_params: dict[str, object], + litellm_params: dict[str, object], encoding: "Tokenizer | None", api_key: str | None = None, json_mode: bool | None = None, @@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig): headers=raw_response.headers, ) - if not model_response.data: - model_response.data = [] - - output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {})) - choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", [])) - for choice in choices: - message = _JSON_OBJECT.validate_python(choice.get("message", {})) - content_list = _JSON_OBJECTS.validate_python(message.get("content", [])) - for content_item in content_list: - image_url = content_item.get("image") - if image_url: - model_response.data.append(ImageObject(url=image_url)) - - return model_response + images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))] + usage: Final = response_object.get("usage") + usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {} + width: Final = usage_fields.get("output_width", usage_fields.get("width")) + height: Final = usage_fields.get("output_height", usage_fields.get("height")) + image_response: Final = model_response.model_copy( + update={ + "data": images, + **({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}), + } + ) + input_image_count: Final = usage_fields.get("input_image_count") + if isinstance(input_image_count, int) and input_image_count > 0: + image_response.hidden_params = { + **model_response.hidden_params, + "billable_input_image_count": input_image_count, + } + return image_response diff --git a/litellm/llms/dashscope/text_to_speech/__init__.py b/litellm/llms/dashscope/text_to_speech/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/dashscope/text_to_speech/transformation.py b/litellm/llms/dashscope/text_to_speech/transformation.py new file mode 100644 index 00000000000..053682040bd --- /dev/null +++ b/litellm/llms/dashscope/text_to_speech/transformation.py @@ -0,0 +1,297 @@ +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from itertools import chain +from typing import TYPE_CHECKING, Final +from urllib.parse import urlparse + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from typing_extensions import override + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped + get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped +) +from litellm.llms.dashscope.common_utils import ( + DASHSCOPE_MULTIMODAL_GENERATION_PATH, + DashScopeError, + dashscope_api_root, + is_aliyun_host, + missing_dashscope_family_key_message, + resolve_dashscope_family_api_key, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + +DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com" +_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-" +_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None]) + + +@dataclass(frozen=True, slots=True) +class _TextToSpeechFamily: + path: str + response_formats: frozenset[str] + sends_format: bool + takes_instructions: bool + + +MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily( + path=DASHSCOPE_MULTIMODAL_GENERATION_PATH, + response_formats=frozenset({"wav"}), + sends_format=False, + takes_instructions=False, +) +SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily( + path="/api/v1/services/audio/tts/SpeechSynthesizer", + response_formats=frozenset({"mp3", "wav", "opus", "pcm"}), + sends_format=True, + takes_instructions=True, +) + + +class _Audio(BaseModel): + model_config = ConfigDict(frozen=True) + url: str + + +class _Output(BaseModel): + model_config = ConfigDict(frozen=True) + audio: _Audio + + +class _Usage(BaseModel): + model_config = ConfigDict(frozen=True) + input_tokens: int | None = None + output_tokens: int | None = None + characters: int | None = None + + +class _SpeechResponse(BaseModel): + model_config = ConfigDict(frozen=True) + output: _Output | None = None + usage: _Usage | None = None + code: str | None = None + message: str | None = None + + +def text_to_speech_family(model: str) -> _TextToSpeechFamily: + if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX): + return SPEECH_SYNTHESIZER_FAMILY + return MULTIMODAL_GENERATION_FAMILY + + +def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None: + match key: + case "response_format" if value in family.response_formats: + return {"format": value} if family.sends_format else {} + case "instructions" if family.takes_instructions: + return {"instruction": value} + case _: + return None + + +def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError: + return UnsupportedParamsError( + message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it", + model=model, + llm_provider="dashscope", + ) + + +def _speech_response(raw_response: httpx.Response) -> _SpeechResponse: + try: + return _SpeechResponse.model_validate_json(raw_response.content) + except ValidationError as e: + raise DashScopeError( + status_code=502, + message=f"Unexpected DashScope text-to-speech response: {e}", + headers=raw_response.headers, + ) from e + + +def _https_on_aliyun(url: str) -> str: + parsed: Final = urlparse(url) + return ( + parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url + ) + + +def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]: + parsed: Final = _speech_response(raw_response) + if parsed.output is None: + raise DashScopeError( + status_code=max(raw_response.status_code, 400), + message=parsed.message or parsed.code or raw_response.text, + headers=raw_response.headers, + ) + return _https_on_aliyun(parsed.output.audio.url), parsed.usage + + +def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None: + extension: Final = raw_response.request.extensions.get("timeout") + if extension is None: + return None + phases: Final = _TIMEOUT_PHASES.validate_python(extension) + return httpx.Timeout( + connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool") + ) + + +def _token_usage(usage: _Usage | None) -> Usage | None: + if usage is None or usage.input_tokens is None or usage.output_tokens is None: + return None + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ) + + +def _refused_audio_url(error: SSRFError) -> DashScopeError: + return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}") + + +def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent: + if audio_response.status_code != 200: + raise DashScopeError( + status_code=502, + message=f"DashScope audio download failed with HTTP {audio_response.status_code}", + headers=audio_response.headers, + ) + binary: Final = HttpxBinaryResponseContent(audio_response) + binary.usage = _token_usage(usage) + if usage is not None and usage.characters is not None: + binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters} + return binary + + +def _shared_async_client() -> AsyncHTTPHandler: + return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE) + + +class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig): + def __init__( + self, + sync_client: Callable[[], HTTPHandler] = get_httpx_client, + async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client, + ) -> None: + super().__init__() + self._sync_client: Final = sync_client + self._async_client: Final = async_client + + @override + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())] + + @override + def map_openai_params( + self, + model: str, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract + drop_params: bool = False, + kwargs: Mapping[str, object] | None = None, + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract + family: Final = text_to_speech_family(model) + fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items()) + unsupported: Final = ( + *(f"{key}={value!r}" for key, value, field in fields if field is None), + *(("voice as an object",) if isinstance(voice, dict) else ()), + ) + if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))): + raise _unsupported(model, unsupported) + supported: Final = tuple(field for _, _, field in fields if field is not None) + return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported)) + + @override + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key) + if resolved_api_key is None: + raise ValueError(missing_dashscope_family_key_message("dashscope")) + return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"} + + @override + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + ) -> str: + configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE") + root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE + return f"{root}{text_to_speech_family(model).path}" + + @override + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict[str, object], # mutable-ok: mirrors override contract + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + headers: dict[str, str], # mutable-ok: mirrors override contract + ) -> TextToSpeechRequestData: + voice_field: Final = {"voice": voice} if voice is not None else {} + return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}} + + @override + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response)) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + async def async_transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> HttpxBinaryResponseContent: + audio_url, usage = _audio_url(raw_response) + try: + audio_response: Final = await async_safe_get( + self._async_client(), audio_url, timeout=_request_timeout(raw_response) + ) + except SSRFError as e: + raise _refused_audio_url(e) from e + return _binary_audio(audio_response, usage) + + @override + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract + ) -> BaseLLMException: + return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers)) diff --git a/litellm/main.py b/litellm/main.py index 0162742de18..d469aa629fe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8826,8 +8826,9 @@ def speech( extra_headers: Final = kwargs.get("extra_headers", None) model_info: Final = kwargs.get("model_info", None) shared_session: Final = kwargs.get("shared_session", None) + caller_api_base: Final = api_base or base_url model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url + model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base ) kwargs.pop("tags", []) @@ -8882,13 +8883,13 @@ def speech( custom_llm_provider=custom_llm_provider, ) response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None - if custom_llm_provider == "edenai": - litellm_params_dict["api_base"] = api_base + if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None: + litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base response = base_llm_http_handler.text_to_speech_handler( model=model, input=input, voice=voice if isinstance(voice, str) else None, - text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(), + text_to_speech_provider_config=text_to_speech_provider_config, text_to_speech_optional_params=optional_params, custom_llm_provider=custom_llm_provider, litellm_params=litellm_params_dict, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6bbb63b6fc..50cfc6007ca 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 1d487f33835..069ecfb82b2 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, diff --git a/litellm/utils.py b/litellm/utils.py index e46e8ca1d52..904d154f8df 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10923,19 +10923,8 @@ class ProviderConfigManager: return None @staticmethod - def get_provider_image_edit_config( - model: str, - provider: LlmProviders, - ) -> BaseImageEditConfig | None: - if LlmProviders.OPENAI == provider: - from litellm.llms.openai.image_edit import OpenAIImageEditConfig - - return OpenAIImageEditConfig() - elif LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config - - return get_hosted_vllm_image_edit_config(model=model) - elif LlmProviders.AZURE == provider: + def _get_model_independent_image_edit_config(provider: LlmProviders) -> BaseImageEditConfig | None: + if LlmProviders.AZURE == provider: from litellm.llms.azure.image_edit.transformation import ( AzureImageEditConfig, ) @@ -10953,6 +10942,36 @@ class ProviderConfigManager: ) return BlackForestLabsImageEditConfig() + elif LlmProviders.LITELLM_PROXY == provider: + from litellm.llms.litellm_proxy.image_edit.transformation import ( + LiteLLMProxyImageEditConfig, + ) + + return LiteLLMProxyImageEditConfig() + elif LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.image_edit.transformation import ( + DashScopeImageEditConfig, + ) + + return DashScopeImageEditConfig() + return None + + @staticmethod + def get_provider_image_edit_config( + model: str, + provider: LlmProviders, + ) -> BaseImageEditConfig | None: + model_independent_config: Final = ProviderConfigManager._get_model_independent_image_edit_config(provider) + if model_independent_config is not None: + return model_independent_config + if LlmProviders.OPENAI == provider: + from litellm.llms.openai.image_edit import OpenAIImageEditConfig + + return OpenAIImageEditConfig() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config + + return get_hosted_vllm_image_edit_config(model=model) elif LlmProviders.FAL_AI == provider: from litellm.llms.fal_ai.image_edit import get_fal_ai_image_edit_config @@ -10965,12 +10984,6 @@ class ProviderConfigManager: from litellm.llms.gemini.image_edit import get_gemini_image_edit_config return get_gemini_image_edit_config(model) - elif LlmProviders.LITELLM_PROXY == provider: - from litellm.llms.litellm_proxy.image_edit.transformation import ( - LiteLLMProxyImageEditConfig, - ) - - return LiteLLMProxyImageEditConfig() elif LlmProviders.VERTEX_AI == provider: from litellm.llms.vertex_ai.image_edit import ( get_vertex_ai_image_edit_config, @@ -11192,6 +11205,12 @@ class ProviderConfigManager: return AWSPollyTextToSpeechConfig() elif litellm.LlmProviders.EDENAI == provider: return litellm.EdenAITextToSpeechConfig() + elif litellm.LlmProviders.DASHSCOPE == provider: + from litellm.llms.dashscope.text_to_speech.transformation import ( + DashScopeTextToSpeechConfig, + ) + + return DashScopeTextToSpeechConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6bbb63b6fc..50cfc6007ca 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -18346,25 +18346,31 @@ "dashscope/qwen-image-2.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.035, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-2.0-pro": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.075, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0": { "litellm_provider": "dashscope", "mode": "image_generation", - "source": "https://www.alibabacloud.com/help/en/model-studio/models", + "output_cost_per_image": 0.03, + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" ] }, "dashscope/qwen-image-3.0-pro": { @@ -18372,7 +18378,54 @@ "mode": "image_generation", "source": "https://www.alibabacloud.com/help/en/model-studio/models", "supported_endpoints": [ - "/v1/images/generations" + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "dashscope/qwen3-tts-flash": { + "input_cost_per_character": 1e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-flash": { + "input_cost_per_token": 2.3e-07, + "output_cost_per_token": 1.87e-06, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.1-tts-plus": { + "input_cost_per_character": 3e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-plus": { + "input_cost_per_character": 2e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "dashscope/qwen-audio-3.0-tts-flash": { + "input_cost_per_character": 1.5e-05, + "litellm_provider": "dashscope", + "mode": "audio_speech", + "source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing", + "supported_endpoints": [ + "/v1/audio/speech" ] }, "dashscope/wan3.0-video": { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 1d487f33835..069ecfb82b2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1148,9 +1148,9 @@ "text_completion": true, "embeddings": false, "image_generations": true, - "image_edits": false, + "image_edits": true, "audio_transcriptions": false, - "audio_speech": false, + "audio_speech": true, "rerank": false, "ocr": false, "search": false, diff --git a/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py new file mode 100644 index 00000000000..ea16633d81d --- /dev/null +++ b/tests/integration/providers/test_dashscope_image_edit_and_speech_wire.py @@ -0,0 +1,259 @@ +import base64 +import json +import os +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment, list_value, object_value +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._dashscope_video_support import deployed, json_reply, rate, settled_spend +from pydantic import JsonValue + +_API_KEY: Final = "synthetic-dashscope-key" +_MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +_SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +_OSS_PATH: Final = "/oss/" +_PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +_PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(_PNG).decode()}" +_EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +_PROMPT: Final = "Turn the red circle into a blue square" +_TEXT: Final = "Hello from LiteLLM." +_BILLED_CHARACTERS: Final = len(_TEXT) +_WAV: Final = b"RIFF\x24\x00\x00\x00WAVEfmt " + uuid.uuid4().bytes +_MP3: Final = b"ID3\x04\x00\x00\x00\x00\x00\x00" + uuid.uuid4().bytes +_SIGNED_QUERY: Final = "Expires=1793316671&OSSAccessKeyId=LTAI5t&Signature=ab%2Bc%3D" +_QWEN_TTS: Final = "dashscope-qwen-tts" +_QWEN_AUDIO_TTS: Final = "dashscope-qwen-audio-tts" +_INPUT_TOKENS: Final = 8 +_OUTPUT_TOKENS: Final = 30 +_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) + + +def _edit_api(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", _MULTIMODAL_PATH), (request.method, request.target) + return json_reply( + { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": _EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, + } + ) + + +def _synthesized(request: Request, file_name: str, usage: dict[str, JsonValue]) -> Reply: + audio_url: Final = f"http://{request.headers['host']}{_OSS_PATH}{file_name}?{_SIGNED_QUERY}" + return json_reply({"output": {"audio": {"url": audio_url, "data": "", "id": "audio-1"}}, "usage": usage}) + + +def _speech_api(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.wav"): + return Reply(body=_WAV, content_type="audio/wav") + if request.method == "GET" and request.target.startswith(f"{_OSS_PATH}speech.mp3"): + return Reply(body=_MP3, content_type="audio/mpeg") + if (request.method, request.target) == ("POST", _MULTIMODAL_PATH): + return _synthesized(request, "speech.wav", {"characters": _BILLED_CHARACTERS}) + assert (request.method, request.target) == ("POST", _SYNTHESIZER_PATH), (request.method, request.target) + return _synthesized( + request, + "speech.mp3", + {"characters": _BILLED_CHARACTERS, "input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, + ) + + +def _pooled_database_url(url: str) -> str: + parts: Final = urlsplit(url) + query: Final = "&".join(part for part in (parts.query, "connection_limit=5") if part) + return urlunsplit(parts._replace(query=query)) + + +def _tts_deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": _API_KEY}} + + +def _error_message(body: bytes) -> str: + return str(object_value(JSON_OBJECT.validate_json(body)["error"])["message"]) + + +@pytest.fixture(scope="module") +def rig() -> Iterator[Gateway]: + with gateway_from_environment() as gateway: + yield gateway + + +@pytest.fixture(scope="module") +def speech_wire() -> Iterator[Wire]: + with wire_server(_speech_api) as served: + yield served + + +@pytest.fixture(scope="module") +def speech_proxy(rig: Gateway, speech_wire: Wire, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("dashscope-speech") + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + path: Final = directory / "dashscope-speech.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + _tts_deployment(_QWEN_TTS, "dashscope/qwen3-tts-flash", speech_wire.url), + _tts_deployment(_QWEN_AUDIO_TTS, "dashscope/qwen-audio-3.1-tts-flash", speech_wire.url), + ], + "general_settings": { + **object_value(config["general_settings"]), + "user_url_allowed_hosts": ["127.0.0.1"], + }, + } + ) + ) + with owned_proxy( + rig, directory, {"DATABASE_URL": _pooled_database_url(os.environ["DATABASE_URL"])}, config=path + ) as owned: + yield owned + + +def test_a_multipart_edit_reaches_dashscope_as_inline_images_and_bills_the_edited_image(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + key: Final = scenario.key(models=[model]) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT, "size": "1024x1024", "negative_prompt": "blurry"}, + files=[("image[]", ("a.png", _PNG, "image/png")), ("image[]", ("b.png", _PNG, "image/png"))], + headers={"Authorization": f"Bearer {key}"}, + ) + + assert response.status_code == 200, response.text + data: Final = list_value(JSON_OBJECT.validate_json(response.content)["data"]) + assert [object_value(item)["url"] for item in data] == [_EDITED_URL] + (request,) = wire.drain() + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(request.body) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [{"image": _PNG_DATA_URI}, {"image": _PNG_DATA_URI}, {"text": _PROMPT}], + } + ] + }, + "parameters": {"size": "1024*1024", "negative_prompt": "blurry"}, + } + settled_spend(key, rate("dashscope", "output_cost_per_image", model="qwen-image-2.0")) + + +def test_a_fourth_image_is_refused_with_a_400_before_dashscope_is_called(gateway: Gateway) -> None: + with wire_server(_edit_api) as wire, gateway.scenario() as scenario: + model: Final = deployed( + gateway, + scenario, + model="dashscope/qwen-image-2.0", + api_base=f"{wire.url}{_MULTIMODAL_PATH}", + api_key=_API_KEY, + ) + response: Final = gateway.client.post( + "/v1/images/edits", + data={"model": model, "prompt": _PROMPT}, + files=[("image[]", (f"{index}.png", _PNG, "image/png")) for index in range(4)], + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + + assert response.status_code == 400, response.text + assert "at most 3 input images, got 4" in _error_message(response.content) + assert wire.drain() == () + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen3_tts_returns_the_downloaded_wav_and_bills_each_input_character( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_TTS]) + response: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry"}, key=key + ) + + assert response.status_code == 200, response.text + assert response.content == _WAV + assert response.headers["content-type"] == "audio/wav" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _MULTIMODAL_PATH) + assert synthesis.headers["authorization"] == f"Bearer {_API_KEY}" + assert json.loads(synthesis.body) == {"model": "qwen3-tts-flash", "input": {"text": _TEXT, "voice": "Cherry"}} + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.wav?{_SIGNED_QUERY}") + assert "authorization" not in download.headers + settled_spend(key, _BILLED_CHARACTERS * rate("dashscope", "input_cost_per_character", "qwen3-tts-flash")) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_qwen_audio_tts_sends_format_and_instruction_returns_the_mp3_and_bills_reported_tokens( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + model: Final = "qwen-audio-3.1-tts-flash" + with speech_proxy.scenario() as scenario: + key: Final = scenario.key(models=[_QWEN_AUDIO_TTS]) + response: Final = speech_proxy.request( + "POST", + "/v1/audio/speech", + { + "model": _QWEN_AUDIO_TTS, + "input": _TEXT, + "voice": "longanhuan_v3.1", + "response_format": "mp3", + "instructions": "Speak slowly.", + }, + key=key, + ) + + assert response.status_code == 200, response.text + assert response.content == _MP3 + assert response.headers["content-type"] == "audio/mpeg" + synthesis, download = speech_wire.drain() + assert (synthesis.method, synthesis.target) == ("POST", _SYNTHESIZER_PATH) + assert json.loads(synthesis.body) == { + "model": model, + "input": {"text": _TEXT, "voice": "longanhuan_v3.1", "format": "mp3", "instruction": "Speak slowly."}, + } + assert (download.method, download.target) == ("GET", f"{_OSS_PATH}speech.mp3?{_SIGNED_QUERY}") + settled_spend( + key, + _INPUT_TOKENS * rate("dashscope", "input_cost_per_token", model) + + _OUTPUT_TOKENS * rate("dashscope", "output_cost_per_token", model), + ) + + +@pytest.mark.timeout(_OWNED_TIMEOUT) +def test_a_format_qwen3_tts_cannot_produce_answers_400_without_reaching_dashscope( + speech_proxy: Gateway, speech_wire: Wire +) -> None: + speech_wire.drain() + refused: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "mp3"} + ) + served: Final = speech_proxy.request( + "POST", "/v1/audio/speech", {"model": _QWEN_TTS, "input": _TEXT, "voice": "Cherry", "response_format": "wav"} + ) + + assert refused.status_code == 400, refused.text + assert "response_format='mp3'" in _error_message(refused.content) + assert served.status_code == 200, served.text + assert [(request.method, request.target.split("?")[0]) for request in speech_wire.drain()] == [ + ("POST", _MULTIMODAL_PATH), + ("GET", f"{_OSS_PATH}speech.wav"), + ] diff --git a/tests/unit/images/test_image_edit.py b/tests/unit/images/test_image_edit.py index 4f858050e3e..06372a29d92 100644 --- a/tests/unit/images/test_image_edit.py +++ b/tests/unit/images/test_image_edit.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict, override import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import FileTypes, ImageResponse _PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" @@ -34,6 +35,26 @@ _EDIT_RESPONSE: Final = { }, } +_DASHSCOPE_IMAGE_URL: Final = ( + "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +) +_DASHSCOPE_EDIT_RESPONSE: Final = { + "output": { + "choices": [ + { + "finish_reason": "stop", + "message": {"role": "assistant", "content": [{"image": "https://example.com/edited.png"}]}, + } + ] + }, + "usage": {"image_count": 1}, +} + +_FAL_EDIT_URL: Final = "https://fal.run/fal-ai/flux-2/edit" +_FAL_EDIT_RESPONSE: Final = { + "images": [{"url": "https://example.com/edited.png", "width": 1024, "height": 768, "content_type": "image/png"}] +} + class _LoggedImageEdit(TypedDict): model: ReadOnly[str] @@ -229,3 +250,101 @@ async def test_router_image_edit_returns_the_provider_image( assert isinstance(result, ImageResponse) assert result.data is not None and result.data[0].b64_json == _EDITED_IMAGE_B64 assert route.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deployment_price", "size", "expected_cost"), + ( + ({"input_cost_per_image": 0.0123}, None, 0.0123), + ({"input_cost_per_pixel": 1e-08}, "2048x2048", 2048 * 2048 * 1e-08), + ), +) +async def test_an_edit_bills_the_deployment_price_at_its_size_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, + deployment_price: dict[str, float], + size: str | None, + expected_cost: float, +) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + respx_mock.post(_DASHSCOPE_IMAGE_URL).mock(return_value=httpx.Response(200, json=_DASHSCOPE_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="qwen-image-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="edit-call", + function_id="edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qwen-image-edit", + "litellm_params": { + "model": "dashscope/qwen-image-2.0", + "api_key": "fake-key", + **deployment_price, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="qwen-image-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + size=size, + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_a_fal_edit_bills_the_deployment_price_per_image_on_a_logging_object_made_before_routing( + respx_mock: respx.MockRouter, + httpx_transport: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("FAL_AI_API_BASE", raising=False) + respx_mock.post(_FAL_EDIT_URL).mock(return_value=httpx.Response(200, json=_FAL_EDIT_RESPONSE)) + logging_made_before_routing: Final = Logging( + model="flux-edit", + messages=[], + stream=False, + call_type="aimage_edit", + start_time=datetime.now(), + litellm_call_id="fal-edit-call", + function_id="fal-edit-call", + ) + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "flux-edit", + "litellm_params": { + "model": "fal_ai/fal-ai/flux-2/edit", + "api_key": "fake-key", + "output_cost_per_image": 0.0456, + }, + } + ] + ) + + result: Final = await router.aimage_edit( + model="flux-edit", + image=_FIRST_IMAGE, + prompt="Turn the circle into a square", + litellm_logging_obj=logging_made_before_routing, + litellm_call_id="fal-edit-call", + ) + + assert isinstance(result, ImageResponse) + assert result._hidden_params["response_cost"] == pytest.approx(0.0456) diff --git a/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py new file mode 100644 index 00000000000..c3d186fdd8a --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_image_edit_transformation.py @@ -0,0 +1,358 @@ +import base64 +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.dashscope.image_edit.transformation import DashScopeImageEditConfig +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.utils import ModelInfo + +PNG: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URI: Final = f"data:image/png;base64,{base64.b64encode(PNG).decode()}" +INTL_ENDPOINT: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" +EDITED_URL: Final = "https://dashscope-result.example.com/edited.png" +EDITED: Final = { + "output": {"choices": [{"message": {"role": "assistant", "content": [{"image": EDITED_URL}]}}]}, + "usage": {"width": 1024, "height": 1024, "image_count": 1}, +} + + +def _recording( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] +) -> httpx.MockTransport: # mutable-ok: test records the requests the transport receives + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _edited(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=EDITED) + + +def _sync_client( + seen: list[httpx.Request], reply: Callable[[httpx.Request], httpx.Response] = _edited +) -> HTTPHandler: # mutable-ok: shared request log + return HTTPHandler(client=httpx.Client(transport=_recording(seen, reply))) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE_IMAGE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize("asynchronous", (False, True)) +async def test_an_edit_posts_every_image_inline_with_provider_params_at_the_top_of_parameters( + asynchronous: bool, +) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + arguments: Final = { + "model": "dashscope/qwen-image-2.0", + "image": [PNG, PNG], + "prompt": "Turn the circle into a square", + "size": "1024x1024", + "n": 1, + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + "api_key": "sk-dashscope-test", + } + response: Final = ( + await litellm.aimage_edit(**arguments, client=AsyncHTTPHandler(transport=_recording(seen, _edited))) + if asynchronous + else litellm.image_edit(**arguments, client=_sync_client(seen)) + ) + + assert [image.url for image in response.data or []] == [EDITED_URL] + (request,) = seen + assert str(request.url) == INTL_ENDPOINT + assert request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(request.content) == { + "model": "qwen-image-2.0", + "input": { + "messages": [ + { + "role": "user", + "content": [ + {"image": PNG_DATA_URI}, + {"image": PNG_DATA_URI}, + {"text": "Turn the circle into a square"}, + ], + } + ] + }, + "parameters": { + "n": 1, + "size": "1024*1024", + "negative_prompt": "blurry", + "prompt_extend": False, + "watermark": False, + "seed": 42, + }, + } + + +@pytest.mark.parametrize(("size", "dashscope_size"), (("1792x1024", "1792*1024"), ("1328x1328", "1328*1328"))) +def test_an_openai_size_reaches_dashscope_in_its_star_format(size: str, dashscope_size: str) -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", size=size, api_key="k", client=_sync_client(seen) + ) + + assert json.loads(seen[0].content)["parameters"] == {"size": dashscope_size} + + +CHINA_ENDPOINT: Final = "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + + +@pytest.mark.parametrize( + ("api_base", "env_base", "image_base", "endpoint"), + ( + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/compatible-mode/v1/", None, None, INTL_ENDPOINT), + ("https://dashscope.aliyuncs.com", None, None, CHINA_ENDPOINT), + ("https://dashscope-intl.aliyuncs.com/apps/anthropic", None, None, INTL_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/apps/anthropic", None, CHINA_ENDPOINT), + ( + "https://gw.example.com/compatible-mode/v1", + None, + None, + "https://gw.example.com/api/v1/services/aigc/multimodal-generation/generation", + ), + (None, "https://dashscope.aliyuncs.com/compatible-mode/v1", None, CHINA_ENDPOINT), + (None, "https://dashscope.aliyuncs.com/api/v1", None, CHINA_ENDPOINT), + (None, None, None, INTL_ENDPOINT), + ( + "https://images.example.com/edit", + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "https://images.example.com/edit", + ), + ("https://dashscope.aliyuncs.com/compatible-mode/v1", None, "https://images.example.com/edit", CHINA_ENDPOINT), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "https://images.example.com/edit", + "https://images.example.com/edit", + ), + ), +) +def test_an_edit_goes_to_the_image_endpoint_on_the_host_its_dashscope_base_names( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, env_base: str | None, image_base: str | None, endpoint: str +) -> None: + if env_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", env_base) + if image_base is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE_IMAGE", image_base) + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + api_base=api_base, + client=_sync_client(seen), + ) + + assert str(seen[0].url) == endpoint + + +@pytest.mark.parametrize("api_base", ("https://dashscope.aliyuncs.com/compatible-mode/v1", None)) +def test_an_edit_and_a_speech_call_with_the_same_dashscope_base_reach_the_same_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None +) -> None: + monkeypatch.setenv("DASHSCOPE_API_BASE", "https://gw.example.com/compatible-mode/v1") + edit_url: Final = DashScopeImageEditConfig().get_complete_url( + model="qwen-image-2.0", api_base=api_base, litellm_params={} + ) + speech_url: Final = DashScopeTextToSpeechConfig().get_complete_url( + model="qwen3-tts-flash", api_base=api_base, litellm_params={} + ) + + assert httpx.URL(edit_url).host == httpx.URL(speech_url).host + + +def test_a_fourth_image_is_refused_with_a_400_before_any_request_even_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + with pytest.raises(litellm.BadRequestError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=[PNG, PNG, PNG, PNG], + prompt="p", + api_key="k", + drop_params=True, + client=_sync_client(seen), + ) + + assert refused.value.status_code == 400 + assert "at most 3 input images, got 4" in str(refused.value) + assert seen == [] + + +def test_three_images_are_accepted() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=[PNG, PNG, PNG], prompt="p", api_key="k", client=_sync_client(seen) + ) + + content: Final = json.loads(seen[0].content)["input"]["messages"][0]["content"] + assert content == [{"image": PNG_DATA_URI}] * 3 + [{"text": "p"}] + + +def test_an_unsupported_openai_param_answers_400_and_is_dropped_under_drop_params() -> None: + seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + client: Final = _sync_client(seen) + + with pytest.raises(litellm.UnsupportedParamsError) as refused: + litellm.image_edit( + model="dashscope/qwen-image-2.0", image=PNG, prompt="p", quality="high", api_key="k", client=client + ) + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + quality="high", + api_key="k", + drop_params=True, + client=client, + ) + + assert refused.value.status_code == 400 + assert "quality" in str(refused.value) + (request,) = seen + assert json.loads(request.content)["parameters"] == {} + + +@pytest.mark.parametrize( + ("reply", "error", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "Image is too small"}), + litellm.APIError, + "Image is too small", + ), + ( + httpx.Response(400, json={"code": "InvalidParameter", "message": "Model not exist."}), + litellm.BadRequestError, + "Model not exist.", + ), + ), +) +def test_a_dashscope_rejection_reaches_the_caller_with_its_message( + reply: httpx.Response, error: type[Exception], message: str +) -> None: + with pytest.raises(error, match=message): + litellm.image_edit( + model="dashscope/qwen-image-2.0", + image=PNG, + prompt="p", + api_key="k", + client=_sync_client([], lambda _: reply), + ) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_each_edited_image_is_billed_at_the_model_output_image_price(model: str) -> None: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}, {"url": EDITED_URL}]) + + cost: Final = litellm.completion_cost( + completion_response=response, + model=f"dashscope/{model}", + call_type="aimage_edit", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(2 * litellm.model_cost[f"dashscope/{model}"]["output_cost_per_image"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_the_input_images_dashscope_reports_are_billed_on_top_of_the_edited_images() -> None: + def reply(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={**EDITED, "usage": {"input_image_count": 2, "output_image_count": 1}}) + + response: Final = litellm.image_edit( + model="dashscope/qwen-image-3.0", image=[PNG, PNG], prompt="p", api_key="k", client=_sync_client([], reply) + ) + + assert isinstance(response, litellm.ImageResponse) + assert CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=response, + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) == pytest.approx(2 * 0.004 + 0.04) + + +def _edited_images(count: int, input_images: int) -> litellm.ImageResponse: + response: Final = litellm.ImageResponse(data=[{"url": EDITED_URL}] * count) + response.hidden_params = {"billable_input_image_count": input_images} + return response + + +@pytest.mark.parametrize( + ("model", "deployment_price", "size", "expected_cost"), + ( + ("qwen-image-2.0", ModelInfo(input_cost_per_image=0.02), None, 2 * 0.02), + ("qwen-image-3.0-pro", ModelInfo(input_cost_per_pixel=1e-08), "1024x1024", 2 * 1024 * 1024 * 1e-08), + ), +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_image_price_bills_the_edit_the_way_it_bills_any_image_call( + model: str, deployment_price: ModelInfo, size: str | None, expected_cost: float +) -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + size=size, + call_type="aimage_edit", + model_info=deployment_price, + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_that_copies_both_image_prices_bills_edited_and_input_images_at_its_own_prices() -> None: + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=2, input_images=3), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_image=0.004, output_cost_per_image=0.04), + ) + + assert cost == pytest.approx(2 * 0.04 + 3 * 0.004) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_deployment_token_price_keeps_the_shared_per_image_billing() -> None: + prices: Final = litellm.model_cost["dashscope/qwen-image-3.0"] + + cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model="qwen-image-3.0", + completion_response=_edited_images(count=1, input_images=2), + custom_llm_provider="dashscope", + call_type="aimage_edit", + model_info=ModelInfo(input_cost_per_token=1e-06), + ) + + assert cost == pytest.approx(2 * prices.get("input_cost_per_image", 0.0) + prices["output_cost_per_image"]) diff --git a/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py new file mode 100644 index 00000000000..d47c66be3cd --- /dev/null +++ b/tests/unit/llms/dashscope/test_dashscope_text_to_speech_transformation.py @@ -0,0 +1,415 @@ +import datetime +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.dashscope.common_utils import DashScopeError +from litellm.llms.dashscope.text_to_speech.transformation import DashScopeTextToSpeechConfig +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import Usage + +INTL: Final = "https://dashscope-intl.aliyuncs.com" +MULTIMODAL_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation" +SYNTHESIZER_PATH: Final = "/api/v1/services/audio/tts/SpeechSynthesizer" +AUDIO_URL: Final = ( + "http://93.184.216.34/prod/qwen-tts/a1.mp3?Expires=1793316671&OSSAccessKeyId=LTAI&Signature=ab%2Bc%3D" +) +AUDIO: Final = b"ID3\x04\x00synthetic-mp3-frames" +SYNTHESIZED: Final = { + "output": {"audio": {"url": AUDIO_URL, "data": "", "id": "audio_1"}}, + "usage": {"characters": 6, "input_tokens": 8, "output_tokens": 30, "total_tokens": 38}, +} + +API_REQUEST: Final = httpx.Request("POST", f"{INTL}{MULTIMODAL_PATH}") + +Reply = Callable[[httpx.Request], httpx.Response] + + +def _api(reply: dict[str, object]) -> Reply: # mutable-ok: JSON body fixture + return lambda _: httpx.Response(200, json=reply) + + +def _audio(_: httpx.Request) -> httpx.Response: + return httpx.Response(200, content=AUDIO, headers={"content-type": "audio/mpeg"}) + + +def _recording(seen: list[httpx.Request], reply: Reply) -> httpx.MockTransport: # mutable-ok: request log + def respond(request: httpx.Request) -> httpx.Response: + seen.append(request) + return reply(request) + + return httpx.MockTransport(respond) + + +def _refusing_blocking_download(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"blocking client used for {request.url} on the event loop") + + +def _logging() -> Logging: + return Logging( + model="dashscope-tts", + messages=[], + stream=False, + call_type="aspeech", + start_time=datetime.datetime(2026, 10, 28, tzinfo=datetime.timezone.utc), + litellm_call_id="call-tts", + function_id="fn-tts", + ) + + +@pytest.fixture(autouse=True) +def _no_ambient_dashscope_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + + +@pytest.mark.parametrize( + ("model", "optional_params", "path", "provider_input"), + ( + ( + "qwen3-tts-flash", + {"response_format": "wav"}, + MULTIMODAL_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ( + "qwen-audio-3.1-tts-flash", + {"response_format": "mp3", "instructions": "Speak slowly."}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry", "format": "mp3", "instruction": "Speak slowly."}, + ), + ( + "qwen-audio-3.0-tts-plus", + {}, + SYNTHESIZER_PATH, + {"text": "Hello.", "voice": "Cherry"}, + ), + ), +) +async def test_each_family_posts_to_its_endpoint_and_the_async_handler_returns_the_downloaded_audio( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + path: str, + provider_input: dict[str, object], # mutable-ok: parametrized fixture +) -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler( + client=httpx.Client(transport=httpx.MockTransport(_refusing_blocking_download)) + ), + async_client=lambda: AsyncHTTPHandler(transport=_recording(download_seen, _audio)), + ) + voice, mapped = config.map_openai_params(model=model, optional_params=optional_params, voice="Cherry") + + speech: Final = await BaseLLMHTTPHandler().async_text_to_speech_handler( + model=model, + input="Hello.", + voice=voice, + text_to_speech_provider_config=config, + text_to_speech_optional_params=mapped, + custom_llm_provider="dashscope", + litellm_params={"api_key": "sk-dashscope-test", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(SYNTHESIZED))), + ) + + assert speech.content == AUDIO + (api_request,) = api_seen + assert str(api_request.url) == f"{INTL}{path}" + assert api_request.headers["authorization"] == "Bearer sk-dashscope-test" + assert json.loads(api_request.content) == {"model": model, "input": provider_input} + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert "authorization" not in download.headers + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +def test_the_sync_handler_downloads_through_the_injected_blocking_client_within_the_call_timeout() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler(client=httpx.Client(transport=_recording(api_seen, _api(SYNTHESIZED)))), + ) + + assert isinstance(speech, HttpxBinaryResponseContent) + assert speech.content == AUDIO + (api_request,) = api_seen + (download,) = download_seen + assert str(download.url) == AUDIO_URL + assert download.extensions["timeout"] == api_request.extensions["timeout"] + + +@pytest.mark.parametrize( + ("audio_url", "download_url"), + ( + ( + "http://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav?Expires=1&Signature=ab%2Bc%3D", + ), + ( + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + "https://dashscope-result-sh.oss-cn-shanghai.aliyuncs.com/a1.wav", + ), + ("http://audio.notaliyuncs.com/a1.wav", "http://audio.notaliyuncs.com/a1.wav"), + (AUDIO_URL, AUDIO_URL), + ), +) +def test_an_alibaba_cloud_audio_url_is_downloaded_over_https( + monkeypatch: pytest.MonkeyPatch, audio_url: str, download_url: str +) -> None: + monkeypatch.setattr(litellm, "user_url_validation", False) + download_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + config: Final = DashScopeTextToSpeechConfig( + sync_client=lambda: HTTPHandler(client=httpx.Client(transport=_recording(download_seen, _audio))) + ) + + speech: Final = BaseLLMHTTPHandler().text_to_speech_handler( + model="qwen3-tts-flash", + input="Hello.", + voice="Cherry", + text_to_speech_provider_config=config, + text_to_speech_optional_params={}, + custom_llm_provider="dashscope", + litellm_params={"api_key": "k", "api_base": None}, + logging_obj=_logging(), + timeout=30, + client=HTTPHandler( + client=httpx.Client( + transport=_recording([], _api({**SYNTHESIZED, "output": {"audio": {"url": audio_url}}})) + ) + ), + ) + + assert speech.content == AUDIO + (download,) = download_seen + assert str(download.url) == download_url + + +@pytest.mark.parametrize( + ("api_base", "environment", "model", "url"), + ( + (None, None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + (None, None, "qwen-audio-3.1-tts-plus", f"{INTL}{SYNTHESIZER_PATH}"), + ( + "https://dashscope.aliyuncs.com/compatible-mode/v1", + None, + "qwen-audio-3.1-tts-plus", + f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}", + ), + (f"{INTL}{SYNTHESIZER_PATH}", None, "qwen3-tts-flash", f"{INTL}{MULTIMODAL_PATH}"), + ( + "https://dashscope.aliyuncs.com/apps/anthropic", + None, + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ( + "https://relay.example.com/dashscope/", + None, + "qwen3-tts-flash", + f"https://relay.example.com/dashscope{MULTIMODAL_PATH}", + ), + ( + None, + "https://dashscope.aliyuncs.com/compatible-mode/v1", + "qwen3-tts-flash", + f"https://dashscope.aliyuncs.com{MULTIMODAL_PATH}", + ), + ), +) +def test_the_endpoint_follows_the_model_family_on_the_configured_host( + monkeypatch: pytest.MonkeyPatch, api_base: str | None, environment: str | None, model: str, url: str +) -> None: + if environment is not None: + monkeypatch.setenv("DASHSCOPE_API_BASE", environment) + + assert DashScopeTextToSpeechConfig().get_complete_url(model=model, api_base=api_base, litellm_params={}) == url + + +@pytest.mark.parametrize( + ("model", "optional_params", "voice", "refused"), + ( + ("qwen3-tts-flash", {"response_format": "mp3"}, "Cherry", "response_format='mp3'"), + ("qwen3-tts-flash", {"instructions": "Whisper."}, "Cherry", "instructions='Whisper.'"), + ("qwen-audio-3.1-tts-flash", {"response_format": "aac"}, "Cherry", "response_format='aac'"), + ("qwen-audio-3.1-tts-flash", {"speed": 1.5}, "Cherry", "speed=1.5"), + ("qwen-audio-3.1-tts-flash", {}, {"voice_id": "Cherry"}, "voice as an object"), + ), +) +def test_an_unsupported_param_answers_400_and_is_dropped_under_drop_params( + model: str, + optional_params: dict[str, object], # mutable-ok: parametrized fixture + voice: str | dict[str, object], # mutable-ok: parametrized fixture + refused: str, +) -> None: + config: Final = DashScopeTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError) as rejected: + config.map_openai_params(model=model, optional_params=optional_params, voice=voice) + dropped_voice, dropped = config.map_openai_params( + model=model, optional_params=optional_params, voice=voice, kwargs={"drop_params": True} + ) + + assert rejected.value.status_code == 400 + assert refused in str(rejected.value) + assert dropped == {} + assert dropped_voice == (voice if isinstance(voice, str) else None) + + +async def test_aspeech_uses_the_native_endpoint_on_the_caller_api_base_and_refuses_a_private_audio_url() -> None: + api_seen: Final[list[httpx.Request]] = [] # mutable-ok: request log + private_audio: Final = {"output": {"audio": {"url": "http://169.254.169.254/latest/meta-data"}}} + + with pytest.raises(litellm.BadGatewayError) as refused: + await litellm.aspeech( + model="dashscope/qwen-audio-3.1-tts-flash", + input="Hello.", + voice="longanhuan_v3.1", + response_format="opus", + api_key="sk-dashscope-test", + api_base="https://dashscope.aliyuncs.com/compatible-mode/v1", + client=AsyncHTTPHandler(transport=_recording(api_seen, _api(private_audio))), + ) + + assert refused.value.status_code == 502 + assert "audio URL that was refused" in str(refused.value) + (api_request,) = api_seen + assert str(api_request.url) == f"https://dashscope.aliyuncs.com{SYNTHESIZER_PATH}" + assert json.loads(api_request.content) == { + "model": "qwen-audio-3.1-tts-flash", + "input": {"text": "Hello.", "voice": "longanhuan_v3.1", "format": "opus"}, + } + + +@pytest.mark.parametrize( + ("api_reply", "audio_reply", "status", "message"), + ( + ( + httpx.Response(200, json={"code": "InvalidParameter", "message": "The voice property is required."}), + _audio, + 400, + "The voice property is required.", + ), + ( + httpx.Response(200, content=b"gateway"), + _audio, + 502, + "Unexpected DashScope text-to-speech response", + ), + ( + httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + lambda _: httpx.Response(403), + 502, + "audio download failed with HTTP 403", + ), + ), +) +async def test_a_failed_synthesis_or_download_raises_a_dashscope_error( + api_reply: httpx.Response, audio_reply: Reply, status: int, message: str +) -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(audio_reply)) + ) + + with pytest.raises(DashScopeError) as failed: + await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", raw_response=api_reply, logging_obj=_logging() + ) + + assert failed.value.status_code == status + assert message in failed.value.message + + +async def test_provider_token_usage_rides_on_the_audio_so_token_priced_speech_is_billed() -> None: + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen-audio-3.1-tts-flash", + raw_response=httpx.Response(200, json=SYNTHESIZED, request=API_REQUEST), + logging_obj=_logging(), + ) + + assert speech.usage == Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + + +@pytest.mark.usefixtures("local_model_cost_map") +async def test_character_priced_speech_bills_the_characters_dashscope_reports() -> None: + text: Final = "你好 世界。" + reported: Final = {**SYNTHESIZED, "usage": {"characters": 10}} + config: Final = DashScopeTextToSpeechConfig( + async_client=lambda: AsyncHTTPHandler(transport=httpx.MockTransport(_audio)) + ) + speech: Final = await config.async_transform_text_to_speech_response( + model="qwen3-tts-flash", + raw_response=httpx.Response(200, json=reported, request=API_REQUEST), + logging_obj=_logging(), + ) + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen3-tts-flash", + call_type="aspeech", + prompt=text, + custom_llm_provider="dashscope", + ) + + assert litellm.utils.count_characters(text) != 10 + assert cost == pytest.approx(10 * litellm.model_cost["dashscope/qwen3-tts-flash"]["input_cost_per_character"]) + + +@pytest.mark.parametrize( + "model", ("qwen3-tts-flash", "qwen-audio-3.1-tts-plus", "qwen-audio-3.0-tts-plus", "qwen-audio-3.0-tts-flash") +) +@pytest.mark.usefixtures("local_model_cost_map") +def test_character_priced_speech_bills_each_input_character_at_the_model_rate(model: str) -> None: + cost: Final = litellm.completion_cost( + completion_response=HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)), + model=f"dashscope/{model}", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(6 * litellm.model_cost[f"dashscope/{model}"]["input_cost_per_character"]) + assert cost > 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_token_priced_speech_bills_the_provider_reported_input_and_output_tokens() -> None: + speech: Final = HttpxBinaryResponseContent(httpx.Response(200, content=AUDIO)) + speech.usage = Usage(prompt_tokens=8, completion_tokens=30, total_tokens=38) + row: Final = litellm.model_cost["dashscope/qwen-audio-3.1-tts-flash"] + + cost: Final = litellm.completion_cost( + completion_response=speech, + model="dashscope/qwen-audio-3.1-tts-flash", + call_type="aspeech", + prompt="Hello.", + custom_llm_provider="dashscope", + ) + + assert cost == pytest.approx(8 * row["input_cost_per_token"] + 30 * row["output_cost_per_token"]) + assert row["output_cost_per_token"] > row["input_cost_per_token"] > 0 diff --git a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py index 2e4d68a02da..981ac3e4855 100644 --- a/tests/unit/llms/runwayml/test_text_to_speech_transformation.py +++ b/tests/unit/llms/runwayml/test_text_to_speech_transformation.py @@ -2,13 +2,51 @@ Test RunwayML text-to-speech transformation """ +from collections.abc import Iterator +from typing import Final +import httpx +import pytest +import respx +import litellm from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, ) +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_async_speech_polls_the_runwayml_task_with_the_caller_key_and_returns_its_audio( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + respx_mock.post("https://api.dev.runwayml.com/v1/text_to_speech").mock( + return_value=httpx.Response(200, json={"id": "task_1", "status": "PENDING"}) + ) + poll: Final = respx_mock.get("https://api.dev.runwayml.com/v1/tasks/task_1").mock( + return_value=httpx.Response( + 200, json={"id": "task_1", "status": "SUCCEEDED", "output": ["https://storage.example.com/a1.mp3"]} + ) + ) + respx_mock.get("https://storage.example.com/a1.mp3").mock( + return_value=httpx.Response(200, content=b"ID3runway-audio") + ) + + speech: Final = await litellm.aspeech( + model="runwayml/eleven_multilingual_v2", input="Hello.", voice="alloy", api_key="rw-key" + ) + + assert speech.content == b"ID3runway-audio" + assert poll.calls.last.request.headers["authorization"] == "Bearer rw-key" + + def test_openai_voice_mapping_to_runwayml(): """ Test that OpenAI voice names are correctly mapped to RunwayML preset IDs diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py index 43a74e56028..e35e21c67f9 100644 --- a/tests/unit/llms/test_get_complete_url_trailing_slash.py +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -137,6 +137,7 @@ API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( "brave.search.transformation.BraveSearchConfig", "cohere.chat.v2_transformation.CohereV2ChatConfig", "cohere.embed.transformation.CohereEmbeddingConfig", + "dashscope.image_edit.transformation.DashScopeImageEditConfig", "dashscope.image_generation.transformation.DashScopeImageGenerationConfig", "dataforseo.search.transformation.DataForSEOSearchConfig", "datarobot.chat.transformation.DataRobotConfig", diff --git a/tests/unit/test_dashscope_image_generation.py b/tests/unit/test_dashscope_image_generation.py index 9c65399350c..0f35136bac3 100644 --- a/tests/unit/test_dashscope_image_generation.py +++ b/tests/unit/test_dashscope_image_generation.py @@ -14,6 +14,7 @@ import respx from pydantic import ValidationError import litellm +from litellm.cost_calculator import default_image_cost_calculator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.dashscope.common_utils import ( DASHSCOPE_DEFAULT_ORIGIN, @@ -23,7 +24,8 @@ from litellm.llms.dashscope.image_generation.transformation import ( DEFAULT_API_BASE, DashScopeImageGenerationConfig, ) -from litellm.types.utils import ImageResponse +from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils +from litellm.types.utils import ImageResponse, ModelInfo from litellm.utils import get_llm_provider # --------------------------------------------------------------------------- @@ -614,6 +616,16 @@ def test_transform_response_without_image_content_has_no_images( assert _transform_response(payload).data == [] +@pytest.mark.parametrize("usage", [None, "n/a", ["input_image_count", 2]]) +def test_transform_response_returns_the_images_whatever_shape_usage_has(usage: object): + response = _transform_response( + {"output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, "usage": usage} + ) + + assert [image.url for image in response.data] == ["https://a.example/1.png"] + assert "billable_input_image_count" not in response.hidden_params + + def test_transform_response_skips_content_items_without_an_image(): response = _transform_response( { @@ -672,3 +684,50 @@ def test_transform_response_rejects_malformed_payloads_without_echoing_them( _transform_response(payload) assert "input_value" not in str(exc_info.value) + + +@pytest.mark.parametrize( + ("model", "usage"), + ( + ("qwen-image-2.0", {"width": 2048, "height": 1536, "image_count": 1}), + ( + "qwen-image-3.0", + {"output_width": 2048, "output_height": 1536, "output_image_count": 1, "input_image_count": 0}, + ), + ), +) +def test_a_per_pixel_deployment_bills_the_size_dashscope_rendered_when_the_caller_sent_none( + model: str, usage: dict[str, int] +): + response = _transform_response( + { + "output": {"choices": [{"message": {"content": [{"image": "https://a.example/1.png"}]}}]}, + "usage": usage, + } + ) + + cost = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=response, + custom_llm_provider="dashscope", + call_type="image_generation", + model_info=ModelInfo(input_cost_per_pixel=1e-08), + ) + + assert cost == pytest.approx(2048 * 1536 * 1e-08) + + +@pytest.mark.parametrize("model", ("qwen-image-2.0", "qwen-image-2.0-pro", "qwen-image-3.0")) +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_shared_row_bills_a_generated_image_the_same_on_the_default_calculator_released_proxies_use(model: str): + dashscope_cost: Final = CostCalculatorUtils.route_image_generation_cost_calculator( + model=model, + completion_response=ImageResponse(data=[{"url": "https://a.example/1.png"}]), + custom_llm_provider="dashscope", + call_type="image_generation", + ) + + assert dashscope_cost > 0 + assert default_image_cost_calculator(model=f"dashscope/{model}", custom_llm_provider="dashscope") == pytest.approx( + dashscope_cost + )