From cf8f5487922cbb8b8878f838a708c4f1f8032b43 Mon Sep 17 00:00:00 2001 From: jayson wang Date: Tue, 8 Sep 2026 11:27:12 +0800 Subject: [PATCH 1/6] feat(dashscope): support qwen3.7 text reranking --- cookbook/dashscope_rerank.md | 44 +++++ litellm/llms/dashscope/common_utils.py | 14 +- litellm/llms/dashscope/qwen_ai_platform.py | 6 +- litellm/llms/dashscope/qwencloud.py | 6 +- .../dashscope/rerank/native_transformation.py | 83 ++++++++ .../llms/dashscope/rerank/transformation.py | 123 +++++++----- litellm/utils.py | 2 +- .../test_dashscope_rerank_transformation.py | 181 +++++++++++++++++- 8 files changed, 396 insertions(+), 63 deletions(-) create mode 100644 cookbook/dashscope_rerank.md create mode 100644 litellm/llms/dashscope/rerank/native_transformation.py diff --git a/cookbook/dashscope_rerank.md b/cookbook/dashscope_rerank.md new file mode 100644 index 00000000000..6a7173b6cba --- /dev/null +++ b/cookbook/dashscope_rerank.md @@ -0,0 +1,44 @@ +# Qwen3.7 text reranking + +Use `dashscope/qwen3.7-text-rerank` with LiteLLM's rerank interface and a Beijing DashScope API key + +```python +import litellm + +response = litellm.rerank( + model="dashscope/qwen3.7-text-rerank", + query="How can I reset my password?", + documents=[ + "The weather is sunny today.", + "Open account settings and select Reset password.", + "How do I change my password?", + ], + top_n=2, + return_documents=True, + instruction="Retrieve semantically similar text.", +) +``` + +Set `DASHSCOPE_API_KEY` in the environment or pass `api_key` explicitly. The asynchronous equivalent is `await litellm.arerank(...)` + +For the proxy, add a model entry and send the same query, documents and options to `/v1/rerank` using its configured model alias + +```yaml +model_list: + - model_name: qwen37-rerank + litellm_params: + model: dashscope/qwen3.7-text-rerank + api_key: os.environ/DASHSCOPE_API_KEY + model_info: + mode: rerank +``` + +LiteLLM sends the native DashScope request to `https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank`. An explicit `api_base` or `DASHSCOPE_API_BASE_RERANK` can select a different host, an `/api/v1` base, or the complete native endpoint. Chat-compatible DashScope paths are converted to the native path for this model + +`instruction` maps to DashScope's `parameters.instruct`. When omitted, the provider chooses its default relevance criterion. `top_n` and `return_documents` map to the corresponding native parameters. Live calls on 2026-09-08 confirmed that this model returns `document.text` when `return_documents=True`, despite the official parameter table omitting it from the supported-model list + +The response contains the provider request ID, original document indices, relevance scores and requested document text. `meta.tokens.input_tokens` comes from `usage.prompt_tokens`; `meta.billed_units.total_tokens` comes from `usage.total_tokens`. These counters do not add a model price or dollar-cost calculation + +The existing `dashscope/qwen3-rerank` model retains its compatible protocol. This change does not add multimodal reranking or reinterpret structured candidate documents + +Protocol reference: [DashScope text rerank API](https://help.aliyun.com/zh/model-studio/text-rerank-api) diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index b7c97893a15..254c845d509 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -2,7 +2,7 @@ Common utilities for the DashScope LLM provider. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Final import httpx @@ -15,6 +15,7 @@ if TYPE_CHECKING: BaseImageGenerationConfig, ) from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig + from litellm.llms.dashscope.rerank.transformation import DashScopeRerankConfig def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbeddingConfig": @@ -33,7 +34,16 @@ def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbe return DashScopeEmbeddingConfig() -def get_dashscope_family_rerank_config(custom_llm_provider: str) -> "BaseRerankConfig": +def get_dashscope_family_rerank_config(custom_llm_provider: str, model: str) -> "BaseRerankConfig": + provider_config: Final = _get_dashscope_family_rerank_provider_config(custom_llm_provider) + if model == "qwen3.7-text-rerank": + from litellm.llms.dashscope.rerank.native_transformation import DashScopeNativeRerankConfig + + return DashScopeNativeRerankConfig(provider_config) + return provider_config + + +def _get_dashscope_family_rerank_provider_config(custom_llm_provider: str) -> "DashScopeRerankConfig": if custom_llm_provider == "qwencloud": from litellm.llms.dashscope.qwencloud import QwenCloudRerankConfig diff --git a/litellm/llms/dashscope/qwen_ai_platform.py b/litellm/llms/dashscope/qwen_ai_platform.py index 9a44eaf574a..8e98c989c48 100644 --- a/litellm/llms/dashscope/qwen_ai_platform.py +++ b/litellm/llms/dashscope/qwen_ai_platform.py @@ -1,4 +1,4 @@ -from typing import Final +from typing import ClassVar, Final from litellm.secret_managers.main import get_secret_str @@ -47,11 +47,13 @@ class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig): class QwenAIPlatformRerankConfig(DashScopeRerankConfig): + DEFAULT_RERANK_API_BASE: ClassVar[str] = QWEN_AI_PLATFORM_RERANK_API_BASE + def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwen_ai_platform_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or QWEN_AI_PLATFORM_RERANK_API_BASE + return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE class QwenAIPlatformImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/qwencloud.py b/litellm/llms/dashscope/qwencloud.py index d8d53e340ef..827477b3ffc 100644 --- a/litellm/llms/dashscope/qwencloud.py +++ b/litellm/llms/dashscope/qwencloud.py @@ -1,4 +1,4 @@ -from typing import Final +from typing import ClassVar, Final from litellm.secret_managers.main import get_secret_str @@ -47,11 +47,13 @@ class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig): class QwenCloudRerankConfig(DashScopeRerankConfig): + DEFAULT_RERANK_API_BASE: ClassVar[str] = QWENCLOUD_RERANK_API_BASE + def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwencloud_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or QWENCLOUD_RERANK_API_BASE + return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE class QwenCloudImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/rerank/native_transformation.py b/litellm/llms/dashscope/rerank/native_transformation.py new file mode 100644 index 00000000000..bae0433aee9 --- /dev/null +++ b/litellm/llms/dashscope/rerank/native_transformation.py @@ -0,0 +1,83 @@ +"""DashScope native text reranking with input/parameters and output.results envelopes. + +qwen3.7-text-rerank was verified in Beijing, including return_documents=true. +Protocol routing through brand aliases does not establish regional model availability. +Docs: https://help.aliyun.com/zh/model-studio/text-rerank-api +""" + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +from pydantic import TypeAdapter + +from .transformation import DashScopeRerankConfig, DashScopeRerankUsage + + +class DashScopeNativeRerankConfig(DashScopeRerankConfig): + def __init__(self, provider_config: DashScopeRerankConfig) -> None: + # Reuse brand-specific credentials and hosts without duplicating alias subclasses. + self._provider_config: Final = provider_config + + def _resolve_api_key(self, api_key: str | None) -> str: + return self._provider_config._resolve_api_key(api_key) + + def _resolve_rerank_api_base(self, api_base: str | None) -> str: + return self._provider_config._resolve_rerank_api_base(api_base) + + def get_complete_url( + self, + api_base: str | None, + model: str, + optional_params: Mapping[str, object] | None = None, + ) -> str: + # Provider discovery supplies the brand's chat default even when no api_base was passed. + # Resolve the rerank-specific environment override before constructing the native path. + default_chat_base: Final = self._provider_config.DEFAULT_RERANK_API_BASE.replace( + "/compatible-api/v1/reranks", "/compatible-mode/v1" + ) + native_base: Final = self._resolve_rerank_api_base(None if api_base == default_chat_base else api_base) + parsed: Final = urlsplit(native_base.rstrip("/")) + if parsed.path.endswith("/services/rerank/text-rerank/text-rerank"): + return urlunsplit(parsed) + native_path: Final = parsed.path.removesuffix("/compatible-mode/v1").removesuffix("/compatible-api/v1/reranks") + api_path: Final = native_path if native_path.endswith("/api/v1") else f"{native_path}/api/v1" + return urlunsplit(parsed._replace(path=f"{api_path}/services/rerank/text-rerank/text-rerank")) + + def get_supported_cohere_rerank_params(self, model: str) -> list[str]: + return [*super().get_supported_cohere_rerank_params(model), "instruction"] + + def transform_rerank_request( + self, + model: str, + optional_rerank_params: Mapping[str, object], + headers: Mapping[str, object], + litellm_params: Mapping[str, object] | None = None, + ) -> dict[str, object]: + request: Final = super().transform_rerank_request(model, optional_rerank_params, headers, litellm_params) + return { + "model": model, + "input": {"query": request["query"], "documents": request["documents"]}, + "parameters": { + ("instruct" if name == "instruction" else name): optional_rerank_params[name] + for name in ("top_n", "return_documents", "instruction") + if optional_rerank_params.get(name) is not None + }, + } + + def _get_request_query(self, request_data: Mapping[str, object]) -> object: + return ( + TypeAdapter(Mapping[str, object]) + .validate_python(request_data.get("input", MappingProxyType({}))) + .get("query") + ) + + def _get_response_fields( + self, response_json: Mapping[str, object], usage: DashScopeRerankUsage + ) -> tuple[object, object, int | None]: + output: Final = TypeAdapter(Mapping[str, object]).validate_python( + response_json.get("output", MappingProxyType({})) + ) + # Native usage separates prompt_tokens from total_tokens; keep both provider counters. + return output.get("results"), response_json.get("request_id"), usage.get("prompt_tokens") diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 490757a0948..d0f9a0b1283 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -4,14 +4,15 @@ Transformation logic for DashScope's OpenAI-compatible /v1/reranks API. Supports - qwen3-rerank -(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — share the same -endpoint but have not been validated against this transformer. Behavior with -those models is undefined.) +(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — have not been +validated against this transformer. Behavior with those models is undefined.) + +The native qwen3.7-text-rerank protocol is implemented in native_transformation.py. Endpoint - https://dashscope.aliyuncs.com/compatible-api/v1/reranks -Note: chat/embed live under `/compatible-mode/v1/`, but DashScope's rerank +Note: chat/embed live under `/compatible-mode/v1/`, but qwen3-rerank's route is exposed under `/compatible-api/v1/reranks` per the docs. Override with `DASHSCOPE_API_BASE_RERANK` to point at a different host or path. @@ -23,9 +24,12 @@ Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api """ from collections.abc import Mapping -from typing import Any, Final +from types import MappingProxyType +from typing import ClassVar, Final import httpx +from pydantic import TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -33,7 +37,6 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( - OptionalRerankParams, RerankBilledUnits, RerankResponse, RerankResponseMeta, @@ -45,6 +48,11 @@ from ..common_utils import DashScopeError DEFAULT_RERANK_URL: Final = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" +class DashScopeRerankUsage(TypedDict, total=False): + prompt_tokens: ReadOnly[int | None] + total_tokens: ReadOnly[int | None] + + class DashScopeRerankConfig(BaseRerankConfig): """ Reference: https://help.aliyun.com/zh/model-studio/text-rerank-api @@ -53,8 +61,12 @@ class DashScopeRerankConfig(BaseRerankConfig): documents, top_n, return_documents. Response: results[].index, results[].relevance_score, optionally results[].document.text (when return_documents=true), plus a top-level usage.total_tokens counter. + + Brand aliases supply their own API key and base resolvers. """ + DEFAULT_RERANK_API_BASE: ClassVar[str] = DEFAULT_RERANK_URL + def __init__(self) -> None: pass @@ -69,13 +81,13 @@ class DashScopeRerankConfig(BaseRerankConfig): def _resolve_rerank_api_base(self, api_base: str | None) -> str: if api_base is not None: return api_base - return get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL + return get_secret_str("DASHSCOPE_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE def get_complete_url( self, api_base: str | None, model: str, - optional_params: dict | None = None, + optional_params: Mapping[str, object] | None = None, ) -> str: resolved_api_base: Final = self._resolve_rerank_api_base(api_base) if resolved_api_base == DEFAULT_RERANK_URL: @@ -93,12 +105,12 @@ class DashScopeRerankConfig(BaseRerankConfig): def validate_environment( self, - headers: dict, + headers: Mapping[str, object], model: str, api_key: str | None = None, - optional_params: dict | None = None, + optional_params: Mapping[str, object] | None = None, litellm_params: Mapping[str, object] | None = None, - ) -> dict: + ) -> dict[str, object]: return { "Authorization": f"Bearer {self._resolve_api_key(api_key)}", "accept": "application/json", @@ -106,16 +118,16 @@ class DashScopeRerankConfig(BaseRerankConfig): **headers, } - def get_supported_cohere_rerank_params(self, model: str) -> list: + def get_supported_cohere_rerank_params(self, model: str) -> list[str]: return ["query", "documents", "top_n", "return_documents"] def map_cohere_rerank_params( self, - non_default_params: dict | None, + non_default_params: Mapping[str, object] | None, model: str, drop_params: bool, query: str, - documents: list[str | dict[str, Any]], + documents: list[str | dict[str, object]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, @@ -123,26 +135,27 @@ class DashScopeRerankConfig(BaseRerankConfig): max_chunks_per_doc: int | None = None, max_tokens_per_doc: int | None = None, instruction: str | None = None, - ) -> dict: - # qwen3-rerank accepts query/documents/top_n/return_documents. The - # rest (rank_fields, max_*_per_doc) are silently dropped. - params: Final[OptionalRerankParams] = OptionalRerankParams( - query=query, - documents=documents, + ) -> dict[str, object]: + # rank_fields and max_*_per_doc have no supported mapping and are omitted. + params: Final = MappingProxyType( + { + "query": query, + "documents": documents, + "top_n": top_n, + "return_documents": return_documents, + "instruction": instruction, + } ) - if top_n is not None: - params["top_n"] = top_n - if return_documents is not None: - params["return_documents"] = return_documents - return dict(params) + supported_params: Final = self.get_supported_cohere_rerank_params(model) + return {name: value for name, value in params.items() if value is not None and name in supported_params} def transform_rerank_request( self, model: str, - optional_rerank_params: dict, - headers: dict, - litellm_params: dict | None = None, - ) -> dict: + optional_rerank_params: Mapping[str, object], + headers: Mapping[str, object], + litellm_params: Mapping[str, object] | None = None, + ) -> dict[str, object]: if "query" not in optional_rerank_params: raise ValueError("query is required for DashScope rerank") if "documents" not in optional_rerank_params: @@ -167,22 +180,20 @@ class DashScopeRerankConfig(BaseRerankConfig): logging_obj: LiteLLMLoggingObj, api_key: str | None = None, request_data: dict | None = None, - optional_params: dict | None = None, - litellm_params: dict | None = None, + optional_params: Mapping[str, object] | None = None, + litellm_params: Mapping[str, object] | None = None, ) -> RerankResponse: request_data = request_data or {} - optional_params = optional_params or {} - litellm_params = litellm_params or {} try: - response_json: Final = raw_response.json() - except Exception: + response_json: Final = TypeAdapter(Mapping[str, object]).validate_json(raw_response.content) + except ValidationError as exc: raise DashScopeError( status_code=raw_response.status_code, message=raw_response.text, - ) + ) from exc logging_obj.post_call( - input=request_data.get("query"), + input=self._get_request_query(request_data), api_key=api_key, additional_args={"complete_input_dict": request_data}, original_response=response_json, @@ -192,23 +203,26 @@ class DashScopeRerankConfig(BaseRerankConfig): if "code" in response_json and "results" not in response_json: raise DashScopeError( status_code=raw_response.status_code, - message=response_json.get("message", str(response_json)), + message=str(response_json.get("message", response_json)), ) - results: Final = response_json.get("results") + usage: Final = TypeAdapter(DashScopeRerankUsage).validate_python( + response_json.get("usage") or MappingProxyType({}) + ) + results, response_id, input_tokens = self._get_response_fields(response_json, usage) if results is None: raise DashScopeError( status_code=raw_response.status_code, message=f"No results in DashScope rerank response: {response_json}", ) - # qwen3-rerank returns: + # Both protocols return: # {"index": int, "relevance_score": float} # plus, when return_documents=true was sent: # "document": {"text": "..."} # which already matches LiteLLM's RerankResponseDocument shape. transformed_results: Final[list[dict]] = [] - for r in results: + for r in TypeAdapter(tuple[Mapping[str, object], ...]).validate_python(results): item: dict[str, object] = { "index": r["index"], "relevance_score": r["relevance_score"], @@ -221,18 +235,29 @@ class DashScopeRerankConfig(BaseRerankConfig): item["document"] = {"text": doc} transformed_results.append(item) - usage: Final = response_json.get("usage") or {} - total_tokens: Final = usage.get("total_tokens") - billed_units: Final = RerankBilledUnits(total_tokens=total_tokens) - tokens: Final = RerankTokens(input_tokens=total_tokens) + billed_units: Final = RerankBilledUnits(total_tokens=usage.get("total_tokens")) + tokens: Final = RerankTokens(input_tokens=input_tokens) meta: Final = RerankResponseMeta(billed_units=billed_units, tokens=tokens) - return RerankResponse( - id=response_json.get("id") or str(uuid.uuid4()), - results=transformed_results, - meta=meta, + return RerankResponse.model_validate( + MappingProxyType( + { + "id": response_id or str(uuid.uuid4()), + "results": transformed_results, + "meta": meta, + } + ) ) + def _get_request_query(self, request_data: Mapping[str, object]) -> object: + return request_data.get("query") + + def _get_response_fields( + self, response_json: Mapping[str, object], usage: DashScopeRerankUsage + ) -> tuple[object, object, int | None]: + # The compatible API reports its input count only as total_tokens. + return response_json.get("results"), response_json.get("id"), usage.get("total_tokens") + def get_error_class( self, error_message: str, diff --git a/litellm/utils.py b/litellm/utils.py index d0e11bc9551..7df926e7a20 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8439,7 +8439,7 @@ class ProviderConfigManager: get_dashscope_family_rerank_config, ) - return get_dashscope_family_rerank_config(provider.value) + return get_dashscope_family_rerank_config(provider.value, model) return litellm.CohereRerankConfig() @staticmethod diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index 936de812bc6..26f7f9c2226 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -7,9 +7,10 @@ from unittest.mock import MagicMock import httpx import pytest +import respx -from litellm.llms.dashscope.common_utils import DashScopeError +from litellm.llms.dashscope.common_utils import DashScopeError, get_dashscope_family_rerank_config from litellm.llms.dashscope.rerank.transformation import ( DEFAULT_RERANK_URL, DashScopeRerankConfig, @@ -103,6 +104,7 @@ class TestDashScopeRerankRequest: return_documents=True, max_chunks_per_doc=5, max_tokens_per_doc=100, + instruction="Unsupported on the compatible protocol", ) assert params == { "query": "什么是文本排序模型", @@ -274,30 +276,32 @@ class TestDashScopeRerankResponse: ) assert out.id is not None and len(out.id) > 0 - def test_error_envelope_raises(self): + @pytest.mark.parametrize("model", ["qwen3-rerank", "qwen3.7-text-rerank"]) + def test_error_envelope_raises(self, model): body = { "code": "InvalidApiKey", "message": "Invalid API-key provided.", "request_id": "fb53", } with pytest.raises(DashScopeError) as exc_info: - self.config.transform_rerank_response( - model="qwen3-rerank", + get_dashscope_family_rerank_config("dashscope", model).transform_rerank_response( + model=model, raw_response=self._resp(body, status_code=401), model_response=RerankResponse(), logging_obj=self.logging, ) assert "Invalid API-key provided." in str(exc_info.value) - def test_non_json_response_raises(self): + @pytest.mark.parametrize("model", ["qwen3-rerank", "qwen3.7-text-rerank"]) + def test_non_json_response_raises(self, model): bad = httpx.Response( status_code=500, content=b"bad gateway", request=httpx.Request("POST", "https://example.com"), ) with pytest.raises(DashScopeError): - self.config.transform_rerank_response( - model="qwen3-rerank", + get_dashscope_family_rerank_config("dashscope", model).transform_rerank_response( + model=model, raw_response=bad, model_response=RerankResponse(), logging_obj=self.logging, @@ -323,3 +327,166 @@ class TestProviderConfigManagerDispatch: present_version_params=[], ) assert isinstance(cfg, DashScopeRerankConfig) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("return_documents", [False, True]) +@pytest.mark.parametrize( + "provider,host", + [ + ("dashscope", "dashscope.aliyuncs.com"), + ("qwencloud", "dashscope-intl.aliyuncs.com"), + ("qwen_ai_platform", "dashscope.aliyuncs.com"), + ], +) +async def test_qwen37_rerank_public_call( + is_async, return_documents, provider, host, respx_mock: respx.MockRouter, monkeypatch +): + import litellm + + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) + monkeypatch.delenv(f"{provider.upper()}_API_BASE", raising=False) + monkeypatch.delenv(f"{provider.upper()}_API_BASE_RERANK", raising=False) + monkeypatch.setenv(f"{provider.upper()}_API_KEY", "fake-brand-key") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + route = respx_mock.post(f"https://{host}/api/v1/services/rerank/text-rerank/text-rerank") + results = [{"index": 1, "relevance_score": 0.88, **({"document": {"text": "answer"}} if return_documents else {})}] + route.respond( + 200, + json={ + "output": {"results": results}, + "usage": {"prompt_tokens": 237, "total_tokens": 261, "details": {"provider_metadata": True}}, + "request_id": "qwen37-request-id", + }, + ) + kwargs = { + "model": f"{provider}/qwen3.7-text-rerank", + "query": "question", + "documents": ["unrelated", "answer"], + "top_n": 1, + "return_documents": return_documents, + "instruction": "Retrieve semantically similar text.", + } + + response = await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) + + assert json.loads(route.calls[0].request.content) == { + "model": "qwen3.7-text-rerank", + "input": {"query": "question", "documents": ["unrelated", "answer"]}, + "parameters": { + "top_n": 1, + "return_documents": return_documents, + "instruct": "Retrieve semantically similar text.", + }, + } + assert route.calls[0].request.headers["authorization"] == "Bearer fake-brand-key" + assert response.id == "qwen37-request-id" + assert response.results == results + assert response.meta == {"billed_units": {"total_tokens": 261}, "tokens": {"input_tokens": 237}} + + +@pytest.mark.parametrize( + "api_base", + [ + "https://proxy.example/api/v1", + "https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank/", + ], +) +def test_qwen37_rerank_custom_url(api_base): + assert get_dashscope_family_rerank_config("dashscope", "qwen3.7-text-rerank").get_complete_url( + api_base, "qwen3.7-text-rerank" + ) == ("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") + + +@pytest.mark.parametrize( + "provider,host", + [("dashscope", "dashscope-intl.aliyuncs.com"), ("qwencloud", "dashscope.aliyuncs.com")], +) +def test_qwen37_rerank_explicit_region(provider, host): + config = get_dashscope_family_rerank_config(provider, "qwen3.7-text-rerank") + assert config.get_complete_url(f"https://{host}/compatible-mode/v1", "qwen3.7-text-rerank") == ( + f"https://{host}/api/v1/services/rerank/text-rerank/text-rerank" + ) + + +def test_qwen37_rerank_response_logging(): + config = get_dashscope_family_rerank_config("dashscope", "qwen3.7-text-rerank") + logging = MagicMock() + request = {"model": "qwen3.7-text-rerank", "input": {"query": "question", "documents": ["answer"]}, "parameters": {}} + payload = {"request_id": "request-id", "output": {"results": [{"index": 0, "relevance_score": 0.88}]}} + + response = config.transform_rerank_response( + model="qwen3.7-text-rerank", + raw_response=httpx.Response(200, json=payload), + model_response=RerankResponse(), + logging_obj=logging, + request_data=request, + ) + + logging.post_call.assert_called_once_with( + input="question", api_key=None, additional_args={"complete_input_dict": request}, original_response=payload + ) + assert response.id == "request-id" + assert response.results == [{"index": 0, "relevance_score": 0.88}] + + +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +def test_qwen37_rerank_environment_url(provider, respx_mock: respx.MockRouter, monkeypatch): + import litellm + + monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) + monkeypatch.delenv(f"{provider.upper()}_API_BASE", raising=False) + monkeypatch.setenv(f"{provider.upper()}_API_BASE_RERANK", "https://proxy.example/api/v1") + route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") + route.respond( + 200, + json={ + "output": {"results": [{"index": 0, "relevance_score": 0.88}]}, + "request_id": "all-results", + "usage": {"prompt_tokens": 10, "total_tokens": 10}, + }, + ) + + response = litellm.rerank( + model=f"{provider}/qwen3.7-text-rerank", + query="question", + documents=["answer"], + return_documents=None, + api_key="fake-dashscope-key", + ) + + assert json.loads(route.calls[0].request.content) == { + "model": "qwen3.7-text-rerank", + "input": {"query": "question", "documents": ["answer"]}, + "parameters": {}, + } + assert response.results == [{"index": 0, "relevance_score": 0.88}] + assert response.meta["tokens"]["input_tokens"] == 10 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +async def test_qwen37_rerank_preserves_provider_error(is_async, respx_mock: respx.MockRouter, monkeypatch): + import litellm + + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") + route.respond( + 400, + json={"code": "InvalidParameter", "message": "documents must not be empty", "request_id": "invalid-documents"}, + ) + kwargs = { + "model": "dashscope/qwen3.7-text-rerank", + "query": "question", + "documents": [], + "api_key": "fake-dashscope-key", + "api_base": "https://proxy.example/api/v1", + } + + with pytest.raises(litellm.BadRequestError, match="documents must not be empty") as error: + await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) + + assert error.value.status_code == 400 + assert "DashscopeException" in str(error.value) From 331db73127ba33990f0374ab842058bf8c22ab7f Mon Sep 17 00:00:00 2001 From: jayson wang Date: Tue, 8 Sep 2026 12:08:22 +0800 Subject: [PATCH 2/6] fix(dashscope): honor rerank base overrides and model metadata --- cookbook/dashscope_rerank.md | 44 ------- litellm/llms/dashscope/common_utils.py | 57 ++++++++- litellm/llms/dashscope/qwen_ai_platform.py | 6 +- litellm/llms/dashscope/qwencloud.py | 6 +- .../dashscope/rerank/native_transformation.py | 12 +- .../llms/dashscope/rerank/transformation.py | 10 +- ...odel_prices_and_context_window_backup.json | 8 ++ litellm/utils.py | 2 +- model_prices_and_context_window.json | 8 ++ .../test_dashscope_rerank_transformation.py | 120 +++++++++++++++++- 10 files changed, 198 insertions(+), 75 deletions(-) delete mode 100644 cookbook/dashscope_rerank.md diff --git a/cookbook/dashscope_rerank.md b/cookbook/dashscope_rerank.md deleted file mode 100644 index 6a7173b6cba..00000000000 --- a/cookbook/dashscope_rerank.md +++ /dev/null @@ -1,44 +0,0 @@ -# Qwen3.7 text reranking - -Use `dashscope/qwen3.7-text-rerank` with LiteLLM's rerank interface and a Beijing DashScope API key - -```python -import litellm - -response = litellm.rerank( - model="dashscope/qwen3.7-text-rerank", - query="How can I reset my password?", - documents=[ - "The weather is sunny today.", - "Open account settings and select Reset password.", - "How do I change my password?", - ], - top_n=2, - return_documents=True, - instruction="Retrieve semantically similar text.", -) -``` - -Set `DASHSCOPE_API_KEY` in the environment or pass `api_key` explicitly. The asynchronous equivalent is `await litellm.arerank(...)` - -For the proxy, add a model entry and send the same query, documents and options to `/v1/rerank` using its configured model alias - -```yaml -model_list: - - model_name: qwen37-rerank - litellm_params: - model: dashscope/qwen3.7-text-rerank - api_key: os.environ/DASHSCOPE_API_KEY - model_info: - mode: rerank -``` - -LiteLLM sends the native DashScope request to `https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank`. An explicit `api_base` or `DASHSCOPE_API_BASE_RERANK` can select a different host, an `/api/v1` base, or the complete native endpoint. Chat-compatible DashScope paths are converted to the native path for this model - -`instruction` maps to DashScope's `parameters.instruct`. When omitted, the provider chooses its default relevance criterion. `top_n` and `return_documents` map to the corresponding native parameters. Live calls on 2026-09-08 confirmed that this model returns `document.text` when `return_documents=True`, despite the official parameter table omitting it from the supported-model list - -The response contains the provider request ID, original document indices, relevance scores and requested document text. `meta.tokens.input_tokens` comes from `usage.prompt_tokens`; `meta.billed_units.total_tokens` comes from `usage.total_tokens`. These counters do not add a model price or dollar-cost calculation - -The existing `dashscope/qwen3-rerank` model retains its compatible protocol. This change does not add multimodal reranking or reinterpret structured candidate documents - -Protocol reference: [DashScope text rerank API](https://help.aliyun.com/zh/model-studio/text-rerank-api) diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 254c845d509..862a5f60bec 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -2,10 +2,15 @@ Common utilities for the DashScope LLM provider. """ -from typing import TYPE_CHECKING, Final +from collections.abc import Mapping +from functools import lru_cache +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal import httpx +from pydantic import TypeAdapter +import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -34,15 +39,59 @@ def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbe return DashScopeEmbeddingConfig() -def get_dashscope_family_rerank_config(custom_llm_provider: str, model: str) -> "BaseRerankConfig": +def get_dashscope_family_rerank_config( + custom_llm_provider: str, model: str, api_base: str | None = None +) -> "BaseRerankConfig": provider_config: Final = _get_dashscope_family_rerank_provider_config(custom_llm_provider) - if model == "qwen3.7-text-rerank": + model_cost: Final[Mapping[str, object]] = litellm.model_cost + runtime_api: Final = next( + ( + declared_api + for key in (f"{custom_llm_provider}/{model}", f"dashscope/{model}", model) + if (declared_api := _rerank_api_from_model_info(model_cost.get(key))) is not None + ), + None, + ) + rerank_api: Final = runtime_api or _bundled_dashscope_rerank_apis().get(f"dashscope/{model}") + if rerank_api == "native": from litellm.llms.dashscope.rerank.native_transformation import DashScopeNativeRerankConfig - return DashScopeNativeRerankConfig(provider_config) + return DashScopeNativeRerankConfig( + provider_config, api_base=api_base or get_secret_str(f"{custom_llm_provider.upper()}_API_BASE_RERANK") + ) return provider_config +def _rerank_api_from_model_info(raw_model_info: object) -> str | None: + if raw_model_info is None: + return None + model_info: Final = TypeAdapter(Mapping[str, object]).validate_python(raw_model_info) + provider_info: Final = model_info.get("provider_specific_entry") + if provider_info is None: + return None + metadata: Final = TypeAdapter(Mapping[str, object]).validate_python(provider_info) + rerank_api: Final[str | None] = TypeAdapter(Literal["native", "compatible"] | None).validate_python( + metadata.get("rerank_api") + ) + return rerank_api + + +@lru_cache(maxsize=1) +def _bundled_dashscope_rerank_apis() -> Mapping[str, str]: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + model_infos: Final = TypeAdapter(Mapping[str, object]).validate_json( + GetModelCostMap.read_local_model_cost_map_text() + ) + return MappingProxyType( + { + key: rerank_api + for key, model_info in model_infos.items() + if key.startswith("dashscope/") and (rerank_api := _rerank_api_from_model_info(model_info)) is not None + } + ) + + def _get_dashscope_family_rerank_provider_config(custom_llm_provider: str) -> "DashScopeRerankConfig": if custom_llm_provider == "qwencloud": from litellm.llms.dashscope.qwencloud import QwenCloudRerankConfig diff --git a/litellm/llms/dashscope/qwen_ai_platform.py b/litellm/llms/dashscope/qwen_ai_platform.py index 8e98c989c48..9a44eaf574a 100644 --- a/litellm/llms/dashscope/qwen_ai_platform.py +++ b/litellm/llms/dashscope/qwen_ai_platform.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Final +from typing import Final from litellm.secret_managers.main import get_secret_str @@ -47,13 +47,11 @@ class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig): class QwenAIPlatformRerankConfig(DashScopeRerankConfig): - DEFAULT_RERANK_API_BASE: ClassVar[str] = QWEN_AI_PLATFORM_RERANK_API_BASE - def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwen_ai_platform_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE + return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or QWEN_AI_PLATFORM_RERANK_API_BASE class QwenAIPlatformImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/qwencloud.py b/litellm/llms/dashscope/qwencloud.py index 827477b3ffc..d8d53e340ef 100644 --- a/litellm/llms/dashscope/qwencloud.py +++ b/litellm/llms/dashscope/qwencloud.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Final +from typing import Final from litellm.secret_managers.main import get_secret_str @@ -47,13 +47,11 @@ class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig): class QwenCloudRerankConfig(DashScopeRerankConfig): - DEFAULT_RERANK_API_BASE: ClassVar[str] = QWENCLOUD_RERANK_API_BASE - def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwencloud_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE + return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or QWENCLOUD_RERANK_API_BASE class QwenCloudImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/rerank/native_transformation.py b/litellm/llms/dashscope/rerank/native_transformation.py index bae0433aee9..9e34375ab12 100644 --- a/litellm/llms/dashscope/rerank/native_transformation.py +++ b/litellm/llms/dashscope/rerank/native_transformation.py @@ -16,9 +16,9 @@ from .transformation import DashScopeRerankConfig, DashScopeRerankUsage class DashScopeNativeRerankConfig(DashScopeRerankConfig): - def __init__(self, provider_config: DashScopeRerankConfig) -> None: - # Reuse brand-specific credentials and hosts without duplicating alias subclasses. + def __init__(self, provider_config: DashScopeRerankConfig, api_base: str | None = None) -> None: self._provider_config: Final = provider_config + self._api_base: Final = api_base def _resolve_api_key(self, api_key: str | None) -> str: return self._provider_config._resolve_api_key(api_key) @@ -32,12 +32,7 @@ class DashScopeNativeRerankConfig(DashScopeRerankConfig): model: str, optional_params: Mapping[str, object] | None = None, ) -> str: - # Provider discovery supplies the brand's chat default even when no api_base was passed. - # Resolve the rerank-specific environment override before constructing the native path. - default_chat_base: Final = self._provider_config.DEFAULT_RERANK_API_BASE.replace( - "/compatible-api/v1/reranks", "/compatible-mode/v1" - ) - native_base: Final = self._resolve_rerank_api_base(None if api_base == default_chat_base else api_base) + native_base: Final = self._resolve_rerank_api_base(self._api_base or api_base) parsed: Final = urlsplit(native_base.rstrip("/")) if parsed.path.endswith("/services/rerank/text-rerank/text-rerank"): return urlunsplit(parsed) @@ -79,5 +74,4 @@ class DashScopeNativeRerankConfig(DashScopeRerankConfig): output: Final = TypeAdapter(Mapping[str, object]).validate_python( response_json.get("output", MappingProxyType({})) ) - # Native usage separates prompt_tokens from total_tokens; keep both provider counters. return output.get("results"), response_json.get("request_id"), usage.get("prompt_tokens") diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index d0f9a0b1283..b75a5042e9a 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -25,7 +25,7 @@ Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api from collections.abc import Mapping from types import MappingProxyType -from typing import ClassVar, Final +from typing import Final import httpx from pydantic import TypeAdapter, ValidationError @@ -61,12 +61,8 @@ class DashScopeRerankConfig(BaseRerankConfig): documents, top_n, return_documents. Response: results[].index, results[].relevance_score, optionally results[].document.text (when return_documents=true), plus a top-level usage.total_tokens counter. - - Brand aliases supply their own API key and base resolvers. """ - DEFAULT_RERANK_API_BASE: ClassVar[str] = DEFAULT_RERANK_URL - def __init__(self) -> None: pass @@ -81,7 +77,7 @@ class DashScopeRerankConfig(BaseRerankConfig): def _resolve_rerank_api_base(self, api_base: str | None) -> str: if api_base is not None: return api_base - return get_secret_str("DASHSCOPE_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE + return get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL def get_complete_url( self, @@ -136,7 +132,6 @@ class DashScopeRerankConfig(BaseRerankConfig): max_tokens_per_doc: int | None = None, instruction: str | None = None, ) -> dict[str, object]: - # rank_fields and max_*_per_doc have no supported mapping and are omitted. params: Final = MappingProxyType( { "query": query, @@ -255,7 +250,6 @@ class DashScopeRerankConfig(BaseRerankConfig): def _get_response_fields( self, response_json: Mapping[str, object], usage: DashScopeRerankUsage ) -> tuple[object, object, int | None]: - # The compatible API reports its input count only as total_tokens. return response_json.get("results"), response_json.get("id"), usage.get("total_tokens") def get_error_class( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b1ffc1583e4..1633158b264 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14753,6 +14753,14 @@ } ] }, + "dashscope/qwen3.7-text-rerank": { + "litellm_provider": "dashscope", + "mode": "rerank", + "provider_specific_entry": { + "rerank_api": "native" + }, + "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api" + }, "dashscope/qwen-turbo": { "input_cost_per_token": 5e-08, "litellm_provider": "dashscope", diff --git a/litellm/utils.py b/litellm/utils.py index 7df926e7a20..c5de9435aee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8439,7 +8439,7 @@ class ProviderConfigManager: get_dashscope_family_rerank_config, ) - return get_dashscope_family_rerank_config(provider.value, model) + return get_dashscope_family_rerank_config(provider.value, model, api_base) return litellm.CohereRerankConfig() @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1ffc1583e4..1633158b264 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14753,6 +14753,14 @@ } ] }, + "dashscope/qwen3.7-text-rerank": { + "litellm_provider": "dashscope", + "mode": "rerank", + "provider_specific_entry": { + "rerank_api": "native" + }, + "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api" + }, "dashscope/qwen-turbo": { "input_cost_per_token": 5e-08, "litellm_provider": "dashscope", diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index 26f7f9c2226..7b1f6d9dc09 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -414,7 +414,11 @@ def test_qwen37_rerank_explicit_region(provider, host): def test_qwen37_rerank_response_logging(): config = get_dashscope_family_rerank_config("dashscope", "qwen3.7-text-rerank") logging = MagicMock() - request = {"model": "qwen3.7-text-rerank", "input": {"query": "question", "documents": ["answer"]}, "parameters": {}} + request = { + "model": "qwen3.7-text-rerank", + "input": {"query": "question", "documents": ["answer"]}, + "parameters": {}, + } payload = {"request_id": "request-id", "output": {"results": [{"index": 0, "relevance_score": 0.88}]}} response = config.transform_rerank_response( @@ -490,3 +494,117 @@ async def test_qwen37_rerank_preserves_provider_error(is_async, respx_mock: resp assert error.value.status_code == 400 assert "DashscopeException" in str(error.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +@pytest.mark.parametrize("base_case", ["rerank_override", "explicit", "explicit_default", "general_only"]) +async def test_native_rerank_base_precedence(provider, is_async, base_case, respx_mock, monkeypatch): + import litellm + + prefix = provider.upper() + default_host = "dashscope-intl.aliyuncs.com" if provider == "qwencloud" else "dashscope.aliyuncs.com" + monkeypatch.setenv(f"{prefix}_API_BASE", "https://chat.example/compatible-mode/v1") + monkeypatch.delenv(f"{prefix}_API_BASE_RERANK", raising=False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + if base_case != "general_only": + monkeypatch.setenv(f"{prefix}_API_BASE_RERANK", "https://rerank.example/api/v1") + explicit_base = { + "rerank_override": None, + "explicit": "https://explicit.example/api/v1", + "explicit_default": f"https://{default_host}/compatible-mode/v1", + "general_only": None, + }[base_case] + expected_host = { + "rerank_override": "rerank.example", + "explicit": "explicit.example", + "explicit_default": default_host, + "general_only": "chat.example", + }[base_case] + route = respx_mock.post(f"https://{expected_host}/api/v1/services/rerank/text-rerank/text-rerank") + route.respond( + 200, json={"request_id": "base-precedence", "output": {"results": [{"index": 0, "relevance_score": 0.9}]}} + ) + kwargs = { + "model": f"{provider}/qwen3.7-text-rerank", + "query": "question", + "documents": ["answer"], + "api_key": "fake-review-key", + "api_base": explicit_base, + } + + response = await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) + + assert json.loads(route.calls[0].request.content)["input"] == {"query": "question", "documents": ["answer"]} + assert response.id == "base-precedence" + assert response.results == [{"index": 0, "relevance_score": 0.9}] + + +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +@pytest.mark.parametrize("rerank_api", ["native", "compatible"]) +def test_rerank_protocol_uses_runtime_model_metadata(provider, rerank_api, respx_mock, monkeypatch): + import litellm + + model = "custom-rerank" if rerank_api == "native" else "qwen3.7-text-rerank" + monkeypatch.setitem( + litellm.model_cost, + model if provider == "dashscope" and rerank_api == "native" else f"{provider}/{model}", + { + "litellm_provider": provider, + "mode": "rerank", + "provider_specific_entry": {"rerank_api": rerank_api}, + }, + ) + results = [{"index": 0, "relevance_score": 0.9}] + url = ( + "https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank" + if rerank_api == "native" + else "https://proxy.example/api/v1/reranks" + ) + route = respx_mock.post(url) + route.respond( + 200, + json={"request_id": "metadata", "output": {"results": results}} + if rerank_api == "native" + else {"id": "metadata", "results": results}, + ) + + response = litellm.rerank( + model=f"{provider}/{model}", + query="question", + documents=["answer"], + api_key="fake-key", + api_base="https://proxy.example/api/v1", + return_documents=None, + ) + + assert json.loads(route.calls[0].request.content) == ( + {"model": model, "input": {"query": "question", "documents": ["answer"]}, "parameters": {}} + if rerank_api == "native" + else {"model": model, "query": "question", "documents": ["answer"]} + ) + assert response.id == "metadata" + assert response.results == results + + +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +def test_rerank_uses_bundled_metadata_when_remote_map_lacks_model(provider, respx_mock, monkeypatch): + import litellm + + for prefix in ("dashscope", "qwencloud", "qwen_ai_platform"): + monkeypatch.delitem(litellm.model_cost, f"{prefix}/qwen3.7-text-rerank", raising=False) + route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") + route.respond(200, json={"request_id": "bundled", "output": {"results": [{"index": 0, "relevance_score": 0.9}]}}) + + response = litellm.rerank( + model=f"{provider}/qwen3.7-text-rerank", + query="question", + documents=["answer"], + api_key="fake-key", + api_base="https://proxy.example/api/v1", + ) + + assert json.loads(route.calls[0].request.content)["input"] == {"query": "question", "documents": ["answer"]} + assert response.id == "bundled" + assert response.results == [{"index": 0, "relevance_score": 0.9}] From 3fd1d4e74151bc0427cf07c0ab78fea13870a0ab Mon Sep 17 00:00:00 2001 From: jayson wang Date: Fri, 11 Sep 2026 10:10:07 +0800 Subject: [PATCH 3/6] refactor(dashscope): keep rerank logging input immutable --- litellm/llms/dashscope/rerank/transformation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index fba310a0830..7bf7d117d67 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -180,7 +180,7 @@ class DashScopeRerankConfig(BaseRerankConfig): optional_params: Mapping[str, object] | None = None, litellm_params: Mapping[str, object] | None = None, ) -> RerankResponse: - request_data = request_data or {} + request: Final = request_data or MappingProxyType({}) try: response_json: Final = TypeAdapter(Mapping[str, object]).validate_json(raw_response.content) except ValidationError as exc: @@ -190,9 +190,9 @@ class DashScopeRerankConfig(BaseRerankConfig): ) from exc logging_obj.post_call( - input=self._get_request_query(request_data), + input=self._get_request_query(request), api_key=api_key, - additional_args={"complete_input_dict": request_data}, + additional_args={"complete_input_dict": request}, original_response=response_json, ) From 4ae6403be90ed88cf89e7c52ffd64e644d424ba6 Mon Sep 17 00:00:00 2001 From: jayson wang Date: Fri, 11 Sep 2026 10:23:56 +0800 Subject: [PATCH 4/6] fix(dashscope): forward rerank instructions --- litellm/llms/dashscope/common_utils.py | 63 +--- .../dashscope/rerank/native_transformation.py | 77 ----- .../llms/dashscope/rerank/transformation.py | 125 +++---- ...odel_prices_and_context_window_backup.json | 8 - litellm/utils.py | 2 +- model_prices_and_context_window.json | 8 - .../test_dashscope_rerank_transformation.py | 305 ++---------------- 7 files changed, 85 insertions(+), 503 deletions(-) delete mode 100644 litellm/llms/dashscope/rerank/native_transformation.py diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 26d9d3bd590..9ed9c276e43 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -2,16 +2,11 @@ Common utilities for the DashScope LLM provider. """ -from collections.abc import Mapping -from functools import lru_cache -from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal +from typing import TYPE_CHECKING, Final from urllib.parse import urlparse import httpx -from pydantic import TypeAdapter -import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -21,7 +16,6 @@ if TYPE_CHECKING: BaseImageGenerationConfig, ) from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig - from litellm.llms.dashscope.rerank.transformation import DashScopeRerankConfig DASHSCOPE_CHAT_COMPATIBLE_PATH: Final = "/compatible-mode/v1" DASHSCOPE_RERANK_PATH: Final = "/compatible-api/v1/reranks" @@ -61,60 +55,7 @@ def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbe return DashScopeEmbeddingConfig() -def get_dashscope_family_rerank_config( - custom_llm_provider: str, model: str, api_base: str | None = None -) -> "BaseRerankConfig": - provider_config: Final = _get_dashscope_family_rerank_provider_config(custom_llm_provider) - model_cost: Final[Mapping[str, object]] = litellm.model_cost - runtime_api: Final = next( - ( - declared_api - for key in (f"{custom_llm_provider}/{model}", f"dashscope/{model}", model) - if (declared_api := _rerank_api_from_model_info(model_cost.get(key))) is not None - ), - None, - ) - rerank_api: Final = runtime_api or _bundled_dashscope_rerank_apis().get(f"dashscope/{model}") - if rerank_api == "native": - from litellm.llms.dashscope.rerank.native_transformation import DashScopeNativeRerankConfig - - return DashScopeNativeRerankConfig( - provider_config, api_base=api_base or get_secret_str(f"{custom_llm_provider.upper()}_API_BASE_RERANK") - ) - return provider_config - - -def _rerank_api_from_model_info(raw_model_info: object) -> str | None: - if raw_model_info is None: - return None - model_info: Final = TypeAdapter(Mapping[str, object]).validate_python(raw_model_info) - provider_info: Final = model_info.get("provider_specific_entry") - if provider_info is None: - return None - metadata: Final = TypeAdapter(Mapping[str, object]).validate_python(provider_info) - rerank_api: Final[str | None] = TypeAdapter(Literal["native", "compatible"] | None).validate_python( - metadata.get("rerank_api") - ) - return rerank_api - - -@lru_cache(maxsize=1) -def _bundled_dashscope_rerank_apis() -> Mapping[str, str]: - from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap - - model_infos: Final = TypeAdapter(Mapping[str, object]).validate_json( - GetModelCostMap.read_local_model_cost_map_text() - ) - return MappingProxyType( - { - key: rerank_api - for key, model_info in model_infos.items() - if key.startswith("dashscope/") and (rerank_api := _rerank_api_from_model_info(model_info)) is not None - } - ) - - -def _get_dashscope_family_rerank_provider_config(custom_llm_provider: str) -> "DashScopeRerankConfig": +def get_dashscope_family_rerank_config(custom_llm_provider: str) -> "BaseRerankConfig": if custom_llm_provider == "qwencloud": from litellm.llms.dashscope.qwencloud import QwenCloudRerankConfig diff --git a/litellm/llms/dashscope/rerank/native_transformation.py b/litellm/llms/dashscope/rerank/native_transformation.py deleted file mode 100644 index 39ebf4e2de3..00000000000 --- a/litellm/llms/dashscope/rerank/native_transformation.py +++ /dev/null @@ -1,77 +0,0 @@ -"""DashScope native text reranking with input/parameters and output.results envelopes. - -qwen3.7-text-rerank was verified in Beijing, including return_documents=true. -Protocol routing through brand aliases does not establish regional model availability. -Docs: https://help.aliyun.com/zh/model-studio/text-rerank-api -""" - -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final -from urllib.parse import urlsplit, urlunsplit - -from pydantic import TypeAdapter - -from .transformation import DashScopeRerankConfig, DashScopeRerankUsage - - -class DashScopeNativeRerankConfig(DashScopeRerankConfig): - def __init__(self, provider_config: DashScopeRerankConfig, api_base: str | None = None) -> None: - self._provider_config: Final = provider_config - self._api_base: Final = api_base - - def _resolve_api_key(self, api_key: str | None) -> str: - return self._provider_config._resolve_api_key(api_key) - - def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return self._provider_config._resolve_rerank_api_base(api_base) - - def get_complete_url( - self, - api_base: str | None, - model: str, - optional_params: Mapping[str, object] | None = None, - ) -> str: - native_base: Final = self._api_base or self._resolve_rerank_api_base(api_base) - parsed: Final = urlsplit(native_base.rstrip("/")) - if parsed.path.endswith("/services/rerank/text-rerank/text-rerank"): - return urlunsplit(parsed) - native_path: Final = parsed.path.removesuffix("/compatible-mode/v1").removesuffix("/compatible-api/v1/reranks") - api_path: Final = native_path if native_path.endswith("/api/v1") else f"{native_path}/api/v1" - return urlunsplit(parsed._replace(path=f"{api_path}/services/rerank/text-rerank/text-rerank")) - - def get_supported_cohere_rerank_params(self, model: str) -> list[str]: - return [*super().get_supported_cohere_rerank_params(model), "instruction"] - - def transform_rerank_request( - self, - model: str, - optional_rerank_params: Mapping[str, object], - headers: Mapping[str, object], - litellm_params: Mapping[str, object] | None = None, - ) -> dict[str, object]: - request: Final = super().transform_rerank_request(model, optional_rerank_params, headers, litellm_params) - return { - "model": model, - "input": {"query": request["query"], "documents": request["documents"]}, - "parameters": { - ("instruct" if name == "instruction" else name): optional_rerank_params[name] - for name in ("top_n", "return_documents", "instruction") - if optional_rerank_params.get(name) is not None - }, - } - - def _get_request_query(self, request_data: Mapping[str, object]) -> object: - return ( - TypeAdapter(Mapping[str, object]) - .validate_python(request_data.get("input", MappingProxyType({}))) - .get("query") - ) - - def _get_response_fields( - self, response_json: Mapping[str, object], usage: DashScopeRerankUsage - ) -> tuple[object, object, int | None]: - output: Final = TypeAdapter(Mapping[str, object]).validate_python( - response_json.get("output", MappingProxyType({})) - ) - return output.get("results"), response_json.get("request_id"), usage.get("prompt_tokens") diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 7bf7d117d67..90b706bc199 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -3,16 +3,16 @@ Transformation logic for DashScope's OpenAI-compatible /v1/reranks API. Supports - qwen3-rerank +- qwen3.7-text-rerank -(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — have not been -validated against this transformer. Behavior with those models is undefined.) - -The native qwen3.7-text-rerank protocol is implemented in native_transformation.py. +(Other DashScope rerankers — gte-rerank-v2 / qwen3-vl-rerank — share the same +endpoint but have not been validated against this transformer. Behavior with +those models is undefined.) Endpoint - https://dashscope.aliyuncs.com/compatible-api/v1/reranks -Note: chat/embed live under `/compatible-mode/v1/`, but qwen3-rerank's +Note: chat/embed live under `/compatible-mode/v1/`, but DashScope's rerank route is exposed under `/compatible-api/v1/reranks` per the docs. A chat-shaped `.aliyuncs.com/compatible-mode/v1` base reaching this config (the chat default from `get_llm_provider`, or a `DASHSCOPE_API_BASE` env var) is redirected to @@ -28,12 +28,9 @@ Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api """ from collections.abc import Mapping -from types import MappingProxyType from typing import Final import httpx -from pydantic import TypeAdapter, ValidationError -from typing_extensions import ReadOnly, TypedDict from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -41,6 +38,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( + OptionalRerankParams, RerankBilledUnits, RerankResponse, RerankResponseMeta, @@ -52,17 +50,12 @@ from ..common_utils import DashScopeError, resolve_dashscope_family_rerank_api_b DEFAULT_RERANK_URL: Final = "https://dashscope.aliyuncs.com/compatible-api/v1/reranks" -class DashScopeRerankUsage(TypedDict, total=False): - prompt_tokens: ReadOnly[int | None] - total_tokens: ReadOnly[int | None] - - class DashScopeRerankConfig(BaseRerankConfig): """ Reference: https://help.aliyun.com/zh/model-studio/text-rerank-api - Targets DashScope's qwen3-rerank model. Request fields: model, query, - documents, top_n, return_documents. Response: results[].index, + Targets DashScope's qwen3-rerank and qwen3.7-text-rerank models. Request fields: + model, query, documents, top_n, return_documents, instruct. Response: results[].index, results[].relevance_score, optionally results[].document.text (when return_documents=true), plus a top-level usage.total_tokens counter. """ @@ -85,7 +78,7 @@ class DashScopeRerankConfig(BaseRerankConfig): self, api_base: str | None, model: str, - optional_params: Mapping[str, object] | None = None, + optional_params: dict | None = None, ) -> str: resolved_api_base: Final = self._resolve_rerank_api_base(api_base) if resolved_api_base == DEFAULT_RERANK_URL: @@ -103,12 +96,12 @@ class DashScopeRerankConfig(BaseRerankConfig): def validate_environment( self, - headers: Mapping[str, object], + headers: dict, model: str, api_key: str | None = None, - optional_params: Mapping[str, object] | None = None, + optional_params: dict | None = None, litellm_params: Mapping[str, object] | None = None, - ) -> dict[str, object]: + ) -> dict: return { "Authorization": f"Bearer {self._resolve_api_key(api_key)}", "accept": "application/json", @@ -117,7 +110,7 @@ class DashScopeRerankConfig(BaseRerankConfig): } def get_supported_cohere_rerank_params(self, model: str) -> list[str]: - return ["query", "documents", "top_n", "return_documents"] + return ["query", "documents", "top_n", "return_documents", "instruction"] def map_cohere_rerank_params( self, @@ -134,17 +127,14 @@ class DashScopeRerankConfig(BaseRerankConfig): max_tokens_per_doc: int | None = None, instruction: str | None = None, ) -> dict[str, object]: - params: Final = MappingProxyType( - { - "query": query, - "documents": documents, - "top_n": top_n, - "return_documents": return_documents, - "instruction": instruction, - } + params: Final[OptionalRerankParams] = OptionalRerankParams( + query=query, + documents=documents, + top_n=top_n, + return_documents=return_documents, + instruction=instruction, ) - supported_params: Final = self.get_supported_cohere_rerank_params(model) - return {name: value for name, value in params.items() if value is not None and name in supported_params} + return {name: value for name, value in params.items() if value is not None} def transform_rerank_request( self, @@ -158,16 +148,18 @@ class DashScopeRerankConfig(BaseRerankConfig): if "documents" not in optional_rerank_params: raise ValueError("documents is required for DashScope rerank") - request: Final[dict[str, object]] = { - "model": model, - "query": optional_rerank_params["query"], - "documents": optional_rerank_params["documents"], + return { + name: value + for name, value in ( + ("model", model), + ("query", optional_rerank_params["query"]), + ("documents", optional_rerank_params["documents"]), + ("top_n", optional_rerank_params.get("top_n")), + ("return_documents", optional_rerank_params.get("return_documents")), + ("instruct", optional_rerank_params.get("instruction")), + ) + if name in ("model", "query", "documents") or value is not None } - if optional_rerank_params.get("top_n") is not None: - request["top_n"] = optional_rerank_params["top_n"] - if optional_rerank_params.get("return_documents") is not None: - request["return_documents"] = optional_rerank_params["return_documents"] - return request def transform_rerank_response( self, @@ -177,22 +169,24 @@ class DashScopeRerankConfig(BaseRerankConfig): logging_obj: LiteLLMLoggingObj, api_key: str | None = None, request_data: dict | None = None, - optional_params: Mapping[str, object] | None = None, - litellm_params: Mapping[str, object] | None = None, + optional_params: dict | None = None, + litellm_params: dict | None = None, ) -> RerankResponse: - request: Final = request_data or MappingProxyType({}) + request_data = request_data or {} + optional_params = optional_params or {} + litellm_params = litellm_params or {} try: - response_json: Final = TypeAdapter(Mapping[str, object]).validate_json(raw_response.content) - except ValidationError as exc: + response_json: Final = raw_response.json() + except Exception: raise DashScopeError( status_code=raw_response.status_code, message=raw_response.text, - ) from exc + ) logging_obj.post_call( - input=self._get_request_query(request), + input=request_data.get("query"), api_key=api_key, - additional_args={"complete_input_dict": request}, + additional_args={"complete_input_dict": request_data}, original_response=response_json, ) @@ -200,26 +194,23 @@ class DashScopeRerankConfig(BaseRerankConfig): if "code" in response_json and "results" not in response_json: raise DashScopeError( status_code=raw_response.status_code, - message=str(response_json.get("message", response_json)), + message=response_json.get("message", str(response_json)), ) - usage: Final = TypeAdapter(DashScopeRerankUsage).validate_python( - response_json.get("usage") or MappingProxyType({}) - ) - results, response_id, input_tokens = self._get_response_fields(response_json, usage) + results: Final = response_json.get("results") if results is None: raise DashScopeError( status_code=raw_response.status_code, message=f"No results in DashScope rerank response: {response_json}", ) - # Both protocols return: + # qwen3-rerank returns: # {"index": int, "relevance_score": float} # plus, when return_documents=true was sent: # "document": {"text": "..."} # which already matches LiteLLM's RerankResponseDocument shape. transformed_results: Final[list[dict]] = [] - for r in TypeAdapter(tuple[Mapping[str, object], ...]).validate_python(results): + for r in results: item: dict[str, object] = { "index": r["index"], "relevance_score": r["relevance_score"], @@ -232,28 +223,18 @@ class DashScopeRerankConfig(BaseRerankConfig): item["document"] = {"text": doc} transformed_results.append(item) - billed_units: Final = RerankBilledUnits(total_tokens=usage.get("total_tokens")) - tokens: Final = RerankTokens(input_tokens=input_tokens) + usage: Final = response_json.get("usage") or {} + total_tokens: Final = usage.get("total_tokens") + billed_units: Final = RerankBilledUnits(total_tokens=total_tokens) + tokens: Final = RerankTokens(input_tokens=total_tokens) meta: Final = RerankResponseMeta(billed_units=billed_units, tokens=tokens) - return RerankResponse.model_validate( - MappingProxyType( - { - "id": response_id or str(uuid.uuid4()), - "results": transformed_results, - "meta": meta, - } - ) + return RerankResponse( + id=response_json.get("id") or str(uuid.uuid4()), + results=transformed_results, + meta=meta, ) - def _get_request_query(self, request_data: Mapping[str, object]) -> object: - return request_data.get("query") - - def _get_response_fields( - self, response_json: Mapping[str, object], usage: DashScopeRerankUsage - ) -> tuple[object, object, int | None]: - return response_json.get("results"), response_json.get("id"), usage.get("total_tokens") - def get_error_class( self, error_message: str, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 52b76e4103a..b7726290f0e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15015,14 +15015,6 @@ } ] }, - "dashscope/qwen3.7-text-rerank": { - "litellm_provider": "dashscope", - "mode": "rerank", - "provider_specific_entry": { - "rerank_api": "native" - }, - "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api" - }, "dashscope/qwen-turbo": { "input_cost_per_token": 5e-08, "litellm_provider": "dashscope", diff --git a/litellm/utils.py b/litellm/utils.py index 25a4c2e8953..a765e1b1246 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8508,7 +8508,7 @@ class ProviderConfigManager: get_dashscope_family_rerank_config, ) - return get_dashscope_family_rerank_config(provider.value, model, api_base) + return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 52b76e4103a..b7726290f0e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15015,14 +15015,6 @@ } ] }, - "dashscope/qwen3.7-text-rerank": { - "litellm_provider": "dashscope", - "mode": "rerank", - "provider_specific_entry": { - "rerank_api": "native" - }, - "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api" - }, "dashscope/qwen-turbo": { "input_cost_per_token": 5e-08, "litellm_provider": "dashscope", diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index da6ed63a83c..207c71c780d 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -7,10 +7,9 @@ from unittest.mock import MagicMock import httpx import pytest -import respx -from litellm.llms.dashscope.common_utils import DashScopeError, get_dashscope_family_rerank_config +from litellm.llms.dashscope.common_utils import DashScopeError from litellm.llms.dashscope.rerank.transformation import ( DEFAULT_RERANK_URL, DashScopeRerankConfig, @@ -112,6 +111,7 @@ class TestDashScopeRerankRequest: "documents", "top_n", "return_documents", + "instruction", ] def test_map_params_drops_unsupported(self): @@ -128,7 +128,6 @@ class TestDashScopeRerankRequest: return_documents=True, max_chunks_per_doc=5, max_tokens_per_doc=100, - instruction="Unsupported on the compatible protocol", ) assert params == { "query": "什么是文本排序模型", @@ -300,32 +299,30 @@ class TestDashScopeRerankResponse: ) assert out.id is not None and len(out.id) > 0 - @pytest.mark.parametrize("model", ["qwen3-rerank", "qwen3.7-text-rerank"]) - def test_error_envelope_raises(self, model): + def test_error_envelope_raises(self): body = { "code": "InvalidApiKey", "message": "Invalid API-key provided.", "request_id": "fb53", } with pytest.raises(DashScopeError) as exc_info: - get_dashscope_family_rerank_config("dashscope", model).transform_rerank_response( - model=model, + self.config.transform_rerank_response( + model="qwen3-rerank", raw_response=self._resp(body, status_code=401), model_response=RerankResponse(), logging_obj=self.logging, ) assert "Invalid API-key provided." in str(exc_info.value) - @pytest.mark.parametrize("model", ["qwen3-rerank", "qwen3.7-text-rerank"]) - def test_non_json_response_raises(self, model): + def test_non_json_response_raises(self): bad = httpx.Response( status_code=500, content=b"bad gateway", request=httpx.Request("POST", "https://example.com"), ) with pytest.raises(DashScopeError): - get_dashscope_family_rerank_config("dashscope", model).transform_rerank_response( - model=model, + self.config.transform_rerank_response( + model="qwen3-rerank", raw_response=bad, model_response=RerankResponse(), logging_obj=self.logging, @@ -355,280 +352,36 @@ class TestProviderConfigManagerDispatch: @pytest.mark.asyncio @pytest.mark.parametrize("is_async", [False, True]) -@pytest.mark.parametrize("return_documents", [False, True]) -@pytest.mark.parametrize( - "provider,host", - [ - ("dashscope", "dashscope.aliyuncs.com"), - ("qwencloud", "dashscope-intl.aliyuncs.com"), - ("qwen_ai_platform", "dashscope.aliyuncs.com"), - ], -) -async def test_qwen37_rerank_public_call( - is_async, return_documents, provider, host, respx_mock: respx.MockRouter, monkeypatch -): - import litellm - - monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) - monkeypatch.delenv("DASHSCOPE_API_BASE_RERANK", raising=False) - monkeypatch.delenv(f"{provider.upper()}_API_BASE", raising=False) - monkeypatch.delenv(f"{provider.upper()}_API_BASE_RERANK", raising=False) - monkeypatch.setenv(f"{provider.upper()}_API_KEY", "fake-brand-key") - monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") - route = respx_mock.post(f"https://{host}/api/v1/services/rerank/text-rerank/text-rerank") - results = [{"index": 1, "relevance_score": 0.88, **({"document": {"text": "answer"}} if return_documents else {})}] - route.respond( - 200, - json={ - "output": {"results": results}, - "usage": {"prompt_tokens": 237, "total_tokens": 261, "details": {"provider_metadata": True}}, - "request_id": "qwen37-request-id", - }, - ) - kwargs = { - "model": f"{provider}/qwen3.7-text-rerank", - "query": "question", - "documents": ["unrelated", "answer"], - "top_n": 1, - "return_documents": return_documents, - "instruction": "Retrieve semantically similar text.", - } - - response = await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) - - assert json.loads(route.calls[0].request.content) == { - "model": "qwen3.7-text-rerank", - "input": {"query": "question", "documents": ["unrelated", "answer"]}, - "parameters": { - "top_n": 1, - "return_documents": return_documents, - "instruct": "Retrieve semantically similar text.", - }, - } - assert route.calls[0].request.headers["authorization"] == "Bearer fake-brand-key" - assert response.id == "qwen37-request-id" - assert response.results == results - assert response.meta == {"billed_units": {"total_tokens": 261}, "tokens": {"input_tokens": 237}} - - -@pytest.mark.parametrize( - "api_base", - [ - "https://proxy.example/api/v1", - "https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank/", - ], -) -def test_qwen37_rerank_custom_url(api_base): - assert get_dashscope_family_rerank_config("dashscope", "qwen3.7-text-rerank").get_complete_url( - api_base, "qwen3.7-text-rerank" - ) == ("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") - - -@pytest.mark.parametrize( - "provider,host", - [("dashscope", "dashscope-intl.aliyuncs.com"), ("qwencloud", "dashscope.aliyuncs.com")], -) -def test_qwen37_rerank_explicit_region(provider, host): - config = get_dashscope_family_rerank_config(provider, "qwen3.7-text-rerank") - assert config.get_complete_url(f"https://{host}/compatible-mode/v1", "qwen3.7-text-rerank") == ( - f"https://{host}/api/v1/services/rerank/text-rerank/text-rerank" - ) - - -def test_qwen37_rerank_response_logging(): - config = get_dashscope_family_rerank_config("dashscope", "qwen3.7-text-rerank") - logging = MagicMock() - request = { - "model": "qwen3.7-text-rerank", - "input": {"query": "question", "documents": ["answer"]}, - "parameters": {}, - } - payload = {"request_id": "request-id", "output": {"results": [{"index": 0, "relevance_score": 0.88}]}} - - response = config.transform_rerank_response( - model="qwen3.7-text-rerank", - raw_response=httpx.Response(200, json=payload), - model_response=RerankResponse(), - logging_obj=logging, - request_data=request, - ) - - logging.post_call.assert_called_once_with( - input="question", api_key=None, additional_args={"complete_input_dict": request}, original_response=payload - ) - assert response.id == "request-id" - assert response.results == [{"index": 0, "relevance_score": 0.88}] - - @pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) -def test_qwen37_rerank_environment_url(provider, respx_mock: respx.MockRouter, monkeypatch): - import litellm - - monkeypatch.delenv("DASHSCOPE_API_BASE", raising=False) - monkeypatch.delenv(f"{provider.upper()}_API_BASE", raising=False) - monkeypatch.setenv(f"{provider.upper()}_API_BASE_RERANK", "https://proxy.example/api/v1") - route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") - route.respond( - 200, - json={ - "output": {"results": [{"index": 0, "relevance_score": 0.88}]}, - "request_id": "all-results", - "usage": {"prompt_tokens": 10, "total_tokens": 10}, - }, - ) - - response = litellm.rerank( - model=f"{provider}/qwen3.7-text-rerank", - query="question", - documents=["answer"], - return_documents=None, - api_key="fake-dashscope-key", - ) - - assert json.loads(route.calls[0].request.content) == { - "model": "qwen3.7-text-rerank", - "input": {"query": "question", "documents": ["answer"]}, - "parameters": {}, - } - assert response.results == [{"index": 0, "relevance_score": 0.88}] - assert response.meta["tokens"]["input_tokens"] == 10 - - -@pytest.mark.asyncio -@pytest.mark.parametrize("is_async", [False, True]) -async def test_qwen37_rerank_preserves_provider_error(is_async, respx_mock: respx.MockRouter, monkeypatch): +@pytest.mark.parametrize("model", ["qwen3-rerank", "qwen3.7-text-rerank"]) +@pytest.mark.parametrize("instruction", [None, "", "Retrieve semantically similar text."]) +async def test_instruction_reaches_compatible_endpoint(provider, model, is_async, instruction, respx_mock, monkeypatch): import litellm + monkeypatch.setenv(f"{provider.upper()}_API_BASE", "https://rerank.example/compatible-api/v1/reranks") monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") - route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") - route.respond( - 400, - json={"code": "InvalidParameter", "message": "documents must not be empty", "request_id": "invalid-documents"}, - ) + route = respx_mock.post("https://rerank.example/compatible-api/v1/reranks") + route.respond(200, json={"id": "ranking", "results": [{"index": 0, "relevance_score": 0.9}]}) kwargs = { - "model": "dashscope/qwen3.7-text-rerank", - "query": "question", - "documents": [], - "api_key": "fake-dashscope-key", - "api_base": "https://proxy.example/api/v1", - } - - with pytest.raises(litellm.BadRequestError, match="documents must not be empty") as error: - await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) - - assert error.value.status_code == 400 - assert "DashscopeException" in str(error.value) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("is_async", [False, True]) -@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) -@pytest.mark.parametrize("base_case", ["rerank_override", "explicit", "explicit_default", "general_only"]) -async def test_native_rerank_base_precedence(provider, is_async, base_case, respx_mock, monkeypatch): - import litellm - - prefix = provider.upper() - default_host = "dashscope-intl.aliyuncs.com" if provider == "qwencloud" else "dashscope.aliyuncs.com" - monkeypatch.setenv(f"{prefix}_API_BASE", "https://chat.example/compatible-mode/v1") - monkeypatch.delenv(f"{prefix}_API_BASE_RERANK", raising=False) - monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") - if base_case != "general_only": - monkeypatch.setenv(f"{prefix}_API_BASE_RERANK", "https://rerank.example/api/v1") - explicit_base = { - "rerank_override": None, - "explicit": "https://explicit.example/api/v1", - "explicit_default": f"https://{default_host}/compatible-mode/v1", - "general_only": None, - }[base_case] - expected_host = { - "rerank_override": "rerank.example", - "explicit": "explicit.example", - "explicit_default": default_host, - "general_only": "chat.example", - }[base_case] - route = respx_mock.post(f"https://{expected_host}/api/v1/services/rerank/text-rerank/text-rerank") - route.respond( - 200, json={"request_id": "base-precedence", "output": {"results": [{"index": 0, "relevance_score": 0.9}]}} - ) - kwargs = { - "model": f"{provider}/qwen3.7-text-rerank", + "model": f"{provider}/{model}", "query": "question", "documents": ["answer"], - "api_key": "fake-review-key", - "api_base": explicit_base, + "top_n": 1, + "return_documents": False, + "instruction": instruction, + "api_key": "test-key", } response = await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) - assert json.loads(route.calls[0].request.content)["input"] == {"query": "question", "documents": ["answer"]} - assert response.id == "base-precedence" - assert response.results == [{"index": 0, "relevance_score": 0.9}] - - -@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) -@pytest.mark.parametrize("rerank_api", ["native", "compatible"]) -def test_rerank_protocol_uses_runtime_model_metadata(provider, rerank_api, respx_mock, monkeypatch): - import litellm - - model = "custom-rerank" if rerank_api == "native" else "qwen3.7-text-rerank" - monkeypatch.setitem( - litellm.model_cost, - model if provider == "dashscope" and rerank_api == "native" else f"{provider}/{model}", - { - "litellm_provider": provider, - "mode": "rerank", - "provider_specific_entry": {"rerank_api": rerank_api}, - }, - ) - results = [{"index": 0, "relevance_score": 0.9}] - url = ( - "https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank" - if rerank_api == "native" - else "https://proxy.example/api/v1/reranks" - ) - route = respx_mock.post(url) - route.respond( - 200, - json={"request_id": "metadata", "output": {"results": results}} - if rerank_api == "native" - else {"id": "metadata", "results": results}, - ) - - response = litellm.rerank( - model=f"{provider}/{model}", - query="question", - documents=["answer"], - api_key="fake-key", - api_base="https://proxy.example/api/v1", - return_documents=None, - ) - - assert json.loads(route.calls[0].request.content) == ( - {"model": model, "input": {"query": "question", "documents": ["answer"]}, "parameters": {}} - if rerank_api == "native" - else {"model": model, "query": "question", "documents": ["answer"]} - ) - assert response.id == "metadata" - assert response.results == results - - -@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) -def test_rerank_uses_bundled_metadata_when_remote_map_lacks_model(provider, respx_mock, monkeypatch): - import litellm - - for prefix in ("dashscope", "qwencloud", "qwen_ai_platform"): - monkeypatch.delitem(litellm.model_cost, f"{prefix}/qwen3.7-text-rerank", raising=False) - route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") - route.respond(200, json={"request_id": "bundled", "output": {"results": [{"index": 0, "relevance_score": 0.9}]}}) - - response = litellm.rerank( - model=f"{provider}/qwen3.7-text-rerank", - query="question", - documents=["answer"], - api_key="fake-key", - api_base="https://proxy.example/api/v1", - ) - - assert json.loads(route.calls[0].request.content)["input"] == {"query": "question", "documents": ["answer"]} - assert response.id == "bundled" + body = json.loads(route.calls[0].request.content) + assert body == { + "model": model, + "query": "question", + "documents": ["answer"], + "top_n": 1, + "return_documents": False, + **({"instruct": instruction} if instruction is not None else {}), + } + assert response.id == "ranking" assert response.results == [{"index": 0, "relevance_score": 0.9}] From d43146abe24ae409e02fc9a26ab9b977676f6359 Mon Sep 17 00:00:00 2001 From: jayson wang Date: Fri, 11 Sep 2026 10:38:11 +0800 Subject: [PATCH 5/6] refactor(dashscope): minimize rerank instruction diff --- .../llms/dashscope/rerank/transformation.py | 57 ++++++++++--------- .../test_dashscope_rerank_transformation.py | 11 +--- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 90b706bc199..a3f8ef95e32 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -28,7 +28,7 @@ Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api """ from collections.abc import Mapping -from typing import Final +from typing import Any, Final import httpx @@ -54,8 +54,8 @@ class DashScopeRerankConfig(BaseRerankConfig): """ Reference: https://help.aliyun.com/zh/model-studio/text-rerank-api - Targets DashScope's qwen3-rerank and qwen3.7-text-rerank models. Request fields: - model, query, documents, top_n, return_documents, instruct. Response: results[].index, + Targets DashScope's qwen3-rerank and qwen3.7-text-rerank. Request fields: model, query, + documents, top_n, return_documents, instruct. Response: results[].index, results[].relevance_score, optionally results[].document.text (when return_documents=true), plus a top-level usage.total_tokens counter. """ @@ -109,16 +109,16 @@ class DashScopeRerankConfig(BaseRerankConfig): **headers, } - def get_supported_cohere_rerank_params(self, model: str) -> list[str]: + def get_supported_cohere_rerank_params(self, model: str) -> list: return ["query", "documents", "top_n", "return_documents", "instruction"] def map_cohere_rerank_params( self, - non_default_params: Mapping[str, object] | None, + non_default_params: dict | None, model: str, drop_params: bool, query: str, - documents: list[str | dict[str, object]], + documents: list[str | dict[str, Any]], custom_llm_provider: str | None = None, top_n: int | None = None, rank_fields: list[str] | None = None, @@ -126,40 +126,45 @@ class DashScopeRerankConfig(BaseRerankConfig): max_chunks_per_doc: int | None = None, max_tokens_per_doc: int | None = None, instruction: str | None = None, - ) -> dict[str, object]: + ) -> dict: + # qwen3-rerank accepts query/documents/top_n/return_documents/instruct. The + # rest (rank_fields, max_*_per_doc) are silently dropped. params: Final[OptionalRerankParams] = OptionalRerankParams( query=query, documents=documents, - top_n=top_n, - return_documents=return_documents, - instruction=instruction, ) - return {name: value for name, value in params.items() if value is not None} + if top_n is not None: + params["top_n"] = top_n + if return_documents is not None: + params["return_documents"] = return_documents + if instruction is not None: + params["instruction"] = instruction + return dict(params) def transform_rerank_request( self, model: str, - optional_rerank_params: Mapping[str, object], - headers: Mapping[str, object], - litellm_params: Mapping[str, object] | None = None, - ) -> dict[str, object]: + optional_rerank_params: dict, + headers: dict, + litellm_params: dict | None = None, + ) -> dict: if "query" not in optional_rerank_params: raise ValueError("query is required for DashScope rerank") if "documents" not in optional_rerank_params: raise ValueError("documents is required for DashScope rerank") - return { - name: value - for name, value in ( - ("model", model), - ("query", optional_rerank_params["query"]), - ("documents", optional_rerank_params["documents"]), - ("top_n", optional_rerank_params.get("top_n")), - ("return_documents", optional_rerank_params.get("return_documents")), - ("instruct", optional_rerank_params.get("instruction")), - ) - if name in ("model", "query", "documents") or value is not None + request: Final[dict[str, object]] = { + "model": model, + "query": optional_rerank_params["query"], + "documents": optional_rerank_params["documents"], } + if optional_rerank_params.get("top_n") is not None: + request["top_n"] = optional_rerank_params["top_n"] + if optional_rerank_params.get("return_documents") is not None: + request["return_documents"] = optional_rerank_params["return_documents"] + if optional_rerank_params.get("instruction") is not None: + request["instruct"] = optional_rerank_params["instruction"] + return request def transform_rerank_response( self, diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index 207c71c780d..637a03223d9 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -375,13 +375,8 @@ async def test_instruction_reaches_compatible_endpoint(provider, model, is_async response = await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) body = json.loads(route.calls[0].request.content) - assert body == { - "model": model, - "query": "question", - "documents": ["answer"], - "top_n": 1, - "return_documents": False, - **({"instruct": instruction} if instruction is not None else {}), - } + assert body.get("instruct") == instruction + assert ("instruct" in body) == (instruction is not None) + assert "instruction" not in body assert response.id == "ranking" assert response.results == [{"index": 0, "relevance_score": 0.9}] From 23d86c054c7d4c054b49007d3626f53fa266e961 Mon Sep 17 00:00:00 2001 From: Jayson Wang Date: Sat, 12 Sep 2026 17:59:20 +0800 Subject: [PATCH 6/6] Update litellm/llms/dashscope/rerank/transformation.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/llms/dashscope/rerank/transformation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index a3f8ef95e32..50800b70a7e 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -127,8 +127,6 @@ class DashScopeRerankConfig(BaseRerankConfig): max_tokens_per_doc: int | None = None, instruction: str | None = None, ) -> dict: - # qwen3-rerank accepts query/documents/top_n/return_documents/instruct. The - # rest (rank_fields, max_*_per_doc) are silently dropped. params: Final[OptionalRerankParams] = OptionalRerankParams( query=query, documents=documents,