From 331db73127ba33990f0374ab842058bf8c22ab7f Mon Sep 17 00:00:00 2001 From: jayson wang Date: Tue, 8 Sep 2026 12:08:22 +0800 Subject: [PATCH] fix(dashscope): honor rerank base overrides and model metadata --- cookbook/dashscope_rerank.md | 44 ------- litellm/llms/dashscope/common_utils.py | 57 ++++++++- litellm/llms/dashscope/qwen_ai_platform.py | 6 +- litellm/llms/dashscope/qwencloud.py | 6 +- .../dashscope/rerank/native_transformation.py | 12 +- .../llms/dashscope/rerank/transformation.py | 10 +- ...odel_prices_and_context_window_backup.json | 8 ++ litellm/utils.py | 2 +- model_prices_and_context_window.json | 8 ++ .../test_dashscope_rerank_transformation.py | 120 +++++++++++++++++- 10 files changed, 198 insertions(+), 75 deletions(-) delete mode 100644 cookbook/dashscope_rerank.md diff --git a/cookbook/dashscope_rerank.md b/cookbook/dashscope_rerank.md deleted file mode 100644 index 6a7173b6cba..00000000000 --- a/cookbook/dashscope_rerank.md +++ /dev/null @@ -1,44 +0,0 @@ -# Qwen3.7 text reranking - -Use `dashscope/qwen3.7-text-rerank` with LiteLLM's rerank interface and a Beijing DashScope API key - -```python -import litellm - -response = litellm.rerank( - model="dashscope/qwen3.7-text-rerank", - query="How can I reset my password?", - documents=[ - "The weather is sunny today.", - "Open account settings and select Reset password.", - "How do I change my password?", - ], - top_n=2, - return_documents=True, - instruction="Retrieve semantically similar text.", -) -``` - -Set `DASHSCOPE_API_KEY` in the environment or pass `api_key` explicitly. The asynchronous equivalent is `await litellm.arerank(...)` - -For the proxy, add a model entry and send the same query, documents and options to `/v1/rerank` using its configured model alias - -```yaml -model_list: - - model_name: qwen37-rerank - litellm_params: - model: dashscope/qwen3.7-text-rerank - api_key: os.environ/DASHSCOPE_API_KEY - model_info: - mode: rerank -``` - -LiteLLM sends the native DashScope request to `https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank`. An explicit `api_base` or `DASHSCOPE_API_BASE_RERANK` can select a different host, an `/api/v1` base, or the complete native endpoint. Chat-compatible DashScope paths are converted to the native path for this model - -`instruction` maps to DashScope's `parameters.instruct`. When omitted, the provider chooses its default relevance criterion. `top_n` and `return_documents` map to the corresponding native parameters. Live calls on 2026-09-08 confirmed that this model returns `document.text` when `return_documents=True`, despite the official parameter table omitting it from the supported-model list - -The response contains the provider request ID, original document indices, relevance scores and requested document text. `meta.tokens.input_tokens` comes from `usage.prompt_tokens`; `meta.billed_units.total_tokens` comes from `usage.total_tokens`. These counters do not add a model price or dollar-cost calculation - -The existing `dashscope/qwen3-rerank` model retains its compatible protocol. This change does not add multimodal reranking or reinterpret structured candidate documents - -Protocol reference: [DashScope text rerank API](https://help.aliyun.com/zh/model-studio/text-rerank-api) diff --git a/litellm/llms/dashscope/common_utils.py b/litellm/llms/dashscope/common_utils.py index 254c845d509..862a5f60bec 100644 --- a/litellm/llms/dashscope/common_utils.py +++ b/litellm/llms/dashscope/common_utils.py @@ -2,10 +2,15 @@ Common utilities for the DashScope LLM provider. """ -from typing import TYPE_CHECKING, Final +from collections.abc import Mapping +from functools import lru_cache +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal import httpx +from pydantic import TypeAdapter +import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -34,15 +39,59 @@ def get_dashscope_family_embedding_config(custom_llm_provider: str) -> "BaseEmbe return DashScopeEmbeddingConfig() -def get_dashscope_family_rerank_config(custom_llm_provider: str, model: str) -> "BaseRerankConfig": +def get_dashscope_family_rerank_config( + custom_llm_provider: str, model: str, api_base: str | None = None +) -> "BaseRerankConfig": provider_config: Final = _get_dashscope_family_rerank_provider_config(custom_llm_provider) - if model == "qwen3.7-text-rerank": + model_cost: Final[Mapping[str, object]] = litellm.model_cost + runtime_api: Final = next( + ( + declared_api + for key in (f"{custom_llm_provider}/{model}", f"dashscope/{model}", model) + if (declared_api := _rerank_api_from_model_info(model_cost.get(key))) is not None + ), + None, + ) + rerank_api: Final = runtime_api or _bundled_dashscope_rerank_apis().get(f"dashscope/{model}") + if rerank_api == "native": from litellm.llms.dashscope.rerank.native_transformation import DashScopeNativeRerankConfig - return DashScopeNativeRerankConfig(provider_config) + return DashScopeNativeRerankConfig( + provider_config, api_base=api_base or get_secret_str(f"{custom_llm_provider.upper()}_API_BASE_RERANK") + ) return provider_config +def _rerank_api_from_model_info(raw_model_info: object) -> str | None: + if raw_model_info is None: + return None + model_info: Final = TypeAdapter(Mapping[str, object]).validate_python(raw_model_info) + provider_info: Final = model_info.get("provider_specific_entry") + if provider_info is None: + return None + metadata: Final = TypeAdapter(Mapping[str, object]).validate_python(provider_info) + rerank_api: Final[str | None] = TypeAdapter(Literal["native", "compatible"] | None).validate_python( + metadata.get("rerank_api") + ) + return rerank_api + + +@lru_cache(maxsize=1) +def _bundled_dashscope_rerank_apis() -> Mapping[str, str]: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + model_infos: Final = TypeAdapter(Mapping[str, object]).validate_json( + GetModelCostMap.read_local_model_cost_map_text() + ) + return MappingProxyType( + { + key: rerank_api + for key, model_info in model_infos.items() + if key.startswith("dashscope/") and (rerank_api := _rerank_api_from_model_info(model_info)) is not None + } + ) + + def _get_dashscope_family_rerank_provider_config(custom_llm_provider: str) -> "DashScopeRerankConfig": if custom_llm_provider == "qwencloud": from litellm.llms.dashscope.qwencloud import QwenCloudRerankConfig diff --git a/litellm/llms/dashscope/qwen_ai_platform.py b/litellm/llms/dashscope/qwen_ai_platform.py index 8e98c989c48..9a44eaf574a 100644 --- a/litellm/llms/dashscope/qwen_ai_platform.py +++ b/litellm/llms/dashscope/qwen_ai_platform.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Final +from typing import Final from litellm.secret_managers.main import get_secret_str @@ -47,13 +47,11 @@ class QwenAIPlatformEmbeddingConfig(DashScopeEmbeddingConfig): class QwenAIPlatformRerankConfig(DashScopeRerankConfig): - DEFAULT_RERANK_API_BASE: ClassVar[str] = QWEN_AI_PLATFORM_RERANK_API_BASE - def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwen_ai_platform_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE + return api_base or get_secret_str("QWEN_AI_PLATFORM_API_BASE_RERANK") or QWEN_AI_PLATFORM_RERANK_API_BASE class QwenAIPlatformImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/qwencloud.py b/litellm/llms/dashscope/qwencloud.py index 827477b3ffc..d8d53e340ef 100644 --- a/litellm/llms/dashscope/qwencloud.py +++ b/litellm/llms/dashscope/qwencloud.py @@ -1,4 +1,4 @@ -from typing import ClassVar, Final +from typing import Final from litellm.secret_managers.main import get_secret_str @@ -47,13 +47,11 @@ class QwenCloudEmbeddingConfig(DashScopeEmbeddingConfig): class QwenCloudRerankConfig(DashScopeRerankConfig): - DEFAULT_RERANK_API_BASE: ClassVar[str] = QWENCLOUD_RERANK_API_BASE - def _resolve_api_key(self, api_key: str | None) -> str: return _require_qwencloud_api_key(api_key) def _resolve_rerank_api_base(self, api_base: str | None) -> str: - return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE + return api_base or get_secret_str("QWENCLOUD_API_BASE_RERANK") or QWENCLOUD_RERANK_API_BASE class QwenCloudImageGenerationConfig(DashScopeImageGenerationConfig): diff --git a/litellm/llms/dashscope/rerank/native_transformation.py b/litellm/llms/dashscope/rerank/native_transformation.py index bae0433aee9..9e34375ab12 100644 --- a/litellm/llms/dashscope/rerank/native_transformation.py +++ b/litellm/llms/dashscope/rerank/native_transformation.py @@ -16,9 +16,9 @@ from .transformation import DashScopeRerankConfig, DashScopeRerankUsage class DashScopeNativeRerankConfig(DashScopeRerankConfig): - def __init__(self, provider_config: DashScopeRerankConfig) -> None: - # Reuse brand-specific credentials and hosts without duplicating alias subclasses. + def __init__(self, provider_config: DashScopeRerankConfig, api_base: str | None = None) -> None: self._provider_config: Final = provider_config + self._api_base: Final = api_base def _resolve_api_key(self, api_key: str | None) -> str: return self._provider_config._resolve_api_key(api_key) @@ -32,12 +32,7 @@ class DashScopeNativeRerankConfig(DashScopeRerankConfig): model: str, optional_params: Mapping[str, object] | None = None, ) -> str: - # Provider discovery supplies the brand's chat default even when no api_base was passed. - # Resolve the rerank-specific environment override before constructing the native path. - default_chat_base: Final = self._provider_config.DEFAULT_RERANK_API_BASE.replace( - "/compatible-api/v1/reranks", "/compatible-mode/v1" - ) - native_base: Final = self._resolve_rerank_api_base(None if api_base == default_chat_base else api_base) + native_base: Final = self._resolve_rerank_api_base(self._api_base or api_base) parsed: Final = urlsplit(native_base.rstrip("/")) if parsed.path.endswith("/services/rerank/text-rerank/text-rerank"): return urlunsplit(parsed) @@ -79,5 +74,4 @@ class DashScopeNativeRerankConfig(DashScopeRerankConfig): output: Final = TypeAdapter(Mapping[str, object]).validate_python( response_json.get("output", MappingProxyType({})) ) - # Native usage separates prompt_tokens from total_tokens; keep both provider counters. return output.get("results"), response_json.get("request_id"), usage.get("prompt_tokens") diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index d0f9a0b1283..b75a5042e9a 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -25,7 +25,7 @@ Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api from collections.abc import Mapping from types import MappingProxyType -from typing import ClassVar, Final +from typing import Final import httpx from pydantic import TypeAdapter, ValidationError @@ -61,12 +61,8 @@ class DashScopeRerankConfig(BaseRerankConfig): documents, top_n, return_documents. Response: results[].index, results[].relevance_score, optionally results[].document.text (when return_documents=true), plus a top-level usage.total_tokens counter. - - Brand aliases supply their own API key and base resolvers. """ - DEFAULT_RERANK_API_BASE: ClassVar[str] = DEFAULT_RERANK_URL - def __init__(self) -> None: pass @@ -81,7 +77,7 @@ class DashScopeRerankConfig(BaseRerankConfig): def _resolve_rerank_api_base(self, api_base: str | None) -> str: if api_base is not None: return api_base - return get_secret_str("DASHSCOPE_API_BASE_RERANK") or self.DEFAULT_RERANK_API_BASE + return get_secret_str("DASHSCOPE_API_BASE_RERANK") or DEFAULT_RERANK_URL def get_complete_url( self, @@ -136,7 +132,6 @@ class DashScopeRerankConfig(BaseRerankConfig): max_tokens_per_doc: int | None = None, instruction: str | None = None, ) -> dict[str, object]: - # rank_fields and max_*_per_doc have no supported mapping and are omitted. params: Final = MappingProxyType( { "query": query, @@ -255,7 +250,6 @@ class DashScopeRerankConfig(BaseRerankConfig): def _get_response_fields( self, response_json: Mapping[str, object], usage: DashScopeRerankUsage ) -> tuple[object, object, int | None]: - # The compatible API reports its input count only as total_tokens. return response_json.get("results"), response_json.get("id"), usage.get("total_tokens") def get_error_class( diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b1ffc1583e4..1633158b264 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14753,6 +14753,14 @@ } ] }, + "dashscope/qwen3.7-text-rerank": { + "litellm_provider": "dashscope", + "mode": "rerank", + "provider_specific_entry": { + "rerank_api": "native" + }, + "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api" + }, "dashscope/qwen-turbo": { "input_cost_per_token": 5e-08, "litellm_provider": "dashscope", diff --git a/litellm/utils.py b/litellm/utils.py index 7df926e7a20..c5de9435aee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8439,7 +8439,7 @@ class ProviderConfigManager: get_dashscope_family_rerank_config, ) - return get_dashscope_family_rerank_config(provider.value, model) + return get_dashscope_family_rerank_config(provider.value, model, api_base) return litellm.CohereRerankConfig() @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1ffc1583e4..1633158b264 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14753,6 +14753,14 @@ } ] }, + "dashscope/qwen3.7-text-rerank": { + "litellm_provider": "dashscope", + "mode": "rerank", + "provider_specific_entry": { + "rerank_api": "native" + }, + "source": "https://help.aliyun.com/zh/model-studio/text-rerank-api" + }, "dashscope/qwen-turbo": { "input_cost_per_token": 5e-08, "litellm_provider": "dashscope", diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py index 26f7f9c2226..7b1f6d9dc09 100644 --- a/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py +++ b/tests/test_litellm/llms/dashscope/test_dashscope_rerank_transformation.py @@ -414,7 +414,11 @@ def test_qwen37_rerank_explicit_region(provider, host): def test_qwen37_rerank_response_logging(): config = get_dashscope_family_rerank_config("dashscope", "qwen3.7-text-rerank") logging = MagicMock() - request = {"model": "qwen3.7-text-rerank", "input": {"query": "question", "documents": ["answer"]}, "parameters": {}} + request = { + "model": "qwen3.7-text-rerank", + "input": {"query": "question", "documents": ["answer"]}, + "parameters": {}, + } payload = {"request_id": "request-id", "output": {"results": [{"index": 0, "relevance_score": 0.88}]}} response = config.transform_rerank_response( @@ -490,3 +494,117 @@ async def test_qwen37_rerank_preserves_provider_error(is_async, respx_mock: resp assert error.value.status_code == 400 assert "DashscopeException" in str(error.value) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +@pytest.mark.parametrize("base_case", ["rerank_override", "explicit", "explicit_default", "general_only"]) +async def test_native_rerank_base_precedence(provider, is_async, base_case, respx_mock, monkeypatch): + import litellm + + prefix = provider.upper() + default_host = "dashscope-intl.aliyuncs.com" if provider == "qwencloud" else "dashscope.aliyuncs.com" + monkeypatch.setenv(f"{prefix}_API_BASE", "https://chat.example/compatible-mode/v1") + monkeypatch.delenv(f"{prefix}_API_BASE_RERANK", raising=False) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + if base_case != "general_only": + monkeypatch.setenv(f"{prefix}_API_BASE_RERANK", "https://rerank.example/api/v1") + explicit_base = { + "rerank_override": None, + "explicit": "https://explicit.example/api/v1", + "explicit_default": f"https://{default_host}/compatible-mode/v1", + "general_only": None, + }[base_case] + expected_host = { + "rerank_override": "rerank.example", + "explicit": "explicit.example", + "explicit_default": default_host, + "general_only": "chat.example", + }[base_case] + route = respx_mock.post(f"https://{expected_host}/api/v1/services/rerank/text-rerank/text-rerank") + route.respond( + 200, json={"request_id": "base-precedence", "output": {"results": [{"index": 0, "relevance_score": 0.9}]}} + ) + kwargs = { + "model": f"{provider}/qwen3.7-text-rerank", + "query": "question", + "documents": ["answer"], + "api_key": "fake-review-key", + "api_base": explicit_base, + } + + response = await litellm.arerank(**kwargs) if is_async else litellm.rerank(**kwargs) + + assert json.loads(route.calls[0].request.content)["input"] == {"query": "question", "documents": ["answer"]} + assert response.id == "base-precedence" + assert response.results == [{"index": 0, "relevance_score": 0.9}] + + +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +@pytest.mark.parametrize("rerank_api", ["native", "compatible"]) +def test_rerank_protocol_uses_runtime_model_metadata(provider, rerank_api, respx_mock, monkeypatch): + import litellm + + model = "custom-rerank" if rerank_api == "native" else "qwen3.7-text-rerank" + monkeypatch.setitem( + litellm.model_cost, + model if provider == "dashscope" and rerank_api == "native" else f"{provider}/{model}", + { + "litellm_provider": provider, + "mode": "rerank", + "provider_specific_entry": {"rerank_api": rerank_api}, + }, + ) + results = [{"index": 0, "relevance_score": 0.9}] + url = ( + "https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank" + if rerank_api == "native" + else "https://proxy.example/api/v1/reranks" + ) + route = respx_mock.post(url) + route.respond( + 200, + json={"request_id": "metadata", "output": {"results": results}} + if rerank_api == "native" + else {"id": "metadata", "results": results}, + ) + + response = litellm.rerank( + model=f"{provider}/{model}", + query="question", + documents=["answer"], + api_key="fake-key", + api_base="https://proxy.example/api/v1", + return_documents=None, + ) + + assert json.loads(route.calls[0].request.content) == ( + {"model": model, "input": {"query": "question", "documents": ["answer"]}, "parameters": {}} + if rerank_api == "native" + else {"model": model, "query": "question", "documents": ["answer"]} + ) + assert response.id == "metadata" + assert response.results == results + + +@pytest.mark.parametrize("provider", ["dashscope", "qwencloud", "qwen_ai_platform"]) +def test_rerank_uses_bundled_metadata_when_remote_map_lacks_model(provider, respx_mock, monkeypatch): + import litellm + + for prefix in ("dashscope", "qwencloud", "qwen_ai_platform"): + monkeypatch.delitem(litellm.model_cost, f"{prefix}/qwen3.7-text-rerank", raising=False) + route = respx_mock.post("https://proxy.example/api/v1/services/rerank/text-rerank/text-rerank") + route.respond(200, json={"request_id": "bundled", "output": {"results": [{"index": 0, "relevance_score": 0.9}]}}) + + response = litellm.rerank( + model=f"{provider}/qwen3.7-text-rerank", + query="question", + documents=["answer"], + api_key="fake-key", + api_base="https://proxy.example/api/v1", + ) + + assert json.loads(route.calls[0].request.content)["input"] == {"query": "question", "documents": ["answer"]} + assert response.id == "bundled" + assert response.results == [{"index": 0, "relevance_score": 0.9}]