This commit is contained in:
Parijat Biswas 2026-10-27 11:27:44 +05:30 • committed by GitHub
commit efa98fcaad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 527 additions and 7 deletions

View file

@ -1696,6 +1696,9 @@ if TYPE_CHECKING:
from .llms.deepinfra.rerank.transformation import (
DeepinfraRerankConfig as DeepinfraRerankConfig,
)
from .llms.openrouter.rerank.transformation import (
OpenRouterRerankConfig as OpenRouterRerankConfig,
)
from .llms.hosted_vllm.rerank.transformation import (
HostedVLLMRerankConfig as HostedVLLMRerankConfig,
)

View file

@ -159,6 +159,7 @@ LLM_CONFIG_NAMES: Final = (
"JinaAIRerankConfig",
"ScalewayRerankConfig",
"DeepinfraRerankConfig",
"OpenRouterRerankConfig",
"HostedVLLMRerankConfig",
"PerplexityDecisionsConfig",
"TypeSafeDecisionsConfig",
@ -728,6 +729,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
".llms.deepinfra.rerank.transformation",
"DeepinfraRerankConfig",
),
"OpenRouterRerankConfig": (
".llms.openrouter.rerank.transformation",
"OpenRouterRerankConfig",
),
"HostedVLLMRerankConfig": (
".llms.hosted_vllm.rerank.transformation",
"HostedVLLMRerankConfig",

View file

@ -0,0 +1,212 @@
"""
OpenRouter Rerank API Configuration.
OpenRouter serves rerank models (e.g. `cohere/rerank-v3.5`) through
`POST https://openrouter.ai/api/v1/rerank`. The request and response shapes
follow the Cohere rerank format, with OpenRouter-specific response fields
(`provider`, and a `usage` block carrying `search_units` / `total_tokens`).
Reference: https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request
"""
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter, ValidationError
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.rerank import (
RerankBilledUnits,
RerankResponse,
RerankResponseMeta,
RerankResponseResult,
)
from ..common_utils import OpenRouterException
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_JSON_OBJECTS: Final = TypeAdapter(list[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
_INDEX: Final = TypeAdapter(int, config=ConfigDict(strict=True, hide_input_in_errors=True))
_SCORE: Final[TypeAdapter[bool | int | float]] = TypeAdapter(
bool | int | float, config=ConfigDict(strict=True, hide_input_in_errors=True)
)
_BILLED_UNITS: Final = TypeAdapter(RerankBilledUnits)
_STR: Final = TypeAdapter(str)
DEFAULT_API_BASE: Final = "https://openrouter.ai/api/v1"
class OpenRouterRerankConfig(BaseRerankConfig):
"""
Reference: https://openrouter.ai/docs/api/api-reference/rerank/submit-a-rerank-request
"""
def get_supported_cohere_rerank_params(self, model: str) -> list[str]: # mutable-ok: BaseRerankConfig contract
return [
"query",
"documents",
"top_n",
]
def map_cohere_rerank_params(
self,
non_default_params: dict[str, object] | None, # mutable-ok: BaseRerankConfig contract
model: str,
drop_params: bool,
query: str,
documents: list[str | dict[str, object]], # mutable-ok: BaseRerankConfig contract
custom_llm_provider: str | None = None,
top_n: int | None = None,
rank_fields: list[str] | None = None, # mutable-ok: BaseRerankConfig contract
return_documents: bool | None = True,
max_chunks_per_doc: int | None = None,
max_tokens_per_doc: int | None = None,
instruction: str | None = None,
) -> dict[str, object]: # mutable-ok: BaseRerankConfig contract
if non_default_params is None:
return {}
supported_params: Final = self.get_supported_cohere_rerank_params(model)
return {k: v for k, v in non_default_params.items() if k in supported_params}
def get_complete_url(
self,
api_base: str | None,
model: str,
optional_params: Mapping[str, object] | None = None,
) -> str:
base: Final = api_base or get_secret_str("OPENROUTER_API_BASE") or DEFAULT_API_BASE
return join_api_base(base, "rerank")
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None = None,
optional_params: Mapping[str, object] | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict[str, str]: # mutable-ok: BaseRerankConfig contract
key: Final = api_key or get_secret_str("OPENROUTER_API_KEY")
if not key:
raise ValueError(
"OpenRouter API key not found. Pass `api_key=...` or set the OPENROUTER_API_KEY environment variable."
)
provider_headers: Final = {
"accept": "application/json",
"content-type": "application/json",
"Authorization": f"Bearer {key}",
"HTTP-Referer": get_secret_str("OR_SITE_URL") or "https://litellm.ai",
"X-Title": get_secret_str("OR_APP_NAME") or "liteLLM",
}
caller_headers: Final = {name: value for name, value in headers.items() if name != "Authorization"}
return {**caller_headers, **provider_headers}
def transform_rerank_request(
self,
model: str,
optional_rerank_params: dict[str, object], # mutable-ok: BaseRerankConfig contract
headers: Mapping[str, str],
litellm_params: Mapping[str, object] | None = None,
) -> dict[str, object]: # mutable-ok: BaseRerankConfig contract
if "query" not in optional_rerank_params:
raise ValueError("query is required for OpenRouter rerank")
if "documents" not in optional_rerank_params:
raise ValueError("documents is required for OpenRouter rerank")
# OpenRouter expects the upstream model id (e.g. `cohere/rerank-v3.5`),
# without LiteLLM's `openrouter/` routing prefix.
upstream_model: Final = model.removeprefix("openrouter/")
return {"model": upstream_model, **optional_rerank_params}
def transform_rerank_response(
self,
model: str,
raw_response: httpx.Response,
model_response: RerankResponse,
logging_obj: LiteLLMLoggingObj,
api_key: str | None = None,
request_data: dict[str, object] | None = None, # mutable-ok: BaseRerankConfig contract
optional_params: dict[str, object] | None = None, # mutable-ok: BaseRerankConfig contract
litellm_params: dict[str, object] | None = None, # mutable-ok: BaseRerankConfig contract
) -> RerankResponse:
if raw_response.status_code != 200:
raise OpenRouterException(message=raw_response.text, status_code=raw_response.status_code)
logging_obj.post_call(original_response=raw_response.text) # pyright: ignore[reportUnknownMemberType] # Logging.post_call is untyped upstream
_json_response: Final = _JSON_OBJECT.validate_python(raw_response.json())
_results: Final = _json_response.get("results")
if _results is None:
raise ValueError(f"No results found in the response={_json_response}")
# OpenRouter returns Cohere-shaped results:
# {"index": 0, "relevance_score": 0.98, "document": {"text": "..."}}
transformed_results: Final = [
self._transform_result(result) for result in _JSON_OBJECTS.validate_python(_results)
]
usage: Final = _JSON_OBJECT.validate_python(_json_response.get("usage", {}))
billed_units: Final = _BILLED_UNITS.validate_python(usage)
meta: Final[RerankResponseMeta] = {"billed_units": billed_units}
rerank_response: Final = RerankResponse(
id=_STR.validate_python(_json_response.get("id") or str(uuid.uuid4())),
results=transformed_results,
meta=meta,
)
# Keep OpenRouter-specific fields for logging/cost tracking
provider_name: Final = _json_response.get("provider")
if isinstance(provider_name, str):
rerank_response.hidden_params["provider"] = provider_name
rerank_response.hidden_params["model"] = model
return rerank_response
@staticmethod
def _transform_result(result: Mapping[str, object]) -> RerankResponseResult:
transformed_result: Final[RerankResponseResult] = {
"index": _INDEX.validate_python(result["index"]),
"relevance_score": float(_SCORE.validate_python(result["relevance_score"])),
}
document: Final = result.get("document")
if isinstance(document, str):
transformed_result["document"] = {"text": document}
elif isinstance(document, Mapping):
document_mapping: Final = _JSON_OBJECT.validate_python(document)
text: Final = document_mapping.get("text")
if isinstance(text, str):
transformed_result["document"] = {"text": text}
return transformed_result
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseRerankConfig contract
) -> OpenRouterException:
return OpenRouterException(
status_code=status_code,
message=self._extract_error_message(error_message),
headers=headers,
)
@staticmethod
def _extract_error_message(error_message: str) -> str:
# OpenRouter errors are OpenAI-shaped: {"error": {"message": "...", "code": ...}}
try:
parsed: Final = _JSON_OBJECT.validate_json(error_message)
except ValidationError:
return error_message
error_obj: Final = parsed.get("error")
if isinstance(error_obj, Mapping):
error_mapping: Final = _JSON_OBJECT.validate_python(error_obj)
inner: Final = error_mapping.get("message")
if isinstance(inner, str):
return inner
return error_message

View file

@ -734,6 +734,17 @@
"mode": "rerank",
"output_cost_per_token": 0.0
},
"openrouter/cohere/rerank-v3.5": {
"input_cost_per_query": 0.001,
"input_cost_per_token": 0.0,
"litellm_provider": "openrouter",
"max_input_tokens": 4096,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "rerank",
"output_cost_per_token": 0.0,
"source": "https://openrouter.ai/cohere/rerank-v3.5"
},
"amazon.titan-embed-image-v1": {
"input_cost_per_image": 6e-05,
"input_cost_per_token": 8e-07,

View file

@ -33,7 +33,7 @@ async def arerank(
query: str,
documents: list[str | dict[str, object]],
custom_llm_provider: (
Literal["cohere", "together_ai", "deepinfra", "fireworks_ai", "voyage", "watsonx"] | None
Literal["cohere", "together_ai", "deepinfra", "fireworks_ai", "voyage", "watsonx", "openrouter"] | None
) = None,
top_n: int | None = None,
rank_fields: list[str] | None = None,
@ -106,6 +106,7 @@ def rerank(
"fireworks_ai",
"voyage",
"watsonx",
"openrouter",
]
| None
) = None,

View file

@ -9538,11 +9538,12 @@ class ProviderConfigManager:
api_base: str | None,
present_version_params: list[str],
) -> BaseRerankConfig:
if litellm.LlmProviders.COHERE == provider or litellm.LlmProviders.COHERE_CHAT == provider:
if should_use_cohere_v1_client(api_base, present_version_params):
return litellm.CohereRerankConfig()
else:
return litellm.CohereRerankV2Config()
if provider in (litellm.LlmProviders.COHERE, litellm.LlmProviders.COHERE_CHAT):
return (
litellm.CohereRerankConfig()
if should_use_cohere_v1_client(api_base, present_version_params)
else litellm.CohereRerankV2Config()
)
elif litellm.LlmProviders.AZURE_AI == provider:
return litellm.AzureAIRerankConfig()
elif litellm.LlmProviders.INFINITY == provider:
@ -9583,6 +9584,8 @@ class ProviderConfigManager:
)
return get_dashscope_family_rerank_config(provider.value)
elif litellm.LlmProviders.OPENROUTER == provider:
return litellm.OpenRouterRerankConfig()
return litellm.CohereRerankConfig()
@staticmethod

View file

@ -734,6 +734,17 @@
"mode": "rerank",
"output_cost_per_token": 0.0
},
"openrouter/cohere/rerank-v3.5": {
"input_cost_per_query": 0.001,
"input_cost_per_token": 0.0,
"litellm_provider": "openrouter",
"max_input_tokens": 4096,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "rerank",
"output_cost_per_token": 0.0,
"source": "https://openrouter.ai/cohere/rerank-v3.5"
},
"amazon.titan-embed-image-v1": {
"input_cost_per_image": 6e-05,
"input_cost_per_token": 8e-07,

View file

@ -2055,7 +2055,7 @@
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"rerank": true,
"a2a": true,
"interactions": true
}

View file

@ -0,0 +1,274 @@
"""
Unit tests for the OpenRouter rerank provider.
OpenRouter serves rerank models (e.g. `cohere/rerank-v3.5`) via
POST https://openrouter.ai/api/v1/rerank - all tests here are mocked.
"""
import json
from unittest.mock import MagicMock, patch
import httpx
import pytest
import respx
from pydantic import ValidationError
import litellm
from litellm.llms.openrouter.common_utils import OpenRouterException
from litellm.llms.openrouter.rerank.transformation import OpenRouterRerankConfig
from litellm.types.rerank import RerankResponse
from litellm.utils import ProviderConfigManager
OPENROUTER_RERANK_RESPONSE = {
"id": "gen-rerank-1234567890-abc",
"model": "cohere/rerank-v3.5",
"provider": "Cohere",
"results": [
{"document": {"text": "Paris is the capital of France."}, "index": 0, "relevance_score": 0.98},
{"document": {"text": "Berlin is the capital of Germany."}, "index": 1, "relevance_score": 0.12},
],
"usage": {"search_units": 1, "total_tokens": 150},
}
class _RespxAsyncTransport(httpx.AsyncBaseTransport):
def __init__(self, router: respx.MockRouter) -> None:
self.router = router
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
return await self.router.async_handler(request)
def _config() -> OpenRouterRerankConfig:
return OpenRouterRerankConfig()
def _transform(payload: object, status_code: int = 200) -> RerankResponse:
return _config().transform_rerank_response(
model="cohere/rerank-v3.5",
raw_response=httpx.Response(status_code, json=payload),
model_response=RerankResponse(),
logging_obj=MagicMock(),
)
def test_get_provider_rerank_config_returns_openrouter_config():
config = ProviderConfigManager.get_provider_rerank_config(
model="cohere/rerank-v3.5",
provider=litellm.LlmProviders.OPENROUTER,
api_base=None,
present_version_params=[],
)
assert isinstance(config, OpenRouterRerankConfig)
def test_get_complete_url_defaults_to_openrouter_rerank_endpoint():
assert _config().get_complete_url(None, "cohere/rerank-v3.5") == "https://openrouter.ai/api/v1/rerank"
def test_get_complete_url_respects_custom_api_base():
assert (
_config().get_complete_url("https://proxy.example.com/v1/", "cohere/rerank-v3.5")
== "https://proxy.example.com/v1/rerank"
)
def test_validate_environment_builds_auth_headers():
headers = _config().validate_environment(headers={}, model="cohere/rerank-v3.5", api_key="sk-or-test")
assert headers["Authorization"] == "Bearer sk-or-test"
assert headers["content-type"] == "application/json"
assert headers["HTTP-Referer"] == "https://litellm.ai"
assert headers["X-Title"] == "liteLLM"
def test_validate_environment_falls_back_to_env_key(monkeypatch):
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-env")
headers = _config().validate_environment(headers={}, model="cohere/rerank-v3.5")
assert headers["Authorization"] == "Bearer sk-or-env"
def test_validate_environment_requires_an_api_key(monkeypatch):
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
with pytest.raises(ValueError, match="OPENROUTER_API_KEY"):
_config().validate_environment(headers={}, model="cohere/rerank-v3.5")
def test_map_cohere_rerank_params_keeps_only_supported_params():
mapped = _config().map_cohere_rerank_params(
non_default_params={
"query": "q",
"documents": ["a", "b"],
"top_n": 1,
"return_documents": True,
"max_chunks_per_doc": 3,
"some_unknown_param": 1,
},
model="cohere/rerank-v3.5",
drop_params=False,
query="q",
documents=["a", "b"],
top_n=1,
)
assert mapped == {"query": "q", "documents": ["a", "b"], "top_n": 1}
def test_transform_rerank_request_strips_litellm_routing_prefix():
request_data = _config().transform_rerank_request(
model="openrouter/cohere/rerank-v3.5",
optional_rerank_params={"query": "q", "documents": ["a", "b"], "top_n": 1},
headers={},
)
assert request_data == {
"model": "cohere/rerank-v3.5",
"query": "q",
"documents": ["a", "b"],
"top_n": 1,
}
@pytest.mark.parametrize("missing", ["query", "documents"])
def test_transform_rerank_request_requires_query_and_documents(missing: str):
params = {"query": "q", "documents": ["a"]}
del params[missing]
with pytest.raises(ValueError, match=missing):
_config().transform_rerank_request(model="cohere/rerank-v3.5", optional_rerank_params=params, headers={})
def test_transform_rerank_response_maps_openrouter_payload():
response = _transform(OPENROUTER_RERANK_RESPONSE)
assert response.id == "gen-rerank-1234567890-abc"
assert response.results == [
{"index": 0, "relevance_score": 0.98, "document": {"text": "Paris is the capital of France."}},
{"index": 1, "relevance_score": 0.12, "document": {"text": "Berlin is the capital of Germany."}},
]
assert response.meta == {"billed_units": {"search_units": 1, "total_tokens": 150}}
assert response.hidden_params["provider"] == "Cohere"
assert response.hidden_params["model"] == "cohere/rerank-v3.5"
def test_transform_rerank_response_wraps_string_documents():
response = _transform(
{
"id": "gen-rerank-1",
"results": [{"index": 1, "relevance_score": 0.5, "document": "plain string doc"}],
}
)
assert response.results == [{"index": 1, "relevance_score": 0.5, "document": {"text": "plain string doc"}}]
assert response.meta == {"billed_units": {}}
def test_transform_rerank_response_generates_an_id_when_the_provider_sends_none():
response = _transform({"results": [{"index": 0, "relevance_score": 0.5}]})
assert isinstance(response.id, str)
assert response.id != ""
def test_transform_rerank_response_without_results_raises_value_error():
with pytest.raises(ValueError, match="No results found"):
_transform({"id": "gen-rerank-1"})
def test_transform_rerank_response_rejects_malformed_payloads():
with pytest.raises(ValidationError):
_transform({"results": [["leaked document text"]]})
def test_transform_rerank_response_non_200_raises_openrouter_exception():
with pytest.raises(OpenRouterException, match="quota exceeded"):
_transform({"error": {"message": "quota exceeded"}}, status_code=429)
def test_get_error_class_extracts_openrouter_error_message():
error = _config().get_error_class(
error_message=json.dumps({"error": {"message": "Invalid API key", "code": 401}}),
status_code=401,
headers={},
)
assert isinstance(error, OpenRouterException)
assert error.message == "Invalid API key"
assert error.status_code == 401
def test_map_cohere_rerank_params_returns_empty_dict_when_params_are_none():
mapped = _config().map_cohere_rerank_params(
non_default_params=None,
model="cohere/rerank-v3.5",
drop_params=False,
query="q",
documents=["a", "b"],
)
assert mapped == {}
def test_get_error_class_keeps_unparseable_error_message():
error = _config().get_error_class(
error_message="upstream timeout",
status_code=500,
headers={},
)
assert error.message == "upstream timeout"
assert error.status_code == 500
def test_get_error_class_keeps_original_message_without_error_payload():
raw = json.dumps({"detail": "not an OpenRouter error shape"})
error = _config().get_error_class(
error_message=raw,
status_code=400,
headers={},
)
assert error.message == raw
@pytest.mark.respx(assert_all_called=True)
@pytest.mark.asyncio
@pytest.mark.parametrize("sync_mode", [True, False])
async def test_openrouter_rerank_end_to_end(sync_mode: bool, respx_mock: respx.MockRouter, monkeypatch):
"""litellm.rerank(model="openrouter/...") routes to OpenRouter's /rerank endpoint."""
monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test")
route = respx_mock.post("https://openrouter.ai/api/v1/rerank")
route.return_value = httpx.Response(200, json=OPENROUTER_RERANK_RESPONSE)
kwargs = {
"model": "openrouter/cohere/rerank-v3.5",
"query": "What is the capital of France?",
"documents": ["Paris is the capital of France.", "Berlin is the capital of Germany."],
"top_n": 2,
}
if sync_mode:
response = litellm.rerank(**kwargs)
else:
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
client = AsyncHTTPHandler(transport=_RespxAsyncTransport(respx_mock))
response = await litellm.arerank(**kwargs, client=client)
await client.client.aclose()
assert route.called
request_body = json.loads(route.calls[0].request.content)
assert request_body == {
"model": "cohere/rerank-v3.5",
"query": "What is the capital of France?",
"documents": ["Paris is the capital of France.", "Berlin is the capital of Germany."],
"top_n": 2,
}
assert route.calls[0].request.headers["Authorization"] == "Bearer sk-or-test"
assert response.results[0]["index"] == 0
assert response.results[0]["relevance_score"] == 0.98
assert response.meta["billed_units"] == {"search_units": 1, "total_tokens": 150}