From 77016a623a0b1a7c3387b692420da591a6c7a021 Mon Sep 17 00:00:00 2001 From: Jim Smith Date: Thu, 18 Jun 2026 10:52:25 -0400 Subject: [PATCH] Add optional `instruction` passthrough to the rerank API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit vLLM's /v1/rerank and /v1/score accept an optional top-level `instruction` field (folded into the model's chat_template_kwargs and consumed by the chat template — e.g. Qwen3-Reranker). LiteLLM's managed rerank route silently dropped it: RerankRequest / OptionalRerankParams had no such field, so the outgoing body was rebuilt without it. Thread an opt-in `instruction: Optional[str]` through rerank()/arerank(), get_optional_rerank_params, and the hosted_vllm transformation into the request body, only when non-None. When callers omit it, model_dump(exclude_none) drops the field and the outgoing request is byte-for-byte unchanged — fully backward-compatible. (DeepInfra already forwards `instruction` via non_default_params; this formalizes the field in the shared types.) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../llms/hosted_vllm/rerank/transformation.py | 25 ++++++---- litellm/rerank_api/main.py | 4 ++ litellm/rerank_api/rerank_utils.py | 3 ++ litellm/types/rerank.py | 5 ++ .../test_hosted_vllm_rerank_transformation.py | 47 +++++++++++++++++++ 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 60b6dc7d23d..efdff780254 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -61,6 +61,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): "top_n", "rank_fields", "return_documents", + "instruction", ] def map_cohere_rerank_params( @@ -83,16 +84,23 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if max_chunks_per_doc is not None: raise ValueError("Hosted VLLM does not support max_chunks_per_doc") - return dict( - OptionalRerankParams( - query=query, - documents=documents, - top_n=top_n, - rank_fields=rank_fields, - return_documents=return_documents, - ) + mapped_params = OptionalRerankParams( + query=query, + documents=documents, + top_n=top_n, + rank_fields=rank_fields, + return_documents=return_documents, ) + # `instruction` is a vLLM-supported passthrough (folded into the model's + # chat_template_kwargs). It arrives via non_default_params; only forward + # it when explicitly set so omitting it leaves the request unchanged. + instruction = (non_default_params or {}).get("instruction") + if instruction is not None: + mapped_params["instruction"] = instruction + + return dict(mapped_params) + def validate_environment( self, headers: dict, @@ -135,6 +143,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): top_n=optional_rerank_params.get("top_n", None), rank_fields=optional_rerank_params.get("rank_fields", None), return_documents=optional_rerank_params.get("return_documents", None), + instruction=optional_rerank_params.get("instruction", None), ) return rerank_request.model_dump(exclude_none=True) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index e40e12e9197..d24d0bcb8be 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -39,6 +39,7 @@ async def arerank( rank_fields: Optional[List[str]] = None, return_documents: Optional[bool] = None, max_chunks_per_doc: Optional[int] = None, + instruction: Optional[str] = None, **kwargs, ) -> Union[RerankResponse, Coroutine[Any, Any, RerankResponse]]: """ @@ -58,6 +59,7 @@ async def arerank( rank_fields, return_documents, max_chunks_per_doc, + instruction=instruction, **kwargs, ) @@ -98,6 +100,7 @@ def rerank( return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, **kwargs, ) -> Union[RerankResponse, Coroutine[Any, Any, RerankResponse]]: """ @@ -155,6 +158,7 @@ def rerank( return_documents=return_documents, max_chunks_per_doc=max_chunks_per_doc, max_tokens_per_doc=max_tokens_per_doc, + instruction=instruction, non_default_params=kwargs, ) verbose_logger.info(f"optional_rerank_params: {optional_rerank_params}") diff --git a/litellm/rerank_api/rerank_utils.py b/litellm/rerank_api/rerank_utils.py index 38e599ef824..32a1a34e343 100644 --- a/litellm/rerank_api/rerank_utils.py +++ b/litellm/rerank_api/rerank_utils.py @@ -15,6 +15,7 @@ def get_optional_rerank_params( return_documents: Optional[bool] = True, max_chunks_per_doc: Optional[int] = None, max_tokens_per_doc: Optional[int] = None, + instruction: Optional[str] = None, non_default_params: Optional[dict] = None, ) -> Dict: all_non_default_params = non_default_params or {} @@ -30,6 +31,8 @@ def get_optional_rerank_params( all_non_default_params["max_chunks_per_doc"] = max_chunks_per_doc if max_tokens_per_doc is not None: all_non_default_params["max_tokens_per_doc"] = max_tokens_per_doc + if instruction is not None: + all_non_default_params["instruction"] = instruction return rerank_provider_config.map_cohere_rerank_params( model=model, drop_params=drop_params, diff --git a/litellm/types/rerank.py b/litellm/types/rerank.py index d2c252a1e92..376d6f66603 100644 --- a/litellm/types/rerank.py +++ b/litellm/types/rerank.py @@ -19,6 +19,10 @@ class RerankRequest(BaseModel): return_documents: Optional[bool] = None max_chunks_per_doc: Optional[int] = None max_tokens_per_doc: Optional[int] = None + # Optional task/query instruction passed through to providers that support it + # (e.g. hosted vLLM / Qwen3-Reranker, DeepInfra). Omitted from the outgoing + # request when None, so this is fully backward-compatible. + instruction: Optional[str] = None class OptionalRerankParams(TypedDict, total=False): @@ -29,6 +33,7 @@ class OptionalRerankParams(TypedDict, total=False): return_documents: Optional[bool] max_chunks_per_doc: Optional[int] max_tokens_per_doc: Optional[int] + instruction: Optional[str] class RerankBilledUnits(TypedDict, total=False): diff --git a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py index 9e6fa608c50..da60ea24d6c 100644 --- a/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/test_hosted_vllm_rerank_transformation.py @@ -37,6 +37,53 @@ class TestHostedVLLMRerankTransform: assert params["rank_fields"] == ["field1"] assert params["return_documents"] is True + def test_map_cohere_rerank_params_omits_instruction_when_absent(self): + # Backward-compat: when no instruction is supplied, it must not appear + # in the mapped params (and therefore not in the outgoing request body). + params = self.config.map_cohere_rerank_params( + non_default_params=None, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + assert "instruction" not in params + + def test_map_cohere_rerank_params_passes_instruction_when_set(self): + params = self.config.map_cohere_rerank_params( + non_default_params={"instruction": "Rank by relevance to genomics"}, + model=self.model, + drop_params=False, + query="test query", + documents=["doc1", "doc2"], + ) + assert params["instruction"] == "Rank by relevance to genomics" + + def test_transform_request_includes_instruction_when_set(self): + body = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={ + "query": "test query", + "documents": ["doc1", "doc2"], + "instruction": "Rank by relevance to genomics", + }, + headers={}, + ) + assert body["instruction"] == "Rank by relevance to genomics" + + def test_transform_request_omits_instruction_when_absent(self): + # exclude_none must drop the field entirely so the body matches the + # pre-existing (instruction-less) shape exactly. + body = self.config.transform_rerank_request( + model=self.model, + optional_rerank_params={ + "query": "test query", + "documents": ["doc1", "doc2"], + }, + headers={}, + ) + assert "instruction" not in body + def test_map_cohere_rerank_params_raises_on_max_chunks_per_doc(self): with pytest.raises( ValueError, match="Hosted VLLM does not support max_chunks_per_doc"