From ffbcbf62e1b66838ffbb64347f5c57dbe70525e8 Mon Sep 17 00:00:00 2001 From: Tai An Date: Thu, 28 May 2026 06:19:30 -0700 Subject: [PATCH] fix(hosted_vllm/rerank): accept SGLang bare-list response and score/document str shapes SGLang's /v1/rerank returns a bare list of {score, index, document} objects instead of the OpenAI/Cohere envelope {results: [...]}, causing the hosted_vllm rerank transformer to crash with 'list' object has no attribute 'get'. Normalize three response shape divergences in _transform_response: - response is a list -> wrap as {results: response} - result entry uses 'score' instead of 'relevance_score' -> alias it before validation - result.document is a plain string instead of {text: ...} dict -> accept both Fixes BerriAI/litellm#29156 Signed-off-by: Tai An --- .../llms/hosted_vllm/rerank/transformation.py | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index 60b6dc7d23d..a5699fb1c1b 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -168,7 +168,15 @@ class HostedVLLMRerankConfig(BaseRerankConfig): message=error_message, status_code=status_code, headers=headers ) - def _transform_response(self, response: dict) -> RerankResponse: + def _transform_response( + self, response: Union[dict, list] + ) -> RerankResponse: + # SGLang's /v1/rerank endpoint returns a bare list of result objects + # instead of the OpenAI/Cohere-style {"results": [...], "usage": {...}} + # envelope. Normalize both shapes here so hosted_vllm covers both. + if isinstance(response, list): + response = {"results": response} + # Extract usage information usage_data = response.get("usage", {}) _billed_units = RerankBilledUnits( @@ -186,17 +194,25 @@ class HostedVLLMRerankConfig(BaseRerankConfig): rerank_results: List[RerankResponseResult] = [] for result in _results: + # SGLang uses "score" rather than the OpenAI/Cohere "relevance_score". + if "relevance_score" not in result and "score" in result: + result = {**result, "relevance_score": result["score"]} + # Validate required fields exist if not all(key in result for key in ["index", "relevance_score"]): raise ValueError(f"Missing required fields in the result={result}") - # Get document data if it exists - document_data = result.get("document", {}) - document = ( - RerankResponseDocument(text=str(document_data.get("text", ""))) - if document_data - else None - ) + # Get document data if it exists. SGLang returns the document as a + # plain string; the OpenAI/Cohere envelope returns {"text": "..."}. + document_data = result.get("document") + if isinstance(document_data, str): + document = RerankResponseDocument(text=document_data) + elif isinstance(document_data, dict) and document_data: + document = RerankResponseDocument( + text=str(document_data.get("text", "")) + ) + else: + document = None # Create typed result rerank_result = RerankResponseResult(