mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(hosted_vllm/rerank): accept SGLang bare-list response and score/document str shapes
SGLang's /v1/rerank returns a bare list of {score, index, document} objects
instead of the OpenAI/Cohere envelope {results: [...]}, causing the hosted_vllm
rerank transformer to crash with 'list' object has no attribute 'get'.
Normalize three response shape divergences in _transform_response:
- response is a list -> wrap as {results: response}
- result entry uses 'score' instead of 'relevance_score' -> alias it before validation
- result.document is a plain string instead of {text: ...} dict -> accept both
Fixes BerriAI/litellm#29156
Signed-off-by: Tai An <antai12232931@outlook.com>
This commit is contained in:
parent
5699a06413
commit
ffbcbf62e1
1 changed files with 24 additions and 8 deletions
|
|
@ -168,7 +168,15 @@ class HostedVLLMRerankConfig(BaseRerankConfig):
|
|||
message=error_message, status_code=status_code, headers=headers
|
||||
)
|
||||
|
||||
def _transform_response(self, response: dict) -> RerankResponse:
|
||||
def _transform_response(
|
||||
self, response: Union[dict, list]
|
||||
) -> RerankResponse:
|
||||
# SGLang's /v1/rerank endpoint returns a bare list of result objects
|
||||
# instead of the OpenAI/Cohere-style {"results": [...], "usage": {...}}
|
||||
# envelope. Normalize both shapes here so hosted_vllm covers both.
|
||||
if isinstance(response, list):
|
||||
response = {"results": response}
|
||||
|
||||
# Extract usage information
|
||||
usage_data = response.get("usage", {})
|
||||
_billed_units = RerankBilledUnits(
|
||||
|
|
@ -186,17 +194,25 @@ class HostedVLLMRerankConfig(BaseRerankConfig):
|
|||
rerank_results: List[RerankResponseResult] = []
|
||||
|
||||
for result in _results:
|
||||
# SGLang uses "score" rather than the OpenAI/Cohere "relevance_score".
|
||||
if "relevance_score" not in result and "score" in result:
|
||||
result = {**result, "relevance_score": result["score"]}
|
||||
|
||||
# Validate required fields exist
|
||||
if not all(key in result for key in ["index", "relevance_score"]):
|
||||
raise ValueError(f"Missing required fields in the result={result}")
|
||||
|
||||
# Get document data if it exists
|
||||
document_data = result.get("document", {})
|
||||
document = (
|
||||
RerankResponseDocument(text=str(document_data.get("text", "")))
|
||||
if document_data
|
||||
else None
|
||||
)
|
||||
# Get document data if it exists. SGLang returns the document as a
|
||||
# plain string; the OpenAI/Cohere envelope returns {"text": "..."}.
|
||||
document_data = result.get("document")
|
||||
if isinstance(document_data, str):
|
||||
document = RerankResponseDocument(text=document_data)
|
||||
elif isinstance(document_data, dict) and document_data:
|
||||
document = RerankResponseDocument(
|
||||
text=str(document_data.get("text", ""))
|
||||
)
|
||||
else:
|
||||
document = None
|
||||
|
||||
# Create typed result
|
||||
rerank_result = RerankResponseResult(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue