mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(requesty): provider-scoped supported params and streaming exception
This commit is contained in:
parent
677a7f5993
commit
cc499a3692
2 changed files with 182 additions and 27 deletions
|
|
@ -9,29 +9,49 @@ naming convention as OpenRouter, so this config mirrors the OpenRouter one.
|
|||
Docs: https://docs.requesty.ai
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Tuple, Union
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, StreamingChoices
|
||||
|
||||
from ...openrouter.chat.transformation import OpenrouterConfig
|
||||
from ..common_utils import RequestyException
|
||||
|
||||
REQUESTY_REASONING_PARAMS: Final = ("reasoning_effort", "thinking")
|
||||
|
||||
|
||||
class RequestyConfig(OpenrouterConfig):
|
||||
@property
|
||||
def custom_llm_provider(self) -> Optional[str]:
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
return "requesty"
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: Optional[str], api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
api_base = api_base or get_secret_str("REQUESTY_API_BASE") or "https://router.requesty.ai/v1"
|
||||
dynamic_api_key = api_key or get_secret_str("REQUESTY_API_KEY")
|
||||
return api_base, dynamic_api_key
|
||||
self, api_base: str | None, api_key: str | None
|
||||
) -> tuple[str | None, str | None]:
|
||||
resolved_api_base: Final = api_base or get_secret_str("REQUESTY_API_BASE") or "https://router.requesty.ai/v1"
|
||||
dynamic_api_key: Final = api_key or get_secret_str("REQUESTY_API_KEY")
|
||||
return resolved_api_base, dynamic_api_key
|
||||
|
||||
def _supports_reasoning(self, model: str) -> bool:
|
||||
try:
|
||||
return litellm.supports_reasoning(
|
||||
model=model, custom_llm_provider="requesty"
|
||||
) or litellm.supports_reasoning(model=model)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
supported_params: Final[list[str]] = super(OpenrouterConfig, self).get_supported_openai_params(model=model)
|
||||
if not self._supports_reasoning(model):
|
||||
return supported_params
|
||||
return list(dict.fromkeys((*supported_params, *REQUESTY_REASONING_PARAMS)))
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
|
|
@ -40,39 +60,86 @@ class RequestyConfig(OpenrouterConfig):
|
|||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
if non_default_params.get("reasoning_effort") == "max":
|
||||
non_default_params = {**non_default_params, "reasoning_effort": "xhigh"}
|
||||
|
||||
return super(OpenrouterConfig, self).map_openai_params(non_default_params, optional_params, model, drop_params)
|
||||
mapped_params: Final = (
|
||||
{**non_default_params, "reasoning_effort": "xhigh"}
|
||||
if non_default_params.get("reasoning_effort") == "max"
|
||||
else non_default_params
|
||||
)
|
||||
return super(OpenrouterConfig, self).map_openai_params(mapped_params, optional_params, model, drop_params)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: dict,
|
||||
) -> dict:
|
||||
if self._supports_cache_control_in_content(model):
|
||||
messages = self._move_cache_control_to_content(messages)
|
||||
transformed_messages: Final = (
|
||||
self._move_cache_control_to_content(messages)
|
||||
if self._supports_cache_control_in_content(model)
|
||||
else messages
|
||||
)
|
||||
|
||||
extra_body = optional_params.pop("extra_body", {})
|
||||
response = super(OpenrouterConfig, self).transform_request(
|
||||
model, messages, optional_params, litellm_params, headers
|
||||
extra_body: Final = optional_params.pop("extra_body", {})
|
||||
response: Final = super(OpenrouterConfig, self).transform_request(
|
||||
model, transformed_messages, optional_params, litellm_params, headers
|
||||
)
|
||||
# `extra_body` is client-controlled. Do not let it overwrite the canonical
|
||||
# request fields that have already been resolved and authorized (e.g. `model`,
|
||||
# `messages`), otherwise a caller could route to an unauthorized model after
|
||||
# model-authorization and request-inspection checks have run.
|
||||
protected_fields = {"model", "messages"}
|
||||
response.update({key: value for key, value in extra_body.items() if key not in protected_fields})
|
||||
return response
|
||||
protected_fields: Final = frozenset({"model", "messages"})
|
||||
return {**response, **{key: value for key, value in extra_body.items() if key not in protected_fields}}
|
||||
|
||||
def get_error_class(
|
||||
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
||||
) -> BaseLLMException:
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return RequestyException(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def get_model_response_iterator(
|
||||
self,
|
||||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> BaseModelResponseIterator:
|
||||
return RequestyChatCompletionStreamingHandler(
|
||||
streaming_response=streaming_response,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
|
||||
class RequestyChatCompletionStreamingHandler(BaseModelResponseIterator):
|
||||
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
|
||||
try:
|
||||
if "error" in chunk:
|
||||
error_chunk: Final = chunk["error"]
|
||||
raise RequestyException(
|
||||
message="Message: {}, Metadata: {}".format(error_chunk["message"], error_chunk.get("metadata", {})),
|
||||
status_code=error_chunk["code"],
|
||||
headers=error_chunk.get("metadata", {}).get("headers", {}),
|
||||
)
|
||||
|
||||
choices: Final = [
|
||||
StreamingChoices(
|
||||
**{**choice, "delta": {**choice["delta"], "reasoning_content": choice["delta"].get("reasoning")}}
|
||||
)
|
||||
for choice in chunk["choices"]
|
||||
]
|
||||
return ModelResponseStream(
|
||||
id=chunk["id"],
|
||||
object="chat.completion.chunk",
|
||||
created=chunk["created"],
|
||||
usage=chunk.get("usage"),
|
||||
model=chunk["model"],
|
||||
choices=choices,
|
||||
)
|
||||
except KeyError as e:
|
||||
raise RequestyException(
|
||||
message=f"KeyError: {e}, Got unexpected response from Requesty: {chunk}",
|
||||
status_code=400,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
"""Tests for the Requesty provider (OpenAI-compatible gateway)."""
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
from litellm.llms.requesty.chat.transformation import RequestyConfig
|
||||
from litellm.llms.openrouter.common_utils import OpenRouterException
|
||||
from litellm.llms.requesty.chat.transformation import (
|
||||
RequestyChatCompletionStreamingHandler,
|
||||
RequestyConfig,
|
||||
)
|
||||
from litellm.llms.requesty.common_utils import RequestyException
|
||||
|
||||
|
||||
def test_get_llm_provider_resolves_requesty():
|
||||
"""requesty/<provider>/<model> routes to the requesty provider + base URL."""
|
||||
model, custom_llm_provider, _dynamic_api_key, api_base = get_llm_provider(
|
||||
model="requesty/openai/gpt-4o-mini"
|
||||
)
|
||||
model, custom_llm_provider, _dynamic_api_key, api_base = get_llm_provider(model="requesty/openai/gpt-4o-mini")
|
||||
assert custom_llm_provider == "requesty"
|
||||
assert model == "openai/gpt-4o-mini"
|
||||
assert api_base == "https://router.requesty.ai/v1"
|
||||
|
|
@ -59,3 +64,86 @@ def test_transform_request_extra_body_cannot_override_protected_fields():
|
|||
assert result["messages"] == messages
|
||||
# Non-protected extension params still pass through.
|
||||
assert result["custom_flag"] is True
|
||||
|
||||
|
||||
def test_reasoning_model_registered_under_requesty_gets_reasoning_params(monkeypatch):
|
||||
"""A model known only under litellm_provider="requesty" must still unlock reasoning params.
|
||||
|
||||
The parent OpenRouter config checks supports_reasoning with provider "openrouter",
|
||||
which never matches models registered under "requesty".
|
||||
"""
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"requesty/acme/thinker-1",
|
||||
{"litellm_provider": "requesty", "mode": "chat", "supports_reasoning": True},
|
||||
)
|
||||
|
||||
supported_params = RequestyConfig().get_supported_openai_params(model="acme/thinker-1")
|
||||
|
||||
assert "reasoning_effort" in supported_params
|
||||
assert "thinking" in supported_params
|
||||
assert supported_params.count("reasoning_effort") == 1
|
||||
assert "temperature" in supported_params
|
||||
|
||||
|
||||
def test_non_reasoning_model_does_not_get_reasoning_params(monkeypatch):
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"requesty/acme/plain-1",
|
||||
{"litellm_provider": "requesty", "mode": "chat", "supports_reasoning": False},
|
||||
)
|
||||
|
||||
supported_params = RequestyConfig().get_supported_openai_params(model="acme/plain-1")
|
||||
|
||||
assert "reasoning_effort" not in supported_params
|
||||
assert "thinking" not in supported_params
|
||||
|
||||
|
||||
def test_get_model_response_iterator_returns_requesty_handler():
|
||||
handler = RequestyConfig().get_model_response_iterator(streaming_response=iter(()), sync_stream=True)
|
||||
|
||||
assert isinstance(handler, RequestyChatCompletionStreamingHandler)
|
||||
|
||||
|
||||
class TestRequestyChatCompletionStreamingHandler:
|
||||
def test_chunk_parser_successful(self):
|
||||
handler = RequestyChatCompletionStreamingHandler(streaming_response=None, sync_stream=True)
|
||||
chunk = {
|
||||
"id": "chunk-1",
|
||||
"created": 1234567890,
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
|
||||
"choices": [{"index": 0, "delta": {"content": "hello", "reasoning": "thinking"}}],
|
||||
}
|
||||
|
||||
result = handler.chunk_parser(chunk)
|
||||
|
||||
assert result.id == "chunk-1"
|
||||
assert result.object == "chat.completion.chunk"
|
||||
assert result.created == 1234567890
|
||||
assert result.model == "openai/gpt-4o-mini"
|
||||
assert result.usage.total_tokens == 30
|
||||
assert len(result.choices) == 1
|
||||
assert result.choices[0]["delta"]["content"] == "hello"
|
||||
assert result.choices[0]["delta"]["reasoning_content"] == "thinking"
|
||||
|
||||
def test_chunk_parser_error_response_raises_requesty_exception(self):
|
||||
handler = RequestyChatCompletionStreamingHandler(streaming_response=None, sync_stream=True)
|
||||
error_chunk = {"error": {"message": "rate limited", "code": 429, "metadata": {"headers": {"Retry-After": "1"}}}}
|
||||
|
||||
with pytest.raises(RequestyException) as exc_info:
|
||||
handler.chunk_parser(error_chunk)
|
||||
|
||||
assert not isinstance(exc_info.value, OpenRouterException)
|
||||
assert "rate limited" in str(exc_info.value)
|
||||
assert exc_info.value.status_code == 429
|
||||
|
||||
def test_chunk_parser_malformed_chunk_raises_requesty_exception(self):
|
||||
handler = RequestyChatCompletionStreamingHandler(streaming_response=None, sync_stream=True)
|
||||
|
||||
with pytest.raises(RequestyException) as exc_info:
|
||||
handler.chunk_parser({"incomplete": "data"})
|
||||
|
||||
assert not isinstance(exc_info.value, OpenRouterException)
|
||||
assert "KeyError" in str(exc_info.value)
|
||||
assert exc_info.value.status_code == 400
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue