mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
feat: add Opper provider
OpenRouter-style provider module for the Opper AI gateway (https://opper.ai): opper/ prefix routing, gateway-reported usage.cost into spend tracking (non-streaming and streaming), reasoning_effort and cache_control passthrough, OPPER_API_KEY / OPPER_API_BASE resolution. APIConnectionError now honors its request argument so status-less failures report the actual upstream endpoint.
This commit is contained in:
parent
b3882d8e43
commit
220801e140
16 changed files with 755 additions and 3 deletions
|
|
@ -1561,6 +1561,7 @@ if TYPE_CHECKING:
|
|||
from .llms.openrouter.chat.transformation import (
|
||||
OpenrouterConfig as OpenrouterConfig,
|
||||
)
|
||||
from .llms.opper.chat.transformation import OpperConfig as OpperConfig
|
||||
from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig
|
||||
from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig
|
||||
from .llms.bedrock.claude_platform.transformation import (
|
||||
|
|
|
|||
|
|
@ -131,6 +131,7 @@ LLM_CONFIG_NAMES: Final = (
|
|||
"OobaboogaConfig",
|
||||
"MaritalkConfig",
|
||||
"OpenrouterConfig",
|
||||
"OpperConfig",
|
||||
"DataRobotConfig",
|
||||
"AnthropicConfig",
|
||||
"BedrockClaudePlatformConfig",
|
||||
|
|
@ -624,6 +625,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
|
|||
"OobaboogaConfig": (".llms.oobabooga.chat.transformation", "OobaboogaConfig"),
|
||||
"MaritalkConfig": (".llms.maritalk", "MaritalkConfig"),
|
||||
"OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"),
|
||||
"OpperConfig": (".llms.opper.chat.transformation", "OpperConfig"),
|
||||
"DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"),
|
||||
"AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"),
|
||||
"BedrockClaudePlatformConfig": (
|
||||
|
|
|
|||
|
|
@ -641,6 +641,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"datarobot",
|
||||
"helicone",
|
||||
"openrouter",
|
||||
"opper",
|
||||
"cometapi",
|
||||
"vertex_ai",
|
||||
"vertex_ai_beta",
|
||||
|
|
|
|||
|
|
@ -860,7 +860,7 @@ class APIConnectionError(openai.APIConnectionError):
|
|||
self.model = model
|
||||
self.status_code = 500
|
||||
self.litellm_debug_info = litellm_debug_info
|
||||
self.request = httpx.Request(method="POST", url="https://api.openai.com/v1")
|
||||
self.request = request if request is not None else httpx.Request(method="POST", url="https://api.openai.com/v1")
|
||||
self.max_retries = max_retries
|
||||
self.num_retries = num_retries
|
||||
super().__init__(message=self.message, request=self.request)
|
||||
|
|
|
|||
|
|
@ -2195,12 +2195,15 @@ def _map_openrouter_exception(
|
|||
)
|
||||
else:
|
||||
# if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
|
||||
fallback_api_base: Final = (
|
||||
"https://api.opper.ai/v3/compat" if custom_llm_provider == "opper" else "https://api.openai.com/v1/"
|
||||
)
|
||||
raise APIConnectionError(
|
||||
message=f"APIConnectionError: {exception_provider} - {error_str}",
|
||||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
litellm_debug_info=extra_information,
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"),
|
||||
request=httpx.Request(method="POST", url=fallback_api_base),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -2609,7 +2612,7 @@ def exception_type(
|
|||
exception_provider=exception_provider,
|
||||
extra_information=extra_information,
|
||||
)
|
||||
if custom_llm_provider == "openrouter":
|
||||
if custom_llm_provider in ("openrouter", "opper"):
|
||||
_map_openrouter_exception(
|
||||
model=model,
|
||||
original_exception=mappable_exception,
|
||||
|
|
|
|||
|
|
@ -204,6 +204,12 @@ def get_llm_provider(
|
|||
return remainder, custom_llm_provider, dynamic_api_key, api_base
|
||||
return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
|
||||
if custom_llm_provider == "opper" and model.startswith("opper/"):
|
||||
opper_remainder: Final = model[len("opper/") :]
|
||||
if "/" in opper_remainder:
|
||||
return opper_remainder, custom_llm_provider, dynamic_api_key, api_base
|
||||
return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
|
||||
# Check JSON-configured providers FIRST (before enum-based provider_list)
|
||||
provider_prefix: Final = model.split("/", 1)[0]
|
||||
if len(model.split("/")) > 1 and JSONProviderRegistry.exists(provider_prefix):
|
||||
|
|
@ -692,6 +698,10 @@ def _get_openai_compatible_provider_info(
|
|||
api_base = api_base or get_secret("TENCENT_API_BASE") or "https://tokenhub-intl.tencentcloudmaas.com/v1"
|
||||
|
||||
dynamic_api_key = api_key or get_secret_str("TENCENT_API_KEY")
|
||||
elif custom_llm_provider == "opper":
|
||||
api_base = api_base or get_secret("OPPER_API_BASE") or "https://api.opper.ai/v3/compat"
|
||||
|
||||
dynamic_api_key = api_key or get_secret_str("OPPER_API_KEY")
|
||||
elif custom_llm_provider == "fireworks_ai":
|
||||
# fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1
|
||||
(
|
||||
|
|
|
|||
|
|
@ -142,6 +142,8 @@ def get_supported_openai_params(
|
|||
return litellm.AzureOpenAIConfig().get_supported_openai_params(model=_azure_detection_model)
|
||||
elif custom_llm_provider == "openrouter":
|
||||
return litellm.OpenrouterConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "opper":
|
||||
return litellm.OpperConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "vercel_ai_gateway":
|
||||
return litellm.VercelAIGatewayConfig().get_supported_openai_params(model=model)
|
||||
elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral":
|
||||
|
|
|
|||
0
litellm/llms/opper/__init__.py
Normal file
0
litellm/llms/opper/__init__.py
Normal file
0
litellm/llms/opper/chat/__init__.py
Normal file
0
litellm/llms/opper/chat/__init__.py
Normal file
128
litellm/llms/opper/chat/transformation.py
Normal file
128
litellm/llms/opper/chat/transformation.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
"""
|
||||
Translate from OpenAI's `/v1/chat/completions` to the Opper AI gateway's `/v3/compat/chat/completions`
|
||||
"""
|
||||
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException, LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, StreamingChoices
|
||||
|
||||
from ...openai.chat.gpt_transformation import (
|
||||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from ..common_utils import OpperException
|
||||
|
||||
|
||||
class OpperConfig(OpenAIGPTConfig):
|
||||
@property
|
||||
def custom_llm_provider(self) -> str | None:
|
||||
return "opper"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
supported_params: Final = super().get_supported_openai_params(model=model)
|
||||
if "reasoning_effort" not in supported_params:
|
||||
supported_params.append("reasoning_effort")
|
||||
return supported_params
|
||||
|
||||
def _should_preserve_cache_control_for_endpoint(
|
||||
self,
|
||||
custom_llm_provider: str | None,
|
||||
api_base: str | None,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues], # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
optional_params: dict, # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
litellm_params: dict, # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
headers: dict, # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
) -> dict: # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
request: Final = super().transform_request(model, messages, optional_params, litellm_params, headers)
|
||||
if request.get("stream"):
|
||||
stream_options: Final = dict(request.get("stream_options") or {})
|
||||
stream_options.setdefault("include_usage", True)
|
||||
request["stream_options"] = stream_options
|
||||
return request
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ModelResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: dict, # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
messages: list[AllMessageValues], # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
optional_params: dict, # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
litellm_params: dict, # mutable-ok: mirrors the OpenAIGPTConfig signature
|
||||
encoding: object,
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
model_response = super().transform_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
logging_obj=logging_obj,
|
||||
request_data=request_data,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
encoding=encoding,
|
||||
api_key=api_key,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
try:
|
||||
response_json: Final = raw_response.json()
|
||||
if response_json.get("usage"):
|
||||
response_cost: Final = response_json["usage"].get("cost")
|
||||
if response_cost is not None:
|
||||
if "additional_headers" not in model_response._hidden_params:
|
||||
model_response._hidden_params["additional_headers"] = {}
|
||||
model_response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(
|
||||
response_cost
|
||||
)
|
||||
except (ValueError, TypeError, KeyError, AttributeError):
|
||||
verbose_logger.debug("Opper: could not extract usage.cost from response")
|
||||
|
||||
return model_response
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict | httpx.Headers, # mutable-ok: mirrors the BaseConfig signature
|
||||
) -> BaseLLMException:
|
||||
return OpperException(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
def get_model_response_iterator(
|
||||
self,
|
||||
streaming_response: Iterator[str] | AsyncIterator[str] | ModelResponse,
|
||||
sync_stream: bool,
|
||||
json_mode: bool | None = False,
|
||||
) -> "OpperChatCompletionStreamingHandler":
|
||||
return OpperChatCompletionStreamingHandler(
|
||||
streaming_response=streaming_response,
|
||||
sync_stream=sync_stream,
|
||||
json_mode=json_mode,
|
||||
)
|
||||
|
||||
|
||||
class OpperChatCompletionStreamingHandler(OpenAIChatCompletionStreamingHandler):
|
||||
def chunk_parser(self, chunk: dict) -> ModelResponseStream: # mutable-ok: mirrors the base chunk_parser signature
|
||||
parsed: Final = super().chunk_parser(chunk)
|
||||
if getattr(parsed, "usage", None) is not None and not parsed.choices:
|
||||
parsed.choices = [StreamingChoices()]
|
||||
return parsed
|
||||
5
litellm/llms/opper/common_utils.py
Normal file
5
litellm/llms/opper/common_utils.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
||||
|
||||
class OpperException(BaseLLMException):
|
||||
pass
|
||||
|
|
@ -3465,6 +3465,54 @@ def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatch
|
|||
return response
|
||||
|
||||
|
||||
def _complete_opper(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
|
||||
acompletion: Final = ctx.acompletion
|
||||
client: Final = ctx.client
|
||||
litellm_params: Final = ctx.litellm_params
|
||||
logging: Final = ctx.logging
|
||||
messages: Final = ctx.messages
|
||||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
shared_session: Final = ctx.shared_session
|
||||
stream: Final = ctx.stream
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
api_base: Final = (
|
||||
ctx.api_base or litellm.api_base or get_secret_str("OPPER_API_BASE") or "https://api.opper.ai/v3/compat"
|
||||
)
|
||||
api_key: Final = ctx.api_key or litellm.api_key or get_secret_str("OPPER_API_KEY")
|
||||
headers: Final = ctx.headers or litellm.headers
|
||||
|
||||
config: Final = litellm.OpperConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if k not in optional_params:
|
||||
optional_params[k] = v
|
||||
|
||||
response: Final = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
shared_session=shared_session,
|
||||
custom_llm_provider="opper",
|
||||
timeout=timeout,
|
||||
headers=headers,
|
||||
encoding=_get_encoding(),
|
||||
api_key=api_key,
|
||||
logging_obj=logging,
|
||||
client=client,
|
||||
)
|
||||
## LOGGING
|
||||
logging.post_call(input=messages, api_key=api_key, original_response=response)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def _complete_vercel_ai_gateway(
|
||||
ctx: _CompletionDispatchContext,
|
||||
) -> _CompletionDispatchResult:
|
||||
|
|
@ -5810,6 +5858,8 @@ def completion(
|
|||
response = _complete_datarobot(_dispatch_ctx)
|
||||
elif custom_llm_provider == "openrouter":
|
||||
response = _complete_openrouter(_dispatch_ctx)
|
||||
elif custom_llm_provider == "opper":
|
||||
response = _complete_opper(_dispatch_ctx) # rebind-ok: dispatch chain assigns response per provider branch
|
||||
elif custom_llm_provider == "vercel_ai_gateway":
|
||||
response = _complete_vercel_ai_gateway(_dispatch_ctx)
|
||||
elif custom_llm_provider == "palm":
|
||||
|
|
|
|||
|
|
@ -3918,6 +3918,7 @@ class LlmProviders(str, Enum):
|
|||
HUGGINGFACE = "huggingface"
|
||||
TOGETHER_AI = "together_ai"
|
||||
OPENROUTER = "openrouter"
|
||||
OPPER = "opper"
|
||||
DATAROBOT = "datarobot"
|
||||
VERTEX_AI = "vertex_ai"
|
||||
VERTEX_AI_BETA = "vertex_ai_beta"
|
||||
|
|
|
|||
|
|
@ -4699,6 +4699,13 @@ def get_optional_params(
|
|||
model=model,
|
||||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif custom_llm_provider == "opper":
|
||||
optional_params = litellm.OpperConfig().map_openai_params( # rebind-ok: provider chain assigns optional_params per branch
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False),
|
||||
)
|
||||
elif custom_llm_provider == "watsonx":
|
||||
optional_params = litellm.IBMWatsonXChatConfig().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
|
|
@ -6441,6 +6448,11 @@ def validate_environment(
|
|||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("OPENROUTER_API_KEY")
|
||||
elif custom_llm_provider == "opper":
|
||||
if "OPPER_API_KEY" in os.environ:
|
||||
keys_in_environment = True # rebind-ok: provider chain rebinds the shared result flag
|
||||
else:
|
||||
missing_keys.append("OPPER_API_KEY")
|
||||
elif custom_llm_provider == "vercel_ai_gateway":
|
||||
if "VERCEL_AI_GATEWAY_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
|
|
@ -8184,6 +8196,7 @@ class ProviderConfigManager:
|
|||
LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
|
||||
LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False),
|
||||
LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
|
||||
LlmProviders.OPPER: (lambda: litellm.OpperConfig(), False),
|
||||
LlmProviders.VERCEL_AI_GATEWAY: (
|
||||
lambda: litellm.VercelAIGatewayConfig(),
|
||||
False,
|
||||
|
|
|
|||
|
|
@ -1950,6 +1950,23 @@
|
|||
"interactions": true
|
||||
}
|
||||
},
|
||||
"opper": {
|
||||
"display_name": "Opper (`opper`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/opper",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"messages": false,
|
||||
"responses": false,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"a2a": false
|
||||
}
|
||||
},
|
||||
"ovhcloud": {
|
||||
"display_name": "OVHCloud AI Endpoints (`ovhcloud`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/ovhcloud",
|
||||
|
|
|
|||
519
tests/test_litellm/llms/opper/test_opper.py
Normal file
519
tests/test_litellm/llms/opper/test_opper.py
Normal file
|
|
@ -0,0 +1,519 @@
|
|||
import json
|
||||
import os
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
OPPER_API_BASE = "https://api.opper.ai/v3/compat"
|
||||
|
||||
|
||||
def test_opper_provider_registered():
|
||||
import litellm
|
||||
|
||||
assert litellm.LlmProviders.OPPER.value == "opper"
|
||||
assert litellm.LlmProviders("opper") == litellm.LlmProviders.OPPER
|
||||
|
||||
|
||||
def test_opper_provider_detection_by_prefix():
|
||||
"""The provider/model remainder after the opper/ prefix is preserved intact."""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
model, provider, _, api_base = get_llm_provider("opper/anthropic/claude-haiku-4-5")
|
||||
|
||||
assert model == "anthropic/claude-haiku-4-5"
|
||||
assert provider == "opper"
|
||||
assert api_base == OPPER_API_BASE
|
||||
|
||||
|
||||
def test_opper_env_var_resolution():
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"OPPER_API_KEY": "test-key",
|
||||
"OPPER_API_BASE": "https://gw.example.com/v3/compat",
|
||||
},
|
||||
):
|
||||
model, provider, api_key, api_base = get_llm_provider("opper/anthropic/claude-haiku-4-5")
|
||||
|
||||
assert provider == "opper"
|
||||
assert api_key == "test-key"
|
||||
assert api_base == "https://gw.example.com/v3/compat"
|
||||
|
||||
|
||||
def test_opper_chat_config_registered():
|
||||
import litellm
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
config = ProviderConfigManager.get_provider_chat_config(
|
||||
model="anthropic/claude-haiku-4-5", provider=LlmProviders.OPPER
|
||||
)
|
||||
assert isinstance(config, litellm.OpperConfig)
|
||||
|
||||
|
||||
def test_opper_cost_tracking_non_streaming():
|
||||
"""usage.cost from the response body is surfaced as the provider-reported response cost."""
|
||||
import litellm
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
config = litellm.OpperConfig()
|
||||
|
||||
mock_response = Mock(spec=httpx.Response)
|
||||
mock_response.json.return_value = {
|
||||
"id": "chatcmpl-123",
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [
|
||||
{
|
||||
"message": {"role": "assistant", "content": "Hello!"},
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 30,
|
||||
"cost": 3.3e-05,
|
||||
},
|
||||
}
|
||||
mock_response.headers = {}
|
||||
|
||||
model_response = ModelResponse(
|
||||
id="chatcmpl-123",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content="Hello!", role="assistant"),
|
||||
)
|
||||
],
|
||||
created=1234567890,
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
object="chat.completion",
|
||||
usage=Usage(prompt_tokens=10, completion_tokens=20, total_tokens=30),
|
||||
)
|
||||
|
||||
with patch.object(OpenAIGPTConfig, "transform_response", return_value=model_response):
|
||||
result = config.transform_response(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=Mock(),
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
assert result._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] == 3.3e-05
|
||||
|
||||
|
||||
def test_opper_missing_cost_does_not_fail_response():
|
||||
import litellm
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
config = litellm.OpperConfig()
|
||||
|
||||
mock_response = Mock(spec=httpx.Response)
|
||||
mock_response.json.return_value = {"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}
|
||||
mock_response.headers = {}
|
||||
|
||||
model_response = ModelResponse(
|
||||
id="chatcmpl-123",
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
message=Message(content="Hello!", role="assistant"),
|
||||
)
|
||||
],
|
||||
created=1234567890,
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
object="chat.completion",
|
||||
usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2),
|
||||
)
|
||||
|
||||
with patch.object(OpenAIGPTConfig, "transform_response", return_value=model_response):
|
||||
result = config.transform_response(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=Mock(),
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
|
||||
assert "llm_provider-x-litellm-response-cost" not in result._hidden_params.get("additional_headers", {})
|
||||
|
||||
mock_response.json.side_effect = ValueError("malformed body")
|
||||
with patch.object(OpenAIGPTConfig, "transform_response", return_value=model_response):
|
||||
result = config.transform_response(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=Mock(),
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
assert "llm_provider-x-litellm-response-cost" not in result._hidden_params.get("additional_headers", {})
|
||||
|
||||
|
||||
def test_opper_streaming_requests_usage_by_default():
|
||||
"""Streaming requests ask for the usage chunk by default; explicit opt-out wins."""
|
||||
import litellm
|
||||
|
||||
config = litellm.OpperConfig()
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
streaming_request = config.transform_request(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
messages=messages,
|
||||
optional_params={"stream": True},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert streaming_request["stream_options"] == {"include_usage": True}
|
||||
|
||||
opted_out_request = config.transform_request(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
messages=messages,
|
||||
optional_params={"stream": True, "stream_options": {"include_usage": False}},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert opted_out_request["stream_options"] == {"include_usage": False}
|
||||
|
||||
non_streaming_request = config.transform_request(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert "stream_options" not in non_streaming_request
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_opper_streaming_dispatch_carries_cost(respx_mock):
|
||||
"""usage.cost from the terminal usage chunk survives dispatch into stream assembly."""
|
||||
import litellm
|
||||
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
mock_chunks = [
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [{"index": 0, "delta": {"content": "Hello"}, "finish_reason": None}],
|
||||
}
|
||||
)
|
||||
+ "\n\n",
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
}
|
||||
)
|
||||
+ "\n\n",
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 20,
|
||||
"total_tokens": 30,
|
||||
"cost": 3.3e-05,
|
||||
},
|
||||
}
|
||||
)
|
||||
+ "\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
]
|
||||
|
||||
respx_mock.post(f"{OPPER_API_BASE}/chat/completions").respond(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content="".join(mock_chunks),
|
||||
)
|
||||
|
||||
response = litellm.completion(
|
||||
model="opper/anthropic/claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key",
|
||||
stream=True,
|
||||
)
|
||||
yielded_chunks = list(response)
|
||||
|
||||
request_body = json.loads(respx_mock.calls[0].request.content)
|
||||
assert request_body["stream_options"] == {"include_usage": True}
|
||||
|
||||
assert [c.choices[0].delta.content for c in yielded_chunks] == ["Hello", None]
|
||||
assert all(getattr(c, "usage", None) is None for c in yielded_chunks)
|
||||
|
||||
collected_usages = [getattr(c, "usage", None) for c in response.chunks]
|
||||
assert any(u is not None and u.cost == 3.3e-05 for u in collected_usages)
|
||||
complete_response = litellm.stream_chunk_builder(response.chunks, messages=[{"role": "user", "content": "Hello"}])
|
||||
assert complete_response.usage.cost == 3.3e-05
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_opper_reasoning_effort_passes_through(respx_mock):
|
||||
"""reasoning_effort is not rejected client-side and reaches the request body."""
|
||||
import litellm
|
||||
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
config = litellm.OpperConfig()
|
||||
assert "reasoning_effort" in config.get_supported_openai_params(model="anthropic/claude-haiku-4-5")
|
||||
|
||||
respx_mock.post(f"{OPPER_API_BASE}/chat/completions").respond(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello!"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30},
|
||||
},
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="opper/anthropic/claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key",
|
||||
reasoning_effort="high",
|
||||
)
|
||||
|
||||
request_body = json.loads(respx_mock.calls[0].request.content)
|
||||
assert request_body["reasoning_effort"] == "high"
|
||||
|
||||
|
||||
def test_opper_preserves_cache_control_in_content():
|
||||
"""Per-content cache_control markers are not stripped from the request."""
|
||||
import litellm
|
||||
|
||||
config = litellm.OpperConfig()
|
||||
|
||||
transformed_request = config.transform_request(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Long cached context",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{"type": "text", "text": "Question"},
|
||||
],
|
||||
}
|
||||
],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
content = transformed_request["messages"][0]["content"]
|
||||
assert content[0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert "cache_control" not in content[1]
|
||||
|
||||
|
||||
def test_opper_connection_error_reports_opper_endpoint():
|
||||
"""Status-less failures carry Opper request metadata, not the OpenAI default."""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError) as excinfo:
|
||||
exception_type(
|
||||
model="anthropic/claude-haiku-4-5",
|
||||
original_exception=Exception("connection reset"),
|
||||
custom_llm_provider="opper",
|
||||
)
|
||||
|
||||
assert "api.opper.ai" in str(excinfo.value.request.url)
|
||||
|
||||
|
||||
def test_opper_get_llm_provider_with_explicit_provider():
|
||||
"""With custom_llm_provider preset, the opper/ routing prefix is stripped from provider/model IDs."""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
model, provider, _, _ = get_llm_provider("opper/anthropic/claude-haiku-4-5", custom_llm_provider="opper")
|
||||
assert model == "anthropic/claude-haiku-4-5"
|
||||
assert provider == "opper"
|
||||
|
||||
model, provider, _, _ = get_llm_provider("opper/auto", custom_llm_provider="opper")
|
||||
assert model == "opper/auto"
|
||||
assert provider == "opper"
|
||||
|
||||
|
||||
def test_opper_validate_environment():
|
||||
"""validate_environment reports OPPER_API_KEY as the required credential."""
|
||||
import litellm
|
||||
|
||||
with patch.dict(os.environ, {"OPPER_API_KEY": "test-key"}, clear=False):
|
||||
present = litellm.validate_environment("opper/anthropic/claude-haiku-4-5")
|
||||
assert present["keys_in_environment"] is True
|
||||
assert present["missing_keys"] == []
|
||||
|
||||
env_without_key = {k: v for k, v in os.environ.items() if k != "OPPER_API_KEY"}
|
||||
with patch.dict(os.environ, env_without_key, clear=True):
|
||||
missing = litellm.validate_environment("opper/anthropic/claude-haiku-4-5")
|
||||
assert missing["keys_in_environment"] is False
|
||||
assert "OPPER_API_KEY" in missing["missing_keys"]
|
||||
|
||||
|
||||
def test_opper_supported_params_dispatch():
|
||||
"""The get_supported_openai_params dispatch resolves the Opper config."""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
)
|
||||
|
||||
assert litellm.OpperConfig().custom_llm_provider == "opper"
|
||||
params = get_supported_openai_params(model="anthropic/claude-haiku-4-5", custom_llm_provider="opper")
|
||||
assert params is not None
|
||||
assert "reasoning_effort" in params
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_opper_unknown_model_raises_not_found(respx_mock):
|
||||
"""A 404 from the gateway maps to a non-retryable NotFoundError."""
|
||||
import litellm
|
||||
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
respx_mock.post(f"{OPPER_API_BASE}/chat/completions").respond(
|
||||
status_code=404,
|
||||
json={
|
||||
"error": {
|
||||
"message": "model nonexistent/model not found",
|
||||
"type": "model_not_found",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.NotFoundError):
|
||||
litellm.completion(
|
||||
model="opper/nonexistent/model",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.respx()
|
||||
def test_opper_async_completion_and_streaming(respx_mock):
|
||||
"""acompletion works non-streaming and streaming through the async dispatch path."""
|
||||
import asyncio
|
||||
|
||||
import litellm
|
||||
|
||||
litellm.disable_aiohttp_transport = True
|
||||
|
||||
respx_mock.post(f"{OPPER_API_BASE}/chat/completions").respond(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "Hello!"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30, "cost": 3.3e-05},
|
||||
},
|
||||
)
|
||||
|
||||
response = asyncio.run(
|
||||
litellm.acompletion(
|
||||
model="opper/anthropic/claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key",
|
||||
)
|
||||
)
|
||||
assert response.choices[0].message.content == "Hello!"
|
||||
assert response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] == 3.3e-05
|
||||
|
||||
stream_chunks = [
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-124",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [{"index": 0, "delta": {"content": "Hi"}, "finish_reason": "stop"}],
|
||||
}
|
||||
)
|
||||
+ "\n\n",
|
||||
"data: "
|
||||
+ json.dumps(
|
||||
{
|
||||
"id": "chatcmpl-124",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1677652288,
|
||||
"model": "anthropic/claude-haiku-4-5",
|
||||
"choices": [],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7, "cost": 1.1e-05},
|
||||
}
|
||||
)
|
||||
+ "\n\n",
|
||||
"data: [DONE]\n\n",
|
||||
]
|
||||
respx_mock.post(f"{OPPER_API_BASE}/chat/completions").respond(
|
||||
status_code=200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content="".join(stream_chunks),
|
||||
)
|
||||
|
||||
async def consume_stream():
|
||||
stream = await litellm.acompletion(
|
||||
model="opper/anthropic/claude-haiku-4-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_key="test-key",
|
||||
stream=True,
|
||||
)
|
||||
collected = []
|
||||
async for chunk in stream:
|
||||
collected.append(chunk)
|
||||
return stream, collected
|
||||
|
||||
stream, chunks = asyncio.run(consume_stream())
|
||||
assert any(c.choices and c.choices[0].delta.content == "Hi" for c in chunks)
|
||||
assert any(getattr(c, "usage", None) is not None and c.usage.cost == 1.1e-05 for c in stream.chunks)
|
||||
Loading…
Add table
Reference in a new issue