mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
feat(sail): add Sail as a provider with service_tier mapped to its completion window (#42840)
Register Sail (providers.json, LlmProviders.SAIL, OpenAI-compatible lists, ProviderConfigManager) for chat, Responses and /v1/messages, and add its 12 models to both cost maps with asap, balanced and flex price columns. Sail picks speed and price with metadata.completion_window and rejects service_tier, so the Sail chat and Responses configs translate the tier: default and priority to asap, flex to flex, balanced to balanced, auto to no window. Billing prices the window that was sent. A tier Sail has no window for, or a window or tier set where billing cannot see it (request metadata, extra_body), is a 400 unless drop_params is set. Add balanced to ServiceTier and its _balanced price columns to the model info types, the Rust catalog and the dashboard schema. A transform_extra_body hook on the chat and Responses base configs, which returns extra_body unchanged by default, lets Sail keep the window when a caller also sends extra_body.metadata. Sail is listed in the Add Model form and model picker. Co-authored-by: shrey kharbanda <shrey@berri.ai>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
dfb5d905ea
commit
3a6744cd02
40 changed files with 2224 additions and 8 deletions
|
|
@ -362,6 +362,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse
|
|||
| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | |
|
||||
| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
|
||||
|
|
|
|||
|
|
@ -104,6 +104,9 @@ pub struct ModelInfo {
|
|||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
|
||||
/// Balanced service-tier rate for the same-named base field.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_balanced: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read_input_token_cost_batches: Option<f64>,
|
||||
/// Flex service-tier rate for the same-named base field.
|
||||
|
|
@ -211,6 +214,9 @@ pub struct ModelInfo {
|
|||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_above_512k_tokens: Option<f64>,
|
||||
/// Balanced service-tier rate for the same-named base field.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_balanced: Option<f64>,
|
||||
/// USD per prompt token via the provider's batch API.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_cost_per_token_batches: Option<f64>,
|
||||
|
|
@ -357,6 +363,9 @@ pub struct ModelInfo {
|
|||
/// Rate applied once the prompt exceeds the token threshold in the field name.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_above_512k_tokens: Option<f64>,
|
||||
/// Balanced service-tier rate for the same-named base field.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_balanced: Option<f64>,
|
||||
/// USD per generated token via the provider's batch API.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_cost_per_token_batches: Option<f64>,
|
||||
|
|
|
|||
|
|
@ -945,6 +945,7 @@ openai_compatible_endpoints: Final[list] = [
|
|||
"https://api.libertai.io/v1",
|
||||
"https://pinstripes.io/v1",
|
||||
"https://api.meta.ai/v1",
|
||||
"https://api.sailresearch.com/v1",
|
||||
"https://api.cognition.ai/v1",
|
||||
"https://api.scx.ai/v1",
|
||||
"https://gigachat.devices.sberbank.ru/api/v1",
|
||||
|
|
@ -1020,6 +1021,7 @@ openai_compatible_providers: Final[list] = [
|
|||
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
|
||||
"cognition",
|
||||
"scx-ai",
|
||||
"sail",
|
||||
]
|
||||
|
||||
OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ _SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple(
|
|||
_SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
ServiceTier.FLEX.value: ServiceTier.FLEX.value,
|
||||
ServiceTier.BALANCED.value: ServiceTier.BALANCED.value,
|
||||
ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value,
|
||||
ServiceTier.FAST.value: ServiceTier.PRIORITY.value,
|
||||
ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value,
|
||||
|
|
@ -252,7 +253,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
|
|||
|
||||
Args:
|
||||
base_key: The base cost key (e.g., "input_cost_per_token")
|
||||
service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard)
|
||||
service_tier: The service tier ("flex", "balanced", "priority", "fast", "ultrafast", or None for standard)
|
||||
|
||||
Returns:
|
||||
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Common base config for all LLM providers
|
|||
|
||||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Union
|
||||
|
||||
import httpx
|
||||
|
|
@ -255,6 +255,15 @@ class BaseConfig(ABC):
|
|||
) -> dict:
|
||||
pass
|
||||
|
||||
def transform_extra_body(
|
||||
self,
|
||||
extra_body: Mapping[str, object],
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Mapping[str, object]:
|
||||
return extra_body
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: dict,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -364,6 +365,15 @@ class BaseResponsesAPIConfig(ABC):
|
|||
out.append(item)
|
||||
return cast(ResponseInputParam, out)
|
||||
|
||||
def transform_extra_body(
|
||||
self,
|
||||
extra_body: Mapping[str, object],
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
) -> Mapping[str, object]:
|
||||
return extra_body
|
||||
|
||||
@staticmethod
|
||||
def normalize_responses_api_request_dict(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Apply provider-agnostic fixes to an outbound Responses API request dict."""
|
||||
|
|
|
|||
|
|
@ -649,7 +649,16 @@ class BaseLLMHTTPHandler:
|
|||
def sign_and_log(
|
||||
transformed: dict[str, object], # mutable-ok: async_completion takes dict
|
||||
) -> tuple[dict[str, object], dict[str, object], bytes | None]: # mutable-ok: async_completion takes dict
|
||||
data: Final = {**transformed, **extra_body} if extra_body is not None else transformed
|
||||
data: Final = (
|
||||
{
|
||||
**transformed,
|
||||
**provider_config.transform_extra_body(
|
||||
extra_body=extra_body, request=transformed, model=model, litellm_params=litellm_params
|
||||
),
|
||||
}
|
||||
if extra_body is not None
|
||||
else transformed
|
||||
)
|
||||
signed: Final = cast( # cast-ok: sign_request is declared as a bare dict
|
||||
"tuple[dict[str, object], bytes | None]",
|
||||
provider_config.sign_request(
|
||||
|
|
@ -2421,7 +2430,11 @@ class BaseLLMHTTPHandler:
|
|||
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
|
||||
|
||||
if extra_body:
|
||||
data.update(extra_body)
|
||||
data.update(
|
||||
responses_api_provider_config.transform_extra_body(
|
||||
extra_body=extra_body, request=data, model=model, litellm_params=litellm_params
|
||||
)
|
||||
)
|
||||
stream = bool(stream or data.get("stream"))
|
||||
|
||||
# Preserve the OpenAI-style request context (not sent to the provider) for streaming
|
||||
|
|
@ -2609,7 +2622,11 @@ class BaseLLMHTTPHandler:
|
|||
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
|
||||
|
||||
if extra_body:
|
||||
data.update(extra_body)
|
||||
data.update(
|
||||
responses_api_provider_config.transform_extra_body(
|
||||
extra_body=extra_body, request=data, model=model, litellm_params=litellm_params
|
||||
)
|
||||
)
|
||||
stream = bool(stream or data.get("stream"))
|
||||
|
||||
# Preserve the OpenAI-style request context (not sent to the provider) for streaming
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ Dynamic configuration class generator for JSON-based providers.
|
|||
"""
|
||||
|
||||
from collections.abc import Coroutine
|
||||
from typing import Any, Final, Literal, overload
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, overload
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -16,6 +16,9 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
|
||||
from .json_loader import SimpleProviderConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
|
||||
|
||||
|
||||
def create_config_class(provider: SimpleProviderConfig):
|
||||
"""Generate config class dynamically from JSON configuration"""
|
||||
|
|
@ -173,7 +176,7 @@ def create_config_class(provider: SimpleProviderConfig):
|
|||
_responses_config_cache: Final[dict] = {}
|
||||
|
||||
|
||||
def create_responses_config_class(provider: SimpleProviderConfig):
|
||||
def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]":
|
||||
"""Generate a Responses API config class dynamically from JSON configuration.
|
||||
|
||||
Parallel to create_config_class() but for /v1/responses endpoints.
|
||||
|
|
|
|||
|
|
@ -200,5 +200,11 @@
|
|||
"temperature_max": 1.99
|
||||
},
|
||||
"supported_endpoints": ["/v1/chat/completions"]
|
||||
},
|
||||
"sail": {
|
||||
"base_url": "https://api.sailresearch.com/v1",
|
||||
"api_key_env": "SAIL_API_KEY",
|
||||
"api_base_env": "SAIL_API_BASE",
|
||||
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
72
litellm/llms/sail/chat/transformation.py
Normal file
72
litellm/llms/sail/chat/transformation.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
||||
from litellm.llms.sail.common_utils import (
|
||||
chat_request_for_sail,
|
||||
completion_window_for_service_tier,
|
||||
extra_body_for_sail,
|
||||
json_body,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
_REJECTED_BY_SAIL: Final = frozenset(
|
||||
{"stop", "seed", "frequency_penalty", "presence_penalty", "logit_bias", "logprobs", "top_logprobs"}
|
||||
)
|
||||
_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user")
|
||||
|
||||
|
||||
class SailChatConfig(OpenAIGPTConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: return type fixed by the base interface
|
||||
inherited: Final = tuple(
|
||||
param for param in super().get_supported_openai_params(model) if param not in _REJECTED_BY_SAIL
|
||||
)
|
||||
added: Final = tuple(param for param in _ACCEPTED_BY_SAIL if param not in inherited)
|
||||
return [*inherited, *added] # mutable-ok: the base interface returns a list
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
optional_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict: # mutable-ok: return type fixed by the base interface
|
||||
completion_window_for_service_tier(non_default_params.get("service_tier"), model=model, drop_params=drop_params)
|
||||
return super().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[AllMessageValues], # mutable-ok: signature fixed by the base interface
|
||||
optional_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
litellm_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
headers: dict, # mutable-ok: signature fixed by the base interface
|
||||
) -> dict: # mutable-ok: return type fixed by the base interface
|
||||
request: Final = chat_request_for_sail(
|
||||
super().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
),
|
||||
model=model,
|
||||
drop_params=bool(litellm_params.get("drop_params")),
|
||||
)
|
||||
return json_body(request)
|
||||
|
||||
def transform_extra_body(
|
||||
self,
|
||||
extra_body: Mapping[str, object],
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Mapping[str, object]:
|
||||
return extra_body_for_sail(
|
||||
extra_body, request.get("metadata"), model=model, drop_params=bool(litellm_params.get("drop_params"))
|
||||
)
|
||||
177
litellm/llms/sail/common_utils.py
Normal file
177
litellm/llms/sail/common_utils.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
SAIL: Final = LlmProviders.SAIL.value
|
||||
|
||||
CompletionWindow: TypeAlias = Literal["asap", "balanced", "flex"]
|
||||
|
||||
_WINDOW_FOR_SERVICE_TIER: Final[Mapping[str, CompletionWindow | None]] = MappingProxyType(
|
||||
{"auto": None, "default": "asap", "priority": "asap", "flex": "flex", "balanced": "balanced"}
|
||||
)
|
||||
_BILLED_TIER_FOR_WINDOW: Final[Mapping[str, str | None]] = MappingProxyType(
|
||||
{"asap": None, "balanced": "balanced", "standard": "balanced", "flex": "flex"}
|
||||
)
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_DROP_PARAMS_HINT: Final = (
|
||||
"To drop it, set `litellm.drop_params=True` or for proxy: `litellm_settings: drop_params: true`"
|
||||
)
|
||||
|
||||
|
||||
def sail_provider_config() -> SimpleProviderConfig:
|
||||
provider: Final = JSONProviderRegistry.get(SAIL)
|
||||
assert provider is not None, "litellm/llms/openai_like/providers.json ships a 'sail' entry"
|
||||
return provider
|
||||
|
||||
|
||||
def _unsupported(message: str, model: str) -> litellm.UnsupportedParamsError:
|
||||
return litellm.UnsupportedParamsError(message=f"{message} {_DROP_PARAMS_HINT}", llm_provider=SAIL, model=model)
|
||||
|
||||
|
||||
def _dropping(drop_params: bool) -> bool:
|
||||
return drop_params or bool(litellm.drop_params)
|
||||
|
||||
|
||||
def without_keys(mapping: Mapping[str, object], keys: frozenset[str]) -> Mapping[str, object]:
|
||||
return MappingProxyType({key: value for key, value in mapping.items() if key not in keys})
|
||||
|
||||
|
||||
def _entry(key: str, value: object) -> Mapping[str, object]:
|
||||
return MappingProxyType({key: value})
|
||||
|
||||
|
||||
def json_body(mapping: Mapping[str, object]) -> dict[str, object]: # mutable-ok: HTTP bodies are plain dicts
|
||||
return {key: _json_value(value) for key, value in mapping.items()} # mutable-ok: HTTP bodies are plain dicts
|
||||
|
||||
|
||||
def _json_value(value: object) -> object:
|
||||
return json_body(value) if isinstance(value, MappingProxyType) else value
|
||||
|
||||
|
||||
def completion_window_for_service_tier(
|
||||
service_tier: object, *, model: str, drop_params: bool
|
||||
) -> CompletionWindow | None:
|
||||
"""Sail picks speed and price by ``metadata.completion_window`` and rejects
|
||||
``service_tier``, so the tier is translated."""
|
||||
if service_tier is None:
|
||||
return None
|
||||
tier: Final = service_tier.lower() if isinstance(service_tier, str) else None
|
||||
if tier in _WINDOW_FOR_SERVICE_TIER:
|
||||
return _WINDOW_FOR_SERVICE_TIER[tier]
|
||||
if _dropping(drop_params):
|
||||
return None
|
||||
raise _unsupported(
|
||||
f"sail does not support service_tier={service_tier!r}. Supported values: {', '.join(_WINDOW_FOR_SERVICE_TIER)}.",
|
||||
model,
|
||||
)
|
||||
|
||||
|
||||
def _metadata_without_caller_window(
|
||||
metadata: object, *, field: str, model: str, drop_params: bool
|
||||
) -> Mapping[str, object]:
|
||||
"""Chat bills from ``service_tier``, so a window written into metadata would
|
||||
run on Sail at a price LiteLLM never charges."""
|
||||
if not isinstance(metadata, Mapping):
|
||||
return _EMPTY
|
||||
if "completion_window" in metadata and not _dropping(drop_params):
|
||||
raise _unsupported(f"sail does not accept {field}.completion_window. Send service_tier instead.", model)
|
||||
return without_keys(metadata, frozenset({"completion_window"}))
|
||||
|
||||
|
||||
def extra_body_for_sail(
|
||||
extra_body: Mapping[str, object], request_metadata: object, *, model: str, drop_params: bool
|
||||
) -> Mapping[str, object]:
|
||||
"""``extra_body`` keys are sent over the request body, so its ``metadata``
|
||||
would replace the metadata carrying the window. The two are merged, and a
|
||||
tier or window set in ``extra_body`` is rejected because billing cannot see it."""
|
||||
if "service_tier" in extra_body and not _dropping(drop_params):
|
||||
raise _unsupported("sail does not accept service_tier inside extra_body. Send service_tier instead.", model)
|
||||
caller_metadata: Final = _metadata_without_caller_window(
|
||||
extra_body.get("metadata"), field="extra_body.metadata", model=model, drop_params=drop_params
|
||||
)
|
||||
merged_metadata: Final = MappingProxyType(
|
||||
{**caller_metadata, **(request_metadata if isinstance(request_metadata, Mapping) else _EMPTY)}
|
||||
)
|
||||
rest: Final = without_keys(extra_body, frozenset({"service_tier", "metadata"}))
|
||||
raw_metadata: Final = extra_body.get("metadata")
|
||||
if merged_metadata:
|
||||
return json_body(MappingProxyType({**rest, "metadata": merged_metadata}))
|
||||
if isinstance(raw_metadata, Mapping) or "metadata" not in extra_body:
|
||||
return json_body(rest)
|
||||
return json_body(MappingProxyType({**rest, "metadata": raw_metadata}))
|
||||
|
||||
|
||||
def chat_request_for_sail(request: Mapping[str, object], *, model: str, drop_params: bool) -> Mapping[str, object]:
|
||||
raw_tier: Final = request.get("service_tier")
|
||||
window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params)
|
||||
caller_metadata: Final = _metadata_without_caller_window(
|
||||
request.get("metadata"), field="metadata", model=model, drop_params=drop_params
|
||||
)
|
||||
metadata: Final = MappingProxyType({**caller_metadata, "completion_window": window}) if window else caller_metadata
|
||||
extra_body: Final = request.get("extra_body")
|
||||
return MappingProxyType(
|
||||
{
|
||||
**without_keys(request, frozenset({"service_tier", "metadata", "extra_body"})),
|
||||
**(_entry("metadata", metadata) if metadata else _EMPTY),
|
||||
**(
|
||||
_entry("extra_body", extra_body_for_sail(extra_body, metadata, model=model, drop_params=drop_params))
|
||||
if isinstance(extra_body, Mapping)
|
||||
else _EMPTY
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _caller_completion_window(window: object, *, model: str, drop_params: bool) -> str | None:
|
||||
if isinstance(window, str) and window.lower() in _BILLED_TIER_FOR_WINDOW:
|
||||
return window.lower()
|
||||
if _dropping(drop_params):
|
||||
return None
|
||||
raise _unsupported(
|
||||
f"sail does not support metadata.completion_window={window!r}. Supported values: "
|
||||
f"{', '.join(_BILLED_TIER_FOR_WINDOW)}.",
|
||||
model,
|
||||
)
|
||||
|
||||
|
||||
def responses_params_with_completion_window(
|
||||
params: Mapping[str, object], *, model: str, drop_params: bool
|
||||
) -> Mapping[str, object]:
|
||||
"""Responses billing reads these mapped params, so ``service_tier`` is kept
|
||||
as the tier whose price columns match the window and stripped from the body later."""
|
||||
raw_tier: Final = params.get("service_tier")
|
||||
raw_metadata: Final = params.get("metadata")
|
||||
metadata: Final[Mapping[str, object]] = raw_metadata if isinstance(raw_metadata, Mapping) else _EMPTY
|
||||
tier_window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params)
|
||||
caller_window: Final = (
|
||||
_caller_completion_window(metadata["completion_window"], model=model, drop_params=drop_params)
|
||||
if "completion_window" in metadata
|
||||
else None
|
||||
)
|
||||
if (
|
||||
caller_window is not None
|
||||
and tier_window is not None
|
||||
and _BILLED_TIER_FOR_WINDOW[caller_window] != _BILLED_TIER_FOR_WINDOW[tier_window]
|
||||
):
|
||||
raise _unsupported(
|
||||
f"sail got service_tier={raw_tier!r} and metadata.completion_window={caller_window!r}, which "
|
||||
"select different completion windows. Send one of them.",
|
||||
model,
|
||||
)
|
||||
window: Final = caller_window or tier_window
|
||||
other_metadata: Final = without_keys(metadata, frozenset({"completion_window"}))
|
||||
wire_metadata: Final = (
|
||||
MappingProxyType({**other_metadata, "completion_window": window}) if window else other_metadata
|
||||
)
|
||||
billed_tier: Final = _BILLED_TIER_FOR_WINDOW[window] if window else None
|
||||
return MappingProxyType(
|
||||
{
|
||||
**without_keys(params, frozenset({"service_tier", "metadata"})),
|
||||
**(_entry("metadata", wire_metadata) if wire_metadata or raw_metadata is not None else _EMPTY),
|
||||
**(_entry("service_tier", billed_tier) if billed_tier else _EMPTY),
|
||||
}
|
||||
)
|
||||
58
litellm/llms/sail/responses/transformation.py
Normal file
58
litellm/llms/sail/responses/transformation.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.openai_like.dynamic_config import create_responses_config_class
|
||||
from litellm.llms.sail.common_utils import (
|
||||
extra_body_for_sail,
|
||||
json_body,
|
||||
responses_params_with_completion_window,
|
||||
sail_provider_config,
|
||||
without_keys,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
class SailResponsesAPIConfig(create_responses_config_class(sail_provider_config())):
|
||||
def map_openai_params(
|
||||
self,
|
||||
response_api_optional_params: ResponsesAPIOptionalRequestParams,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict: # mutable-ok: return type fixed by the base interface
|
||||
params: Final = responses_params_with_completion_window(
|
||||
super().map_openai_params(
|
||||
response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params
|
||||
),
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
return json_body(params)
|
||||
|
||||
def transform_responses_api_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str | ResponseInputParam,
|
||||
response_api_optional_request_params: dict, # mutable-ok: signature fixed by the base interface
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict, # mutable-ok: signature fixed by the base interface
|
||||
) -> dict: # mutable-ok: return type fixed by the base interface
|
||||
request: Final[Mapping[str, object]] = super().transform_responses_api_request(
|
||||
model=model,
|
||||
input=input,
|
||||
response_api_optional_request_params=response_api_optional_request_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
return json_body(without_keys(request, frozenset({"service_tier"})))
|
||||
|
||||
def transform_extra_body(
|
||||
self,
|
||||
extra_body: Mapping[str, object],
|
||||
request: Mapping[str, object],
|
||||
model: str,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
) -> Mapping[str, object]:
|
||||
return extra_body_for_sail(
|
||||
extra_body, request.get("metadata"), model=model, drop_params=bool(litellm_params.drop_params)
|
||||
)
|
||||
|
|
@ -22229,6 +22229,265 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"mode": "search"
|
||||
},
|
||||
"sail/moonshotai/Kimi-K3": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token": 1.25e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token_balanced": 2e-06,
|
||||
"output_cost_per_token_balanced": 1e-05,
|
||||
"cache_read_input_token_cost_balanced": 2e-07,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"output_cost_per_token_flex": 6.25e-06,
|
||||
"cache_read_input_token_cost_flex": 1.5e-07,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/zai-org/GLM-5.3": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 9.8e-07,
|
||||
"output_cost_per_token": 3.08e-06,
|
||||
"cache_read_input_token_cost": 1.8e-07,
|
||||
"input_cost_per_token_balanced": 5e-07,
|
||||
"output_cost_per_token_balanced": 2.5e-06,
|
||||
"cache_read_input_token_cost_balanced": 1.2e-07,
|
||||
"input_cost_per_token_flex": 4e-07,
|
||||
"output_cost_per_token_flex": 1.8e-06,
|
||||
"cache_read_input_token_cost_flex": 8e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/zai-org/GLM-5.3-Flash": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"output_cost_per_token": 3.5e-07,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token_balanced": 8e-08,
|
||||
"output_cost_per_token_balanced": 2.8e-07,
|
||||
"cache_read_input_token_cost_balanced": 2e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 1.8e-07,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/deepseek-ai/DeepSeek-V4-Pro-0813": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 9.2e-07,
|
||||
"output_cost_per_token": 2.77e-06,
|
||||
"cache_read_input_token_cost": 4e-08,
|
||||
"input_cost_per_token_balanced": 7.4e-07,
|
||||
"output_cost_per_token_balanced": 2.22e-06,
|
||||
"cache_read_input_token_cost_balanced": 3e-08,
|
||||
"input_cost_per_token_flex": 4.6e-07,
|
||||
"output_cost_per_token_flex": 1.39e-06,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/deepseek-ai/DeepSeek-V4-Flash-0731": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 9e-08,
|
||||
"output_cost_per_token": 1.8e-07,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token_balanced": 7e-08,
|
||||
"output_cost_per_token_balanced": 1.4e-07,
|
||||
"cache_read_input_token_cost_balanced": 2e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 9e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
"input_cost_per_token_balanced": 1.2e-07,
|
||||
"output_cost_per_token_balanced": 4.8e-07,
|
||||
"cache_read_input_token_cost_balanced": 5e-09,
|
||||
"input_cost_per_token_flex": 8e-08,
|
||||
"output_cost_per_token_flex": 3e-07,
|
||||
"cache_read_input_token_cost_flex": 4e-09,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/moonshotai/Kimi-K2.6": {
|
||||
"max_tokens": 262144,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token_balanced": 4.5e-07,
|
||||
"output_cost_per_token_balanced": 3e-06,
|
||||
"cache_read_input_token_cost_balanced": 2e-07,
|
||||
"input_cost_per_token_flex": 3.5e-07,
|
||||
"output_cost_per_token_flex": 2e-06,
|
||||
"cache_read_input_token_cost_flex": 1e-07,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/google/gemma-4-31B-it": {
|
||||
"max_tokens": 256000,
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 256000,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token_balanced": 1.2e-07,
|
||||
"output_cost_per_token_balanced": 6e-07,
|
||||
"cache_read_input_token_cost_balanced": 8e-08,
|
||||
"input_cost_per_token_flex": 6e-08,
|
||||
"output_cost_per_token_flex": 3e-07,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/nvidia/Gemma-4-31B-IT-NVFP4": {
|
||||
"max_tokens": 262144,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 7e-08,
|
||||
"input_cost_per_token_balanced": 1.1e-07,
|
||||
"output_cost_per_token_balanced": 3.2e-07,
|
||||
"cache_read_input_token_cost_balanced": 6e-08,
|
||||
"input_cost_per_token_flex": 7e-08,
|
||||
"output_cost_per_token_flex": 2e-07,
|
||||
"cache_read_input_token_cost_flex": 4e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/google/gemma-4-12B-it": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 16384,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token_balanced": 1e-07,
|
||||
"output_cost_per_token_balanced": 2e-06,
|
||||
"cache_read_input_token_cost_balanced": 7e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 1e-06,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/openai/gpt-oss-120b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 6e-08,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/Qwen/Qwen3.6-35B-A3B": {
|
||||
"max_tokens": 262144,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 4e-07,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"searxng/search": {
|
||||
"litellm_provider": "searxng",
|
||||
"mode": "search",
|
||||
|
|
|
|||
|
|
@ -2967,6 +2967,34 @@
|
|||
],
|
||||
"default_model_placeholder": "gpt-3.5-turbo"
|
||||
},
|
||||
{
|
||||
"provider": "Sail",
|
||||
"provider_display_name": "Sail",
|
||||
"litellm_provider": "sail",
|
||||
"credential_fields": [
|
||||
{
|
||||
"key": "api_key",
|
||||
"label": "Sail API Key",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": true,
|
||||
"field_type": "password",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
},
|
||||
{
|
||||
"key": "api_base",
|
||||
"label": "API Base",
|
||||
"placeholder": null,
|
||||
"tooltip": null,
|
||||
"required": false,
|
||||
"field_type": "text",
|
||||
"options": null,
|
||||
"default_value": null
|
||||
}
|
||||
],
|
||||
"default_model_placeholder": "sail/openai/gpt-oss-120b"
|
||||
},
|
||||
{
|
||||
"provider": "Sambanova",
|
||||
"provider_display_name": "Sambanova",
|
||||
|
|
|
|||
|
|
@ -276,6 +276,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
input_cost_per_token: Required[float | None]
|
||||
input_cost_per_token_flex: float | None # OpenAI flex service tier pricing
|
||||
input_cost_per_token_priority: float | None # OpenAI priority service tier pricing
|
||||
input_cost_per_token_balanced: ReadOnly[float | None]
|
||||
input_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
cache_creation_input_token_cost: float | None
|
||||
cache_creation_input_token_cost_above_200k_tokens: float | None
|
||||
|
|
@ -291,6 +292,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
cache_read_input_image_token_cost: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing
|
||||
cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing
|
||||
cache_read_input_token_cost_balanced: ReadOnly[float | None]
|
||||
cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
cache_read_input_token_cost_above_200k_tokens: float | None
|
||||
cache_read_input_token_cost_above_200k_tokens_priority: float | None
|
||||
|
|
@ -337,6 +339,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
|
|||
output_cost_per_token: Required[float | None]
|
||||
output_cost_per_token_flex: float | None # OpenAI flex service tier pricing
|
||||
output_cost_per_token_priority: float | None # OpenAI priority service tier pricing
|
||||
output_cost_per_token_balanced: ReadOnly[float | None]
|
||||
output_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing
|
||||
regional_processing_uplift_multiplier_eu: (
|
||||
float | None
|
||||
|
|
@ -3715,6 +3718,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
# This allows any model_info parameter to be set in litellm_params
|
||||
input_cost_per_token_flex: float | None = None
|
||||
input_cost_per_token_priority: float | None = None
|
||||
input_cost_per_token_balanced: float | None = None
|
||||
input_cost_per_token_ultrafast: float | None = None
|
||||
cache_creation_input_token_cost_above_1hr: float | None = None
|
||||
cache_creation_input_token_cost_above_200k_tokens: float | None = None
|
||||
|
|
@ -3727,6 +3731,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
cache_creation_input_audio_token_cost: float | None = None
|
||||
cache_read_input_token_cost_flex: float | None = None
|
||||
cache_read_input_token_cost_priority: float | None = None
|
||||
cache_read_input_token_cost_balanced: float | None = None
|
||||
cache_read_input_token_cost_ultrafast: float | None = None
|
||||
cache_read_input_token_cost_above_200k_tokens: float | None = None
|
||||
cache_read_input_token_cost_above_200k_tokens_priority: float | None = None
|
||||
|
|
@ -3766,6 +3771,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
output_cost_per_token_batches: float | None = None
|
||||
output_cost_per_token_flex: float | None = None
|
||||
output_cost_per_token_priority: float | None = None
|
||||
output_cost_per_token_balanced: float | None = None
|
||||
output_cost_per_token_ultrafast: float | None = None
|
||||
output_cost_per_audio_token: float | None = None
|
||||
output_cost_per_token_above_128k_tokens: float | None = None
|
||||
|
|
@ -4126,6 +4132,7 @@ class LlmProviders(str, Enum):
|
|||
SCX_AI = "scx-ai"
|
||||
DARKBLOOM = "darkbloom"
|
||||
META = "meta"
|
||||
SAIL = "sail"
|
||||
LITELLM_AGENT = "litellm_agent"
|
||||
CURSOR = "cursor"
|
||||
BEDROCK_MANTLE = "bedrock_mantle"
|
||||
|
|
@ -4386,6 +4393,7 @@ class ServiceTier(Enum):
|
|||
|
||||
AUTO = "auto"
|
||||
FLEX = "flex"
|
||||
BALANCED = "balanced"
|
||||
PRIORITY = "priority"
|
||||
FAST = "fast"
|
||||
ULTRAFAST = "ultrafast"
|
||||
|
|
|
|||
|
|
@ -6114,6 +6114,7 @@ def _get_model_info_helper(
|
|||
input_cost_per_token=_input_cost_per_token,
|
||||
input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None),
|
||||
input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None),
|
||||
input_cost_per_token_balanced=_model_info.get("input_cost_per_token_balanced", None),
|
||||
input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None),
|
||||
cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None),
|
||||
cache_creation_input_token_cost_above_200k_tokens=_model_info.get(
|
||||
|
|
@ -6158,6 +6159,7 @@ def _get_model_info_helper(
|
|||
),
|
||||
cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None),
|
||||
cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None),
|
||||
cache_read_input_token_cost_balanced=_model_info.get("cache_read_input_token_cost_balanced", None),
|
||||
cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None),
|
||||
cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"),
|
||||
cache_read_input_token_cost_above_200k_tokens_batches=_model_info.get(
|
||||
|
|
@ -6219,6 +6221,7 @@ def _get_model_info_helper(
|
|||
output_cost_per_token=_output_cost_per_token,
|
||||
output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None),
|
||||
output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None),
|
||||
output_cost_per_token_balanced=_model_info.get("output_cost_per_token_balanced", None),
|
||||
output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None),
|
||||
regional_processing_uplift_multiplier_eu=_model_info.get(
|
||||
"regional_processing_uplift_multiplier_eu", None
|
||||
|
|
@ -8575,6 +8578,7 @@ class ProviderConfigManager:
|
|||
lambda: ProviderConfigManager._get_langgraph_config(),
|
||||
False,
|
||||
),
|
||||
LlmProviders.SAIL: (ProviderConfigManager._get_sail_chat_config, False),
|
||||
LlmProviders.LANGFLOW: (
|
||||
lambda: ProviderConfigManager._get_langflow_config(),
|
||||
False,
|
||||
|
|
@ -8647,6 +8651,12 @@ class ProviderConfigManager:
|
|||
return litellm.CohereV2ChatConfig()
|
||||
return litellm.CohereChatConfig()
|
||||
|
||||
@staticmethod
|
||||
def _get_sail_chat_config() -> BaseConfig:
|
||||
from litellm.llms.sail.chat.transformation import SailChatConfig
|
||||
|
||||
return SailChatConfig()
|
||||
|
||||
@staticmethod
|
||||
def _get_langgraph_config() -> BaseConfig:
|
||||
"""Get LangGraph config."""
|
||||
|
|
@ -9115,6 +9125,10 @@ class ProviderConfigManager:
|
|||
return None
|
||||
elif litellm.LlmProviders.XAI == provider:
|
||||
return litellm.XAIResponsesAPIConfig()
|
||||
elif litellm.LlmProviders.SAIL == provider:
|
||||
from litellm.llms.sail.responses.transformation import SailResponsesAPIConfig
|
||||
|
||||
return SailResponsesAPIConfig()
|
||||
elif litellm.LlmProviders.GITHUB_COPILOT == provider:
|
||||
from litellm.llms.github_copilot.responses.transformation import (
|
||||
github_copilot_supports_responses_api,
|
||||
|
|
|
|||
|
|
@ -22229,6 +22229,265 @@
|
|||
"litellm_provider": "perplexity",
|
||||
"mode": "search"
|
||||
},
|
||||
"sail/moonshotai/Kimi-K3": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 2.5e-06,
|
||||
"output_cost_per_token": 1.25e-05,
|
||||
"cache_read_input_token_cost": 2.5e-07,
|
||||
"input_cost_per_token_balanced": 2e-06,
|
||||
"output_cost_per_token_balanced": 1e-05,
|
||||
"cache_read_input_token_cost_balanced": 2e-07,
|
||||
"input_cost_per_token_flex": 1.25e-06,
|
||||
"output_cost_per_token_flex": 6.25e-06,
|
||||
"cache_read_input_token_cost_flex": 1.5e-07,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/zai-org/GLM-5.3": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 9.8e-07,
|
||||
"output_cost_per_token": 3.08e-06,
|
||||
"cache_read_input_token_cost": 1.8e-07,
|
||||
"input_cost_per_token_balanced": 5e-07,
|
||||
"output_cost_per_token_balanced": 2.5e-06,
|
||||
"cache_read_input_token_cost_balanced": 1.2e-07,
|
||||
"input_cost_per_token_flex": 4e-07,
|
||||
"output_cost_per_token_flex": 1.8e-06,
|
||||
"cache_read_input_token_cost_flex": 8e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/zai-org/GLM-5.3-Flash": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"output_cost_per_token": 3.5e-07,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token_balanced": 8e-08,
|
||||
"output_cost_per_token_balanced": 2.8e-07,
|
||||
"cache_read_input_token_cost_balanced": 2e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 1.8e-07,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/deepseek-ai/DeepSeek-V4-Pro-0813": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 9.2e-07,
|
||||
"output_cost_per_token": 2.77e-06,
|
||||
"cache_read_input_token_cost": 4e-08,
|
||||
"input_cost_per_token_balanced": 7.4e-07,
|
||||
"output_cost_per_token_balanced": 2.22e-06,
|
||||
"cache_read_input_token_cost_balanced": 3e-08,
|
||||
"input_cost_per_token_flex": 4.6e-07,
|
||||
"output_cost_per_token_flex": 1.39e-06,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/deepseek-ai/DeepSeek-V4-Flash-0731": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 9e-08,
|
||||
"output_cost_per_token": 1.8e-07,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token_balanced": 7e-08,
|
||||
"output_cost_per_token_balanced": 1.4e-07,
|
||||
"cache_read_input_token_cost_balanced": 2e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 9e-08,
|
||||
"cache_read_input_token_cost_flex": 1e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/deepseek-ai/DeepSeek-V4.1-Flash": {
|
||||
"max_tokens": 1048576,
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 1048576,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"cache_read_input_token_cost": 6e-09,
|
||||
"input_cost_per_token_balanced": 1.2e-07,
|
||||
"output_cost_per_token_balanced": 4.8e-07,
|
||||
"cache_read_input_token_cost_balanced": 5e-09,
|
||||
"input_cost_per_token_flex": 8e-08,
|
||||
"output_cost_per_token_flex": 3e-07,
|
||||
"cache_read_input_token_cost_flex": 4e-09,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/moonshotai/Kimi-K2.6": {
|
||||
"max_tokens": 262144,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"input_cost_per_token": 1e-06,
|
||||
"output_cost_per_token": 4e-06,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token_balanced": 4.5e-07,
|
||||
"output_cost_per_token_balanced": 3e-06,
|
||||
"cache_read_input_token_cost_balanced": 2e-07,
|
||||
"input_cost_per_token_flex": 3.5e-07,
|
||||
"output_cost_per_token_flex": 2e-06,
|
||||
"cache_read_input_token_cost_flex": 1e-07,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/google/gemma-4-31B-it": {
|
||||
"max_tokens": 256000,
|
||||
"max_input_tokens": 256000,
|
||||
"max_output_tokens": 256000,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"cache_read_input_token_cost": 2e-07,
|
||||
"input_cost_per_token_balanced": 1.2e-07,
|
||||
"output_cost_per_token_balanced": 6e-07,
|
||||
"cache_read_input_token_cost_balanced": 8e-08,
|
||||
"input_cost_per_token_flex": 6e-08,
|
||||
"output_cost_per_token_flex": 3e-07,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/nvidia/Gemma-4-31B-IT-NVFP4": {
|
||||
"max_tokens": 262144,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 7e-08,
|
||||
"input_cost_per_token_balanced": 1.1e-07,
|
||||
"output_cost_per_token_balanced": 3.2e-07,
|
||||
"cache_read_input_token_cost_balanced": 6e-08,
|
||||
"input_cost_per_token_flex": 7e-08,
|
||||
"output_cost_per_token_flex": 2e-07,
|
||||
"cache_read_input_token_cost_flex": 4e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/google/gemma-4-12B-it": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 16384,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 3e-07,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"input_cost_per_token_balanced": 1e-07,
|
||||
"output_cost_per_token_balanced": 2e-06,
|
||||
"cache_read_input_token_cost_balanced": 7e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 1e-06,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/openai/gpt-oss-120b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 6e-08,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"sail/Qwen/Qwen3.6-35B-A3B": {
|
||||
"max_tokens": 262144,
|
||||
"max_input_tokens": 262144,
|
||||
"max_output_tokens": 262144,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"cache_read_input_token_cost": 2e-08,
|
||||
"input_cost_per_token_flex": 5e-08,
|
||||
"output_cost_per_token_flex": 4e-07,
|
||||
"cache_read_input_token_cost_flex": 2e-08,
|
||||
"litellm_provider": "sail",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true,
|
||||
"source": "https://docs.sailresearch.com/models"
|
||||
},
|
||||
"searxng/search": {
|
||||
"litellm_provider": "searxng",
|
||||
"mode": "search",
|
||||
|
|
|
|||
|
|
@ -220,6 +220,10 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"cache_read_input_token_cost_balanced": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"cache_read_input_token_cost_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
|
|
@ -406,6 +410,10 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"input_cost_per_token_balanced": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"input_cost_per_token_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
@ -768,6 +776,10 @@
|
|||
"minimum": 0,
|
||||
"description": "Rate applied once the prompt exceeds the token threshold in the field name."
|
||||
},
|
||||
"output_cost_per_token_balanced": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
},
|
||||
"output_cost_per_token_batches": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
|
|
|
|||
|
|
@ -2075,6 +2075,23 @@
|
|||
"interactions": true
|
||||
}
|
||||
},
|
||||
"sail": {
|
||||
"display_name": "Sail (`sail`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/sail",
|
||||
"endpoints": {
|
||||
"chat_completions": true,
|
||||
"messages": true,
|
||||
"responses": true,
|
||||
"embeddings": false,
|
||||
"image_generations": false,
|
||||
"audio_transcriptions": false,
|
||||
"audio_speech": false,
|
||||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": false,
|
||||
"a2a": false
|
||||
}
|
||||
},
|
||||
"meta": {
|
||||
"display_name": "Meta Model API (`meta`)",
|
||||
"url": "https://docs.litellm.ai/docs/providers/meta",
|
||||
|
|
|
|||
|
|
@ -100,6 +100,10 @@
|
|||
- {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"}
|
||||
- {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"}
|
||||
- {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"}
|
||||
- {id: llm.chat_completions.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "service_tier flex, balanced and auto map to Sail completion windows and bill the matching price columns"}
|
||||
- {id: llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [rejects_unknown_tier], source: "llm_translation/test_sail_e2e.py", rationale: "A service_tier Sail has no completion window for is a 400 without drop_params"}
|
||||
- {id: llm.responses.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "A caller metadata.completion_window of flex on /v1/responses bills Sail flex rates"}
|
||||
- {id: llm.messages.sail.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: sail, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_sail_e2e.py", rationale: "Sail over /v1/messages"}
|
||||
- {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"}
|
||||
- {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"}
|
||||
- {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"}
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ LlmRoute = Literal[
|
|||
"gemini",
|
||||
"hosted_vllm",
|
||||
"openai",
|
||||
"sail",
|
||||
"together_ai",
|
||||
"vertex",
|
||||
"xiaomi_mimo",
|
||||
|
|
|
|||
209
tests/e2e/llm_translation/test_sail_e2e.py
Normal file
209
tests/e2e/llm_translation/test_sail_e2e.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
"""Live e2e: Sail through the gateway, where LiteLLM turns ``service_tier`` into Sail's
|
||||
``metadata.completion_window`` and bills the price columns of the window it sent.
|
||||
|
||||
The deployment carries its own base, balanced and flex rates, each distinct, so a bill at
|
||||
the wrong tier cannot pass. They are registered on the deployment instead of read from the
|
||||
proxy's cost map, because a stack that loads the published map has no ``sail/`` rows until
|
||||
this provider ships. Requires SAIL_API_KEY on the proxy; no skip gate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS, unique_marker
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, SpendLogRow
|
||||
from openai import OpenAI
|
||||
from proxy_client import ProxyClient
|
||||
from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND: Final = "sail/zai-org/GLM-5.3"
|
||||
PricedTier = Literal["base", "balanced", "flex"]
|
||||
PRICED_TIERS: Final[tuple[PricedTier, ...]] = ("base", "balanced", "flex")
|
||||
PROMPT: Final = "Reply with one word."
|
||||
MAX_TOKENS: Final = 512
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Rates:
|
||||
input: float
|
||||
output: float
|
||||
cache_read: float
|
||||
|
||||
|
||||
RATES: Final[Mapping[PricedTier, _Rates]] = {
|
||||
"base": _Rates(input=3e-06, output=9e-06, cache_read=1e-06),
|
||||
"balanced": _Rates(input=2e-06, output=6e-06, cache_read=7e-07),
|
||||
"flex": _Rates(input=1e-06, output=3e-06, cache_read=4e-07),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Tokens:
|
||||
prompt: int
|
||||
cached: int
|
||||
completion: int
|
||||
|
||||
|
||||
def _approx_equal(actual: float, expected: float) -> bool:
|
||||
return abs(actual - expected) <= max(1e-12, abs(expected) * 1e-2)
|
||||
|
||||
|
||||
def _cost(rates: _Rates, tokens: _Tokens) -> float:
|
||||
return (
|
||||
(tokens.prompt - tokens.cached) * rates.input
|
||||
+ tokens.cached * rates.cache_read
|
||||
+ tokens.completion * rates.output
|
||||
)
|
||||
|
||||
|
||||
def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]:
|
||||
model: Final = f"e2e-sail-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(
|
||||
model=BACKEND,
|
||||
api_key="os.environ/SAIL_API_KEY",
|
||||
input_cost_per_token=RATES["base"].input,
|
||||
output_cost_per_token=RATES["base"].output,
|
||||
cache_read_input_token_cost=RATES["base"].cache_read,
|
||||
input_cost_per_token_balanced=RATES["balanced"].input,
|
||||
output_cost_per_token_balanced=RATES["balanced"].output,
|
||||
cache_read_input_token_cost_balanced=RATES["balanced"].cache_read,
|
||||
input_cost_per_token_flex=RATES["flex"].input,
|
||||
output_cost_per_token_flex=RATES["flex"].output,
|
||||
cache_read_input_token_cost_flex=RATES["flex"].cache_read,
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model, resources.key()
|
||||
|
||||
|
||||
def _openai(sdk: SdkClients, key: str) -> OpenAI:
|
||||
return sdk.openai(key).with_options(timeout=SLOW_PROVIDER_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
def _assert_billed_at(tier: PricedTier, tokens: _Tokens, header_cost: str | None) -> float:
|
||||
assert tokens.prompt > 0 and tokens.completion > 0, f"Sail reported no usage, so no cost is real: {tokens}"
|
||||
assert header_cost is not None, "x-litellm-response-cost header missing"
|
||||
costs: Final = {priced: _cost(rates, tokens) for priced, rates in RATES.items()}
|
||||
assert not any(_approx_equal(costs[other], costs[tier]) for other in PRICED_TIERS if other != tier), (
|
||||
f"{BACKEND} tier rates too close together to tell {tier} apart at {tokens}: {costs}"
|
||||
)
|
||||
assert _approx_equal(float(header_cost), costs[tier]), (
|
||||
f"header cost {header_cost} is not the {tier} price at {tokens}: expected {costs[tier]}, all tiers {costs}"
|
||||
)
|
||||
return float(header_cost)
|
||||
|
||||
|
||||
def _assert_spend_row_matches(proxy: ProxyClient, key: str, header_cost: float) -> None:
|
||||
def priced(rows: list[SpendLogRow]) -> bool:
|
||||
return any((row.spend or 0) > 0 for row in rows)
|
||||
|
||||
rows: Final = [row for row in proxy.poll_logs_for_key(key, predicate=priced) if (row.spend or 0) > 0]
|
||||
assert rows, f"no priced spend row landed for key {key}"
|
||||
assert rows[0].custom_llm_provider == "sail", f"spend row misattributed: {rows[0]}"
|
||||
assert rows[0].spend is not None and _approx_equal(rows[0].spend, header_cost), (
|
||||
f"logged spend {rows[0].spend} disagrees with the x-litellm-response-cost header {header_cost}"
|
||||
)
|
||||
|
||||
|
||||
class TestSailChatCompletions:
|
||||
@pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.cost_logged")
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "billed_tier"), [("flex", "flex"), ("balanced", "balanced"), ("auto", "base")]
|
||||
)
|
||||
def test_service_tier_bills_the_matching_completion_window(
|
||||
self,
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
sdk: SdkClients,
|
||||
service_tier: str,
|
||||
billed_tier: PricedTier,
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
raw: Final = _openai(sdk, key).chat.completions.with_raw_response.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": f"{PROMPT} {unique_marker()}"}],
|
||||
max_completion_tokens=MAX_TOKENS,
|
||||
extra_body={**NO_PROXY_CACHE, "service_tier": service_tier},
|
||||
)
|
||||
usage: Final = raw.parse().usage
|
||||
assert usage is not None, "chat response carries no usage"
|
||||
details: Final = usage.prompt_tokens_details
|
||||
tokens: Final = _Tokens(
|
||||
prompt=usage.prompt_tokens,
|
||||
cached=(details.cached_tokens or 0) if details else 0,
|
||||
completion=usage.completion_tokens,
|
||||
)
|
||||
|
||||
header_cost: Final = _assert_billed_at(
|
||||
billed_tier, tokens, response_header(raw.headers, "x-litellm-response-cost")
|
||||
)
|
||||
_assert_spend_row_matches(proxy, key, header_cost)
|
||||
|
||||
@pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier")
|
||||
def test_unknown_service_tier_is_rejected(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
with pytest.raises(openai.BadRequestError) as raised:
|
||||
_ = _openai(sdk, key).chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": PROMPT}],
|
||||
max_completion_tokens=MAX_TOKENS,
|
||||
extra_body={**NO_PROXY_CACHE, "service_tier": "bogus"},
|
||||
)
|
||||
assert "service_tier" in raised.value.message, f"400 does not name service_tier: {raised.value.message}"
|
||||
|
||||
|
||||
class TestSailResponses:
|
||||
@pytest.mark.covers("llm.responses.sail.service_tier.nonstream.cost_logged")
|
||||
def test_flex_completion_window_bills_flex_rates(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
raw: Final = _openai(sdk, key).responses.with_raw_response.create(
|
||||
model=model,
|
||||
input=f"{PROMPT} {unique_marker()}",
|
||||
max_output_tokens=MAX_TOKENS,
|
||||
metadata={"completion_window": "flex"},
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
usage: Final = raw.parse().usage
|
||||
assert usage is not None, "responses answer carries no usage"
|
||||
tokens: Final = _Tokens(
|
||||
prompt=usage.input_tokens,
|
||||
cached=usage.input_tokens_details.cached_tokens,
|
||||
completion=usage.output_tokens,
|
||||
)
|
||||
|
||||
header_cost: Final = _assert_billed_at("flex", tokens, response_header(raw.headers, "x-litellm-response-cost"))
|
||||
_assert_spend_row_matches(proxy, key, header_cost)
|
||||
|
||||
|
||||
class TestSailMessages:
|
||||
@pytest.mark.covers("llm.messages.sail.basic.nonstream.works")
|
||||
def test_plain_call_returns_a_message(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
|
||||
) -> None:
|
||||
model, key = _register(proxy, resources)
|
||||
|
||||
message: Final = sdk.anthropic(key).messages.create(
|
||||
model=model,
|
||||
max_tokens=MAX_TOKENS,
|
||||
messages=[{"role": "user", "content": PROMPT}],
|
||||
extra_body=NO_PROXY_CACHE,
|
||||
)
|
||||
assert message.role == "assistant" and message.content, f"/v1/messages returned no content: {message}"
|
||||
assert message.usage.output_tokens > 0, f"/v1/messages reported no output usage: {message.usage}"
|
||||
|
|
@ -1207,7 +1207,7 @@ class LiteLLMParamsBody(BaseModel):
|
|||
"""POST /model/new litellm_params: `model` is the only required field; `api_key`
|
||||
et al may be an `os.environ/FOO` reference the proxy resolves at call time.
|
||||
The `*_cost_per_token` / `*_token_cost` fields register a per-deployment custom
|
||||
pricing override (the cache and `_priority` rates only apply when both base
|
||||
pricing override (the cache and service-tier rates only apply when both base
|
||||
rates are set, which is what makes the proxy register the deployment's full
|
||||
pricing entry); left None (and dropped from the body) the deployment keeps the
|
||||
backend's canonical rate."""
|
||||
|
|
@ -1243,6 +1243,12 @@ class LiteLLMParamsBody(BaseModel):
|
|||
cache_creation_input_token_cost: float | None = None
|
||||
input_cost_per_token_priority: float | None = None
|
||||
output_cost_per_token_priority: float | None = None
|
||||
input_cost_per_token_balanced: float | None = None
|
||||
output_cost_per_token_balanced: float | None = None
|
||||
cache_read_input_token_cost_balanced: float | None = None
|
||||
input_cost_per_token_flex: float | None = None
|
||||
output_cost_per_token_flex: float | None = None
|
||||
cache_read_input_token_cost_flex: float | None = None
|
||||
extra_headers: dict[str, str] | None = None
|
||||
use_in_pass_through: bool | None = None
|
||||
complexity_router_config: dict[str, object] | None = None
|
||||
|
|
|
|||
181
tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py
Normal file
181
tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
import asyncio
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
|
||||
|
||||
TIER_MODEL: Final = "tier-priced-test-model"
|
||||
TIER_ROW: Final[Mapping[str, float]] = MappingProxyType(
|
||||
{
|
||||
"input_cost_per_token": 4e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
"output_cost_per_token_flex": 2e-06,
|
||||
"cache_read_input_token_cost_flex": 2.5e-07,
|
||||
"input_cost_per_token_balanced": 2e-06,
|
||||
"output_cost_per_token_balanced": 4e-06,
|
||||
"cache_read_input_token_cost_balanced": 5e-07,
|
||||
}
|
||||
)
|
||||
PROMPT_TOKENS: Final = 1000
|
||||
CACHED_TOKENS: Final = 200
|
||||
COMPLETION_TOKENS: Final = 500
|
||||
TIER_API_BASE: Final = "https://tier-pricing.invalid/v1"
|
||||
|
||||
|
||||
def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float:
|
||||
return (
|
||||
(PROMPT_TOKENS - CACHED_TOKENS) * prices[f"input_cost_per_token{column_suffix}"]
|
||||
+ CACHED_TOKENS * prices[f"cache_read_input_token_cost{column_suffix}"]
|
||||
+ COMPLETION_TOKENS * prices[f"output_cost_per_token{column_suffix}"]
|
||||
)
|
||||
|
||||
|
||||
def _register_tier_model() -> None:
|
||||
litellm.register_model({TIER_MODEL: {"litellm_provider": "openai", "mode": "chat", **TIER_ROW}})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "column_suffix"),
|
||||
[
|
||||
pytest.param(None, "", id="no-tier-bills-base"),
|
||||
pytest.param("auto", "", id="auto-bills-base"),
|
||||
pytest.param("default", "", id="default-bills-base"),
|
||||
pytest.param("priority", "", id="tier-without-columns-bills-base"),
|
||||
pytest.param("flex", "_flex", id="flex"),
|
||||
pytest.param("balanced", "_balanced", id="balanced"),
|
||||
pytest.param("BALANCED", "_balanced", id="balanced-any-case"),
|
||||
],
|
||||
)
|
||||
def test_completion_cost_bills_the_price_columns_of_the_service_tier(
|
||||
local_model_cost_map: None, service_tier: str | None, column_suffix: str
|
||||
) -> None:
|
||||
_register_tier_model()
|
||||
response: Final = ModelResponse(
|
||||
model=TIER_MODEL,
|
||||
usage=Usage(
|
||||
prompt_tokens=PROMPT_TOKENS,
|
||||
completion_tokens=COMPLETION_TOKENS,
|
||||
total_tokens=PROMPT_TOKENS + COMPLETION_TOKENS,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=CACHED_TOKENS),
|
||||
),
|
||||
)
|
||||
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response, model=TIER_MODEL, custom_llm_provider="openai", service_tier=service_tier
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix))
|
||||
|
||||
|
||||
class _CostRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.cost_by_model_group: Mapping[str, float] = MappingProxyType({})
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
payload: Final = kwargs.get("standard_logging_object")
|
||||
cost: Final = kwargs.get("response_cost")
|
||||
if isinstance(payload, dict) and isinstance(cost, float):
|
||||
self.cost_by_model_group = MappingProxyType(
|
||||
{**self.cost_by_model_group, str(payload.get("model_group")): cost}
|
||||
)
|
||||
|
||||
|
||||
async def _logged_cost(recorder: _CostRecorder, model_group: str) -> float:
|
||||
await asyncio.sleep(0)
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
|
||||
assert model_group in recorder.cost_by_model_group, recorder.cost_by_model_group
|
||||
return recorder.cost_by_model_group[model_group]
|
||||
|
||||
|
||||
def _chat_completion_body() -> dict[str, object]:
|
||||
return {
|
||||
"id": "chatcmpl-tier",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": TIER_MODEL,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": PROMPT_TOKENS,
|
||||
"completion_tokens": COMPLETION_TOKENS,
|
||||
"total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
|
||||
"prompt_tokens_details": {"cached_tokens": CACHED_TOKENS},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
DEPLOYMENT_OVERRIDE: Final = 9e-06
|
||||
PRICE_COLUMNS: Final = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")
|
||||
PARITY_ROW: Final[Mapping[str, float]] = MappingProxyType(
|
||||
{
|
||||
"input_cost_per_token": 4e-06,
|
||||
"output_cost_per_token": 8e-06,
|
||||
"cache_read_input_token_cost": 1e-06,
|
||||
**{
|
||||
f"{column}_{tier}": price
|
||||
for tier in ("flex", "balanced")
|
||||
for column, price in zip(PRICE_COLUMNS, (1e-06, 2e-06, 2.5e-07), strict=True)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overridden_columns",
|
||||
[
|
||||
pytest.param((), id="catalog-only"),
|
||||
*(pytest.param((column,), id=f"deployment-overrides-{column}") for column in PRICE_COLUMNS),
|
||||
pytest.param(PRICE_COLUMNS, id="deployment-overrides-all"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_prices_balanced_columns_by_the_same_rules_as_flex(
|
||||
local_model_cost_map: None,
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
overridden_columns: tuple[str, ...],
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
recorder: Final = _CostRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
litellm.register_model({TIER_MODEL: {"litellm_provider": "openai", "mode": "chat", **PARITY_ROW}})
|
||||
respx_mock.post(f"{TIER_API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=_chat_completion_body())
|
||||
)
|
||||
group: Final = {tier: f"{tier}-{uuid.uuid4().hex}" for tier in ("flex", "balanced")}
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": group[tier],
|
||||
"litellm_params": {
|
||||
"model": f"openai/{TIER_MODEL}",
|
||||
"api_key": "sk-test",
|
||||
"api_base": TIER_API_BASE,
|
||||
**{f"{column}_{tier}": DEPLOYMENT_OVERRIDE for column in overridden_columns},
|
||||
},
|
||||
}
|
||||
for tier in ("flex", "balanced")
|
||||
]
|
||||
)
|
||||
|
||||
for tier in ("flex", "balanced"):
|
||||
await router.acompletion(model=group[tier], messages=[{"role": "user", "content": "hi"}], service_tier=tier)
|
||||
flex_cost: Final = await _logged_cost(recorder, group["flex"])
|
||||
balanced_cost: Final = await _logged_cost(recorder, group["balanced"])
|
||||
|
||||
assert balanced_cost == pytest.approx(flex_cost)
|
||||
if not overridden_columns:
|
||||
assert balanced_cost == pytest.approx(_cost_at(PARITY_ROW, "_balanced"))
|
||||
0
tests/unit/llms/base_llm/chat/__init__.py
Normal file
0
tests/unit/llms/base_llm/chat/__init__.py
Normal file
32
tests/unit/llms/base_llm/chat/test_transformation.py
Normal file
32
tests/unit/llms/base_llm/chat/test_transformation.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
def test_base_http_handler_sends_a_caller_extra_body_over_the_request_unchanged(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", "True")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route = respx_mock.post(url__regex=r"https://api\.deepseek\.com/.*chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200, json={"id": "c", "object": "chat.completion", "created": 0, "model": "m", "choices": []}
|
||||
)
|
||||
)
|
||||
|
||||
litellm.completion(
|
||||
model="deepseek/deepseek-chat",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_key="sk-test",
|
||||
temperature=0.5,
|
||||
extra_body={"foo": 1, "temperature": 0.9, "metadata": {"b": "2"}},
|
||||
)
|
||||
|
||||
body = json.loads(route.calls.last.request.content)
|
||||
assert body["foo"] == 1
|
||||
assert body["temperature"] == 0.9
|
||||
assert body["metadata"] == {"b": "2"}
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
"""The shared Responses API config contract."""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
|
@ -33,3 +37,32 @@ async def test_default_async_transform_delegates_to_the_sync_transform():
|
|||
)
|
||||
assert async_body == sync_body
|
||||
assert "cache_control" not in async_body["input"][0]["content"][0]
|
||||
|
||||
|
||||
def test_responses_sends_a_caller_extra_body_over_the_request_unchanged(respx_mock, monkeypatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
route = respx_mock.post("https://api.openai.com/v1/responses").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "m",
|
||||
"output": [],
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
litellm.responses(
|
||||
model="openai/gpt-5",
|
||||
input="hi",
|
||||
api_key="sk-test",
|
||||
metadata={"a": "1"},
|
||||
extra_body={"foo": 1, "metadata": {"b": "2"}},
|
||||
)
|
||||
|
||||
body = json.loads(route.calls.last.request.content)
|
||||
assert body["foo"] == 1
|
||||
assert body["metadata"] == {"b": "2"}
|
||||
|
|
|
|||
0
tests/unit/llms/sail/__init__.py
Normal file
0
tests/unit/llms/sail/__init__.py
Normal file
0
tests/unit/llms/sail/chat/__init__.py
Normal file
0
tests/unit/llms/sail/chat/__init__.py
Normal file
353
tests/unit/llms/sail/chat/test_sail_chat_transformation.py
Normal file
353
tests/unit/llms/sail/chat/test_sail_chat_transformation.py
Normal file
|
|
@ -0,0 +1,353 @@
|
|||
import re
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from tests.unit.llms.sail.helpers import (
|
||||
MODEL,
|
||||
SAIL_API_BASE,
|
||||
SpendCapture,
|
||||
chat_completion_stream,
|
||||
cost_at,
|
||||
sent_body,
|
||||
)
|
||||
|
||||
MESSAGES: Final = [{"role": "user", "content": "hi"}]
|
||||
TIER_CASES: Final = [
|
||||
pytest.param(None, None, "", id="no-tier"),
|
||||
pytest.param("auto", None, "", id="auto"),
|
||||
pytest.param("default", "asap", "", id="default"),
|
||||
pytest.param("priority", "asap", "", id="priority"),
|
||||
pytest.param("flex", "flex", "_flex", id="flex"),
|
||||
pytest.param("balanced", "balanced", "_balanced", id="balanced"),
|
||||
pytest.param("FLEX", "flex", "_flex", id="flex-any-case"),
|
||||
]
|
||||
|
||||
|
||||
def _window(body: dict[str, object]) -> object:
|
||||
metadata: Final = body.get("metadata")
|
||||
return metadata.get("completion_window") if isinstance(metadata, dict) else None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("service_tier", "window", "column_suffix"), TIER_CASES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_sends_the_tier_window_and_bills_its_price_columns(
|
||||
sail_env: None,
|
||||
chat_route: respx.Route,
|
||||
spend_capture: SpendCapture,
|
||||
service_tier: str | None,
|
||||
window: str | None,
|
||||
column_suffix: str,
|
||||
) -> None:
|
||||
await litellm.acompletion(
|
||||
model=MODEL, messages=MESSAGES, service_tier=service_tier, litellm_call_id=spend_capture.call_id
|
||||
)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert "service_tier" not in body
|
||||
assert _window(body) == window
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("service_tier", "window", "column_suffix"), TIER_CASES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_stream_sends_the_tier_window_and_bills_its_price_columns(
|
||||
sail_env: None,
|
||||
respx_mock: respx.MockRouter,
|
||||
spend_capture: SpendCapture,
|
||||
service_tier: str | None,
|
||||
window: str | None,
|
||||
column_suffix: str,
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(f"{SAIL_API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200, content=chat_completion_stream(), headers={"content-type": "text/event-stream"}
|
||||
)
|
||||
)
|
||||
|
||||
stream: Final = await litellm.acompletion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
service_tier=service_tier,
|
||||
stream=True,
|
||||
stream_options={"include_usage": True},
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
body: Final = sent_body(route)
|
||||
assert "service_tier" not in body
|
||||
assert _window(body) == window
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "window"), [pytest.param(*case.values[:2], id=case.id) for case in TIER_CASES]
|
||||
)
|
||||
def test_sail_sync_chat_sends_the_tier_window(
|
||||
sail_env: None, chat_route: respx.Route, service_tier: str | None, window: str | None
|
||||
) -> None:
|
||||
litellm.completion(model=MODEL, messages=MESSAGES, service_tier=service_tier)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert "service_tier" not in body
|
||||
assert _window(body) == window
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service_tier", ["scale", "standard", "asap", 5, ["flex"]])
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_rejects_a_tier_with_no_window_before_sending(
|
||||
sail_env: None, chat_route: respx.Route, service_tier: object
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=re.escape(f"service_tier={service_tier!r}")) as error:
|
||||
await litellm.acompletion(model=MODEL, messages=MESSAGES, service_tier=service_tier)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert not chat_route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service_tier", ["scale", 5])
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_drops_an_unknown_tier_under_drop_params_and_bills_asap(
|
||||
sail_env: None, chat_route: respx.Route, spend_capture: SpendCapture, service_tier: object
|
||||
) -> None:
|
||||
await litellm.acompletion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
service_tier=service_tier,
|
||||
drop_params=True,
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert "service_tier" not in body
|
||||
assert "metadata" not in body
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
|
||||
|
||||
|
||||
@pytest.fixture(params=["openai-sdk", "base-http-handler"])
|
||||
def chat_http_path(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", str(request.param == "base-http-handler"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "wire_metadata", "column_suffix"),
|
||||
[
|
||||
pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"),
|
||||
pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_merges_caller_extra_body_metadata_with_the_tier_window(
|
||||
sail_env: None,
|
||||
chat_http_path: None,
|
||||
chat_route: respx.Route,
|
||||
spend_capture: SpendCapture,
|
||||
service_tier: str | None,
|
||||
wire_metadata: dict[str, str],
|
||||
column_suffix: str,
|
||||
) -> None:
|
||||
await litellm.acompletion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
service_tier=service_tier,
|
||||
extra_body={"metadata": {"trace_id": "t-1"}, "foo": 1},
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert body["metadata"] == wire_metadata
|
||||
assert body["foo"] == 1
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_body", "message"),
|
||||
[
|
||||
pytest.param(
|
||||
{"metadata": {"completion_window": "flex"}},
|
||||
"extra_body.metadata.completion_window",
|
||||
id="extra-body-window",
|
||||
),
|
||||
pytest.param({"service_tier": "flex"}, "service_tier inside extra_body", id="extra-body-tier"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("service_tier", [None, "balanced"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_rejects_a_window_billing_cannot_see_before_sending(
|
||||
sail_env: None,
|
||||
chat_http_path: None,
|
||||
chat_route: respx.Route,
|
||||
service_tier: str | None,
|
||||
extra_body: dict[str, object],
|
||||
message: str,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=message) as error:
|
||||
await litellm.acompletion(model=MODEL, messages=MESSAGES, service_tier=service_tier, extra_body=extra_body)
|
||||
|
||||
assert error.value.status_code == 400
|
||||
assert not chat_route.called
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "wire_metadata", "column_suffix"),
|
||||
[
|
||||
pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"),
|
||||
pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_drops_a_window_billing_cannot_see_under_drop_params(
|
||||
sail_env: None,
|
||||
chat_http_path: None,
|
||||
chat_route: respx.Route,
|
||||
spend_capture: SpendCapture,
|
||||
service_tier: str | None,
|
||||
wire_metadata: dict[str, str],
|
||||
column_suffix: str,
|
||||
) -> None:
|
||||
await litellm.acompletion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
service_tier=service_tier,
|
||||
extra_body={"service_tier": "flex", "metadata": {"trace_id": "t-1", "completion_window": "flex"}},
|
||||
drop_params=True,
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert "service_tier" not in body
|
||||
assert body["metadata"] == wire_metadata
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_drops_a_lone_caller_window_under_drop_params_and_bills_asap(
|
||||
sail_env: None, chat_http_path: None, chat_route: respx.Route, spend_capture: SpendCapture
|
||||
) -> None:
|
||||
await litellm.acompletion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
extra_body={"metadata": {"completion_window": "flex"}},
|
||||
drop_params=True,
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
assert "completion_window" not in (sent_body(chat_route).get("metadata") or {})
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_passes_a_non_mapping_extra_body_metadata_through_untouched(
|
||||
sail_env: None, chat_http_path: None, chat_route: respx.Route
|
||||
) -> None:
|
||||
await litellm.acompletion(model=MODEL, messages=MESSAGES, extra_body={"metadata": None, "foo": 1})
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert "metadata" in body
|
||||
assert body["metadata"] is None
|
||||
assert body["foo"] == 1
|
||||
|
||||
|
||||
def test_sail_sync_chat_rejects_an_unknown_tier_as_unsupported_params(sail_env: None, chat_route: respx.Route) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="service_tier='scale'"):
|
||||
litellm.completion(model=MODEL, messages=MESSAGES, service_tier="scale")
|
||||
|
||||
assert not chat_route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_chat_keeps_the_window_when_preview_features_forward_caller_metadata(
|
||||
sail_env: None,
|
||||
chat_http_path: None,
|
||||
chat_route: respx.Route,
|
||||
spend_capture: SpendCapture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "enable_preview_features", True)
|
||||
|
||||
await litellm.acompletion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
service_tier="flex",
|
||||
metadata={"requester_metadata": {"trace_id": "t-1"}},
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "flex"}
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at("_flex"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rejected",
|
||||
[
|
||||
pytest.param({"stop": ["x"]}, id="stop"),
|
||||
pytest.param({"seed": 1}, id="seed"),
|
||||
pytest.param({"frequency_penalty": 0.5}, id="frequency_penalty"),
|
||||
pytest.param({"presence_penalty": 0.5}, id="presence_penalty"),
|
||||
pytest.param({"logit_bias": {"1": 1}}, id="logit_bias"),
|
||||
pytest.param({"logprobs": True}, id="logprobs"),
|
||||
pytest.param({"top_logprobs": 2}, id="top_logprobs"),
|
||||
],
|
||||
)
|
||||
def test_sail_chat_rejects_params_sail_rejects_unless_dropped(
|
||||
sail_env: None, chat_route: respx.Route, rejected: dict[str, object]
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
litellm.completion(model=MODEL, messages=MESSAGES, **rejected)
|
||||
assert not chat_route.called
|
||||
|
||||
litellm.completion(model=MODEL, messages=MESSAGES, drop_params=True, **rejected)
|
||||
assert set(rejected).isdisjoint(sent_body(chat_route))
|
||||
|
||||
|
||||
def test_sail_chat_forwards_params_sail_accepts(sail_env: None, chat_route: respx.Route) -> None:
|
||||
tools: Final = [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}]
|
||||
|
||||
litellm.completion(
|
||||
model=MODEL,
|
||||
messages=MESSAGES,
|
||||
max_tokens=64,
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
response_format={"type": "json_object"},
|
||||
reasoning_effort="low",
|
||||
user="user-1",
|
||||
)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert body["max_tokens"] == 64
|
||||
assert body["tools"] == tools
|
||||
assert body["tool_choice"] == "auto"
|
||||
assert body["response_format"] == {"type": "json_object"}
|
||||
assert body["reasoning_effort"] == "low"
|
||||
assert body["user"] == "user-1"
|
||||
|
||||
|
||||
def test_sail_chat_passes_max_tokens_and_max_completion_tokens_through_as_sent(
|
||||
sail_env: None, chat_route: respx.Route
|
||||
) -> None:
|
||||
litellm.completion(model=MODEL, messages=MESSAGES, max_tokens=64, max_completion_tokens=32)
|
||||
|
||||
body: Final = sent_body(chat_route)
|
||||
assert body["max_tokens"] == 64
|
||||
assert body["max_completion_tokens"] == 32
|
||||
|
||||
|
||||
def test_sail_chat_uses_sail_api_base_env_and_key(
|
||||
sail_env: None, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("SAIL_API_BASE", "https://sail-gateway.invalid/v1")
|
||||
route: Final = respx_mock.post("https://sail-gateway.invalid/v1/chat/completions").mock(
|
||||
return_value=httpx.Response(
|
||||
200, json={"id": "c", "object": "chat.completion", "created": 0, "model": "m", "choices": []}
|
||||
)
|
||||
)
|
||||
|
||||
litellm.completion(model=MODEL, messages=MESSAGES)
|
||||
|
||||
assert route.calls.last.request.headers["Authorization"] == "Bearer sail-test-key"
|
||||
41
tests/unit/llms/sail/conftest.py
Normal file
41
tests/unit/llms/sail/conftest.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from tests.unit.llms.sail.helpers import SAIL_API_BASE, SpendCapture, chat_completion_body
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sail_env(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
monkeypatch.setenv("SAIL_API_KEY", "sail-test-key")
|
||||
monkeypatch.delenv("SAIL_API_BASE", raising=False)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"disable_aiohttp_transport",
|
||||
True,
|
||||
)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def spend_capture(monkeypatch: pytest.MonkeyPatch) -> SpendCapture:
|
||||
GLOBAL_LOGGING_WORKER.start()
|
||||
capture: Final = SpendCapture(call_id=f"sail-{uuid.uuid4()}")
|
||||
monkeypatch.setattr(litellm, "callbacks", [capture])
|
||||
return capture
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chat_route(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post(f"{SAIL_API_BASE}/chat/completions").mock(
|
||||
return_value=httpx.Response(200, json=chat_completion_body())
|
||||
)
|
||||
119
tests/unit/llms/sail/helpers.py
Normal file
119
tests/unit/llms/sail/helpers.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
|
||||
SAIL_API_BASE: Final = "https://api.sailresearch.com/v1"
|
||||
MODEL: Final = "sail/zai-org/GLM-5.3"
|
||||
PROMPT_TOKENS: Final = 1000
|
||||
CACHED_TOKENS: Final = 200
|
||||
COMPLETION_TOKENS: Final = 500
|
||||
|
||||
|
||||
def cost_at(column_suffix: str) -> float:
|
||||
prices: Final[Mapping[str, object]] = litellm.model_cost[MODEL]
|
||||
return (
|
||||
(PROMPT_TOKENS - CACHED_TOKENS) * float(prices[f"input_cost_per_token{column_suffix}"])
|
||||
+ CACHED_TOKENS * float(prices[f"cache_read_input_token_cost{column_suffix}"])
|
||||
+ COMPLETION_TOKENS * float(prices[f"output_cost_per_token{column_suffix}"])
|
||||
)
|
||||
|
||||
|
||||
def sent_body(route: respx.Route) -> dict[str, object]:
|
||||
return json.loads(route.calls.last.request.content)
|
||||
|
||||
|
||||
def chat_completion_body() -> dict[str, object]:
|
||||
return {
|
||||
"id": "chatcmpl-sail",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "zai-org/GLM-5.3",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": PROMPT_TOKENS,
|
||||
"completion_tokens": COMPLETION_TOKENS,
|
||||
"total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
|
||||
"prompt_tokens_details": {"cached_tokens": CACHED_TOKENS},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def chat_completion_stream() -> bytes:
|
||||
chunk: Final = {"id": "chatcmpl-sail", "object": "chat.completion.chunk", "created": 0, "model": "zai-org/GLM-5.3"}
|
||||
events: Final = (
|
||||
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}]},
|
||||
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
||||
{**chunk, "choices": [], "usage": chat_completion_body()["usage"]},
|
||||
)
|
||||
return "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
def responses_body() -> dict[str, object]:
|
||||
return {
|
||||
"id": "resp_sail",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "zai-org/GLM-5.3",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_sail",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"input_tokens": PROMPT_TOKENS,
|
||||
"input_tokens_details": {"cached_tokens": CACHED_TOKENS},
|
||||
"output_tokens": COMPLETION_TOKENS,
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
"total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def messages_body() -> dict[str, object]:
|
||||
return {
|
||||
"id": "msg_sail",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "zai-org/GLM-5.3",
|
||||
"content": [{"type": "text", "text": "ok"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {
|
||||
"input_tokens": PROMPT_TOKENS - CACHED_TOKENS,
|
||||
"cache_read_input_tokens": CACHED_TOKENS,
|
||||
"output_tokens": COMPLETION_TOKENS,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SpendCapture(CustomLogger):
|
||||
"""Records the cost the spend logs would store for one call, matched by its call id."""
|
||||
|
||||
def __init__(self, call_id: str) -> None:
|
||||
super().__init__()
|
||||
self.call_id = call_id
|
||||
self.costs: tuple[object, ...] = ()
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
if kwargs.get("litellm_call_id") == self.call_id:
|
||||
payload: Final = kwargs.get("standard_logging_object")
|
||||
self.costs = (*self.costs, payload.get("response_cost") if isinstance(payload, dict) else None)
|
||||
|
||||
async def settled_cost(self) -> object:
|
||||
await asyncio.sleep(0)
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
|
||||
assert len(self.costs) == 1, self.costs
|
||||
return self.costs[0]
|
||||
0
tests/unit/llms/sail/messages/__init__.py
Normal file
0
tests/unit/llms/sail/messages/__init__.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, messages_body, sent_body
|
||||
|
||||
MESSAGES: Final = [{"role": "user", "content": "hi"}]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def messages_route(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=httpx.Response(200, json=messages_body()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("service_tier", [None, "auto", "priority", "flex", "balanced", "scale"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier(
|
||||
sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture, service_tier: str | None
|
||||
) -> None:
|
||||
await litellm.anthropic_messages(
|
||||
model=MODEL, messages=MESSAGES, max_tokens=16, service_tier=service_tier, litellm_call_id=spend_capture.call_id
|
||||
)
|
||||
|
||||
body: Final = sent_body(messages_route)
|
||||
assert body["messages"] == MESSAGES
|
||||
assert "service_tier" not in body
|
||||
assert "completion_window" not in (body.get("metadata") or {})
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
|
||||
0
tests/unit/llms/sail/responses/__init__.py
Normal file
0
tests/unit/llms/sail/responses/__init__.py
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
import litellm
|
||||
from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, responses_body, sent_body
|
||||
|
||||
INPUT: Final = "hi"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def responses_route(respx_mock: respx.MockRouter) -> respx.Route:
|
||||
return respx_mock.post(f"{SAIL_API_BASE}/responses").mock(return_value=httpx.Response(200, json=responses_body()))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "metadata", "wire_metadata", "column_suffix"),
|
||||
[
|
||||
pytest.param(None, None, None, "", id="no-tier"),
|
||||
pytest.param("auto", None, None, "", id="auto"),
|
||||
pytest.param("default", None, {"completion_window": "asap"}, "", id="default"),
|
||||
pytest.param("priority", None, {"completion_window": "asap"}, "", id="priority"),
|
||||
pytest.param("flex", None, {"completion_window": "flex"}, "_flex", id="flex"),
|
||||
pytest.param("balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced"),
|
||||
pytest.param("Balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"),
|
||||
pytest.param(
|
||||
"flex", {"user_tag": "a"}, {"user_tag": "a", "completion_window": "flex"}, "_flex", id="tier-keeps-metadata"
|
||||
),
|
||||
pytest.param(None, {"completion_window": "flex"}, {"completion_window": "flex"}, "_flex", id="caller-window"),
|
||||
pytest.param(
|
||||
None,
|
||||
{"completion_window": "standard"},
|
||||
{"completion_window": "standard"},
|
||||
"_balanced",
|
||||
id="standard-window",
|
||||
),
|
||||
pytest.param(None, {"completion_window": "FLEX"}, {"completion_window": "flex"}, "_flex", id="window-any-case"),
|
||||
pytest.param(
|
||||
"priority", {"completion_window": "asap"}, {"completion_window": "asap"}, "", id="agreeing-tier-and-window"
|
||||
),
|
||||
pytest.param(None, {"user_tag": "a"}, {"user_tag": "a"}, "", id="metadata-without-window"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_send_the_window_and_bill_its_price_columns(
|
||||
sail_env: None,
|
||||
responses_route: respx.Route,
|
||||
spend_capture: SpendCapture,
|
||||
service_tier: str | None,
|
||||
metadata: dict[str, str] | None,
|
||||
wire_metadata: dict[str, str] | None,
|
||||
column_suffix: str,
|
||||
) -> None:
|
||||
await litellm.aresponses(
|
||||
model=MODEL,
|
||||
input=INPUT,
|
||||
service_tier=service_tier,
|
||||
metadata=metadata,
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
body: Final = sent_body(responses_route)
|
||||
assert "service_tier" not in body
|
||||
assert body.get("metadata") == wire_metadata
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "metadata", "message"),
|
||||
[
|
||||
pytest.param("scale", None, "service_tier='scale'", id="unknown-tier"),
|
||||
pytest.param(5, None, "service_tier=5", id="non-string-tier"),
|
||||
pytest.param(None, {"completion_window": "soon"}, "completion_window='soon'", id="unknown-window"),
|
||||
pytest.param("flex", {"completion_window": "asap"}, "select different completion windows", id="conflict"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_reject_before_sending(
|
||||
sail_env: None,
|
||||
responses_route: respx.Route,
|
||||
service_tier: object,
|
||||
metadata: dict[str, str] | None,
|
||||
message: str,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=message):
|
||||
await litellm.aresponses(model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata)
|
||||
|
||||
assert not responses_route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params(
|
||||
sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture
|
||||
) -> None:
|
||||
await litellm.aresponses(
|
||||
model=MODEL,
|
||||
input=INPUT,
|
||||
service_tier="scale",
|
||||
metadata={"completion_window": "soon", "user_tag": "a"},
|
||||
drop_params=True,
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
body: Final = sent_body(responses_route)
|
||||
assert "service_tier" not in body
|
||||
assert body["metadata"] == {"user_tag": "a"}
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("service_tier", "wire_metadata", "column_suffix"),
|
||||
[
|
||||
pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"),
|
||||
pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_merge_caller_extra_body_metadata_with_the_tier_window(
|
||||
sail_env: None,
|
||||
responses_route: respx.Route,
|
||||
spend_capture: SpendCapture,
|
||||
service_tier: str | None,
|
||||
wire_metadata: dict[str, str],
|
||||
column_suffix: str,
|
||||
) -> None:
|
||||
await litellm.aresponses(
|
||||
model=MODEL,
|
||||
input=INPUT,
|
||||
service_tier=service_tier,
|
||||
extra_body={"metadata": {"trace_id": "t-1"}, "foo": 1},
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
body: Final = sent_body(responses_route)
|
||||
assert body["metadata"] == wire_metadata
|
||||
assert body["foo"] == 1
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("extra_body", "message"),
|
||||
[
|
||||
pytest.param(
|
||||
{"metadata": {"completion_window": "flex"}},
|
||||
"extra_body.metadata.completion_window",
|
||||
id="extra-body-window",
|
||||
),
|
||||
pytest.param({"service_tier": "flex"}, "service_tier inside extra_body", id="extra-body-tier"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_reject_a_window_billing_cannot_see_before_sending(
|
||||
sail_env: None, responses_route: respx.Route, extra_body: dict[str, object], message: str
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match=message):
|
||||
await litellm.aresponses(model=MODEL, input=INPUT, extra_body=extra_body)
|
||||
|
||||
assert not responses_route.called
|
||||
|
||||
|
||||
def test_sail_sync_responses_drop_a_window_billing_cannot_see_under_drop_params(
|
||||
sail_env: None, responses_route: respx.Route
|
||||
) -> None:
|
||||
litellm.responses(
|
||||
model=MODEL,
|
||||
input=INPUT,
|
||||
service_tier="balanced",
|
||||
extra_body={"service_tier": "flex", "metadata": {"trace_id": "t-1", "completion_window": "flex"}},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
body: Final = sent_body(responses_route)
|
||||
assert "service_tier" not in body
|
||||
assert body["metadata"] == {"trace_id": "t-1", "completion_window": "balanced"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_drop_a_lone_caller_window_under_drop_params_and_bill_asap(
|
||||
sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture
|
||||
) -> None:
|
||||
await litellm.aresponses(
|
||||
model=MODEL,
|
||||
input=INPUT,
|
||||
extra_body={"metadata": {"completion_window": "flex"}},
|
||||
drop_params=True,
|
||||
litellm_call_id=spend_capture.call_id,
|
||||
)
|
||||
|
||||
assert "completion_window" not in (sent_body(responses_route).get("metadata") or {})
|
||||
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
|
||||
|
||||
|
||||
def test_sail_responses_pass_a_non_mapping_extra_body_metadata_through_untouched(
|
||||
sail_env: None, responses_route: respx.Route
|
||||
) -> None:
|
||||
litellm.responses(model=MODEL, input=INPUT, extra_body={"metadata": None, "foo": 1})
|
||||
|
||||
body: Final = sent_body(responses_route)
|
||||
assert "metadata" in body
|
||||
assert body["metadata"] is None
|
||||
assert body["foo"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sail_responses_use_sail_api_base_env_and_key(
|
||||
sail_env: None, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv("SAIL_API_BASE", "https://sail-gateway.invalid/v1")
|
||||
route: Final = respx_mock.post("https://sail-gateway.invalid/v1/responses").mock(
|
||||
return_value=httpx.Response(200, json=responses_body())
|
||||
)
|
||||
|
||||
await litellm.aresponses(model=MODEL, input=INPUT)
|
||||
|
||||
assert route.calls.last.request.headers["Authorization"] == "Bearer sail-test-key"
|
||||
|
|
@ -805,10 +805,12 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"input_cost_per_token_above_512k_tokens": {"type": "number"},
|
||||
"cache_read_input_token_cost_flex": {"type": "number"},
|
||||
"cache_read_input_token_cost_priority": {"type": "number"},
|
||||
"cache_read_input_token_cost_balanced": {"type": "number"},
|
||||
"cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"},
|
||||
"cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"},
|
||||
"input_cost_per_token_flex": {"type": "number"},
|
||||
"input_cost_per_token_priority": {"type": "number"},
|
||||
"input_cost_per_token_balanced": {"type": "number"},
|
||||
"input_cost_per_token_above_200k_tokens_priority": {"type": "number"},
|
||||
"input_cost_per_token_above_272k_tokens_priority": {"type": "number"},
|
||||
"input_cost_per_token_above_272k_tokens_batches": {"type": "number"},
|
||||
|
|
@ -816,6 +818,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"input_cost_per_audio_token_priority": {"type": "number"},
|
||||
"output_cost_per_token_flex": {"type": "number"},
|
||||
"output_cost_per_token_priority": {"type": "number"},
|
||||
"output_cost_per_token_balanced": {"type": "number"},
|
||||
"output_cost_per_token_above_200k_tokens_priority": {"type": "number"},
|
||||
"output_cost_per_token_above_272k_tokens_priority": {"type": "number"},
|
||||
"output_cost_per_token_above_272k_tokens_batches": {"type": "number"},
|
||||
|
|
|
|||
|
|
@ -194,6 +194,7 @@ describe("provider_info_helpers", () => {
|
|||
Providers.PETALS,
|
||||
Providers.PG_VECTOR,
|
||||
Providers.PREDIBASE,
|
||||
Providers.Sail,
|
||||
Providers.WANDB,
|
||||
Providers.ZAI,
|
||||
];
|
||||
|
|
@ -403,6 +404,14 @@ describe("provider_info_helpers", () => {
|
|||
expect(result).not.toContain("anthropic-native");
|
||||
});
|
||||
|
||||
it("should list sail models when called with the 'Sail' provider key", () => {
|
||||
const modelMap = {
|
||||
"sail/openai/gpt-oss-120b": { litellm_provider: "sail" },
|
||||
"sagemaker-model": { litellm_provider: "sagemaker" },
|
||||
};
|
||||
expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]);
|
||||
});
|
||||
|
||||
it("should include bedrock converse but exclude standalone bedrock_mantle when called with 'Bedrock' provider key", () => {
|
||||
const modelMap = {
|
||||
"bedrock-base": { litellm_provider: "bedrock" },
|
||||
|
|
|
|||
|
|
@ -160,6 +160,7 @@ export enum Providers {
|
|||
REPLICATE = "Replicate",
|
||||
RunwayML = "RunwayML",
|
||||
SAGEMAKER_LEGACY = "Sagemaker",
|
||||
Sail = "Sail",
|
||||
Sambanova = "Sambanova",
|
||||
SAP = "SAP Generative AI Hub",
|
||||
SCX_AI = "SCX.ai",
|
||||
|
|
@ -278,6 +279,7 @@ export const provider_map: Record<string, string> = {
|
|||
RunwayML: "runwayml",
|
||||
SAGEMAKER_LEGACY: "sagemaker",
|
||||
SageMaker: "sagemaker_chat",
|
||||
Sail: "sail",
|
||||
Sambanova: "sambanova",
|
||||
SAP: "sap",
|
||||
SCX_AI: "scx-ai",
|
||||
|
|
@ -448,6 +450,7 @@ const providerPlaceholderMap: Partial<Record<Providers, string>> = {
|
|||
[Providers.Oracle]: "oci/xai.grok-4",
|
||||
[Providers.RunwayML]: "runwayml/gen4_turbo",
|
||||
[Providers.SageMaker]: "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b",
|
||||
[Providers.Sail]: "sail/openai/gpt-oss-120b",
|
||||
[Providers.SCX_AI]: "scx-ai/GLM-5.2",
|
||||
[Providers.Snowflake]: "snowflake/mistral-7b",
|
||||
[Providers.Vertex_AI]: "gemini-pro",
|
||||
|
|
|
|||
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -33002,6 +33002,8 @@ export interface components {
|
|||
cache_read_input_token_cost_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Read Input Token Cost Above 512K Tokens */
|
||||
cache_read_input_token_cost_above_512k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Balanced */
|
||||
cache_read_input_token_cost_balanced?: number | null;
|
||||
/** Cache Read Input Token Cost Batches */
|
||||
cache_read_input_token_cost_batches?: number | null;
|
||||
/** Cache Read Input Token Cost Flex */
|
||||
|
|
@ -33082,6 +33084,8 @@ export interface components {
|
|||
input_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Input Cost Per Token Above 512K Tokens */
|
||||
input_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Input Cost Per Token Balanced */
|
||||
input_cost_per_token_balanced?: number | null;
|
||||
/** Input Cost Per Token Batches */
|
||||
input_cost_per_token_batches?: number | null;
|
||||
/** Input Cost Per Token Cache Hit */
|
||||
|
|
@ -33207,6 +33211,8 @@ export interface components {
|
|||
output_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Output Cost Per Token Above 512K Tokens */
|
||||
output_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Output Cost Per Token Balanced */
|
||||
output_cost_per_token_balanced?: number | null;
|
||||
/** Output Cost Per Token Batches */
|
||||
output_cost_per_token_batches?: number | null;
|
||||
/** Output Cost Per Token Flex */
|
||||
|
|
@ -46797,6 +46803,8 @@ export interface components {
|
|||
cache_read_input_token_cost_above_272k_tokens_priority?: number | null;
|
||||
/** Cache Read Input Token Cost Above 512K Tokens */
|
||||
cache_read_input_token_cost_above_512k_tokens?: number | null;
|
||||
/** Cache Read Input Token Cost Balanced */
|
||||
cache_read_input_token_cost_balanced?: number | null;
|
||||
/** Cache Read Input Token Cost Batches */
|
||||
cache_read_input_token_cost_batches?: number | null;
|
||||
/** Cache Read Input Token Cost Flex */
|
||||
|
|
@ -46877,6 +46885,8 @@ export interface components {
|
|||
input_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Input Cost Per Token Above 512K Tokens */
|
||||
input_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Input Cost Per Token Balanced */
|
||||
input_cost_per_token_balanced?: number | null;
|
||||
/** Input Cost Per Token Batches */
|
||||
input_cost_per_token_batches?: number | null;
|
||||
/** Input Cost Per Token Cache Hit */
|
||||
|
|
@ -47002,6 +47012,8 @@ export interface components {
|
|||
output_cost_per_token_above_272k_tokens_priority?: number | null;
|
||||
/** Output Cost Per Token Above 512K Tokens */
|
||||
output_cost_per_token_above_512k_tokens?: number | null;
|
||||
/** Output Cost Per Token Balanced */
|
||||
output_cost_per_token_balanced?: number | null;
|
||||
/** Output Cost Per Token Batches */
|
||||
output_cost_per_token_batches?: number | null;
|
||||
/** Output Cost Per Token Flex */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue