From 3a6744cd02dca739ebaf33e7d879652ac1371915 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 12:57:48 -0700 Subject: [PATCH] feat(sail): add Sail as a provider with service_tier mapped to its completion window (#42840) Register Sail (providers.json, LlmProviders.SAIL, OpenAI-compatible lists, ProviderConfigManager) for chat, Responses and /v1/messages, and add its 12 models to both cost maps with asap, balanced and flex price columns. Sail picks speed and price with metadata.completion_window and rejects service_tier, so the Sail chat and Responses configs translate the tier: default and priority to asap, flex to flex, balanced to balanced, auto to no window. Billing prices the window that was sent. A tier Sail has no window for, or a window or tier set where billing cannot see it (request metadata, extra_body), is a 400 unless drop_params is set. Add balanced to ServiceTier and its _balanced price columns to the model info types, the Rust catalog and the dashboard schema. A transform_extra_body hook on the chat and Responses base configs, which returns extra_body unchanged by default, lets Sail keep the window when a caller also sends extra_body.metadata. Sail is listed in the Add Model form and model picker. Co-authored-by: shrey kharbanda --- README.md | 1 + .../crates/model-catalog/src/model_info.rs | 9 + litellm/constants.py | 2 + .../litellm_core_utils/llm_cost_calc/utils.py | 3 +- litellm/llms/base_llm/chat/transformation.py | 11 +- .../llms/base_llm/responses/transformation.py | 10 + litellm/llms/custom_httpx/llm_http_handler.py | 23 +- litellm/llms/openai_like/dynamic_config.py | 7 +- litellm/llms/openai_like/providers.json | 6 + litellm/llms/sail/chat/transformation.py | 72 ++++ litellm/llms/sail/common_utils.py | 177 +++++++++ litellm/llms/sail/responses/transformation.py | 58 +++ ...odel_prices_and_context_window_backup.json | 259 +++++++++++++ .../provider_create_fields.json | 28 ++ litellm/types/utils.py | 8 + litellm/utils.py | 14 + model_prices_and_context_window.json | 259 +++++++++++++ model_prices_and_context_window.schema.json | 12 + provider_endpoints_support.json | 17 + .../coverage_registry/llm_conversational.yaml | 4 + tests/e2e/coverage_registry/schema.py | 1 + tests/e2e/llm_translation/test_sail_e2e.py | 209 +++++++++++ tests/e2e/models.py | 8 +- .../llm_cost_calc/test_utils.py | 181 +++++++++ tests/unit/llms/base_llm/chat/__init__.py | 0 .../llms/base_llm/chat/test_transformation.py | 32 ++ .../base_llm/responses/test_transformation.py | 33 ++ tests/unit/llms/sail/__init__.py | 0 tests/unit/llms/sail/chat/__init__.py | 0 .../chat/test_sail_chat_transformation.py | 353 ++++++++++++++++++ tests/unit/llms/sail/conftest.py | 41 ++ tests/unit/llms/sail/helpers.py | 119 ++++++ tests/unit/llms/sail/messages/__init__.py | 0 .../test_sail_messages_transformation.py | 31 ++ tests/unit/llms/sail/responses/__init__.py | 0 .../test_sail_responses_transformation.py | 217 +++++++++++ tests/unit/test_utils.py | 3 + .../components/provider_info_helpers.test.tsx | 9 + .../src/components/provider_info_helpers.tsx | 3 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 + 40 files changed, 2224 insertions(+), 8 deletions(-) create mode 100644 litellm/llms/sail/chat/transformation.py create mode 100644 litellm/llms/sail/common_utils.py create mode 100644 litellm/llms/sail/responses/transformation.py create mode 100644 tests/e2e/llm_translation/test_sail_e2e.py create mode 100644 tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py create mode 100644 tests/unit/llms/base_llm/chat/__init__.py create mode 100644 tests/unit/llms/base_llm/chat/test_transformation.py create mode 100644 tests/unit/llms/sail/__init__.py create mode 100644 tests/unit/llms/sail/chat/__init__.py create mode 100644 tests/unit/llms/sail/chat/test_sail_chat_transformation.py create mode 100644 tests/unit/llms/sail/conftest.py create mode 100644 tests/unit/llms/sail/helpers.py create mode 100644 tests/unit/llms/sail/messages/__init__.py create mode 100644 tests/unit/llms/sail/messages/test_sail_messages_transformation.py create mode 100644 tests/unit/llms/sail/responses/__init__.py create mode 100644 tests/unit/llms/sail/responses/test_sail_responses_transformation.py diff --git a/README.md b/README.md index e927c80b8b4..98c5343daee 100644 --- a/README.md +++ b/README.md @@ -362,6 +362,7 @@ For MCP OAuth, an upstream may advertise dynamic client registration but refuse | [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) | | | | | ✅ | | | | | | | [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | | | | | | | | | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | +| [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | | | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index 7b8ce15fbd6..c46a7e57104 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -104,6 +104,9 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_above_512k_tokens: Option, + /// Balanced service-tier rate for the same-named base field. + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read_input_token_cost_balanced: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cache_read_input_token_cost_batches: Option, /// Flex service-tier rate for the same-named base field. @@ -211,6 +214,9 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_above_512k_tokens: Option, + /// Balanced service-tier rate for the same-named base field. + #[serde(skip_serializing_if = "Option::is_none")] + pub input_cost_per_token_balanced: Option, /// USD per prompt token via the provider's batch API. #[serde(skip_serializing_if = "Option::is_none")] pub input_cost_per_token_batches: Option, @@ -357,6 +363,9 @@ pub struct ModelInfo { /// Rate applied once the prompt exceeds the token threshold in the field name. #[serde(skip_serializing_if = "Option::is_none")] pub output_cost_per_token_above_512k_tokens: Option, + /// Balanced service-tier rate for the same-named base field. + #[serde(skip_serializing_if = "Option::is_none")] + pub output_cost_per_token_balanced: Option, /// USD per generated token via the provider's batch API. #[serde(skip_serializing_if = "Option::is_none")] pub output_cost_per_token_batches: Option, diff --git a/litellm/constants.py b/litellm/constants.py index 8316761c95b..73fd11fa4d7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -945,6 +945,7 @@ openai_compatible_endpoints: Final[list] = [ "https://api.libertai.io/v1", "https://pinstripes.io/v1", "https://api.meta.ai/v1", + "https://api.sailresearch.com/v1", "https://api.cognition.ai/v1", "https://api.scx.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", @@ -1020,6 +1021,7 @@ openai_compatible_providers: Final[list] = [ "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", "scx-ai", + "sail", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 46bf2ec2960..795911cafe2 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -70,6 +70,7 @@ _SERVICE_TIER_SUFFIXES: Final[tuple[str, ...]] = tuple( _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType( { ServiceTier.FLEX.value: ServiceTier.FLEX.value, + ServiceTier.BALANCED.value: ServiceTier.BALANCED.value, ServiceTier.PRIORITY.value: ServiceTier.PRIORITY.value, ServiceTier.FAST.value: ServiceTier.PRIORITY.value, ServiceTier.ULTRAFAST.value: ServiceTier.ULTRAFAST.value, @@ -252,7 +253,7 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str: Args: base_key: The base cost key (e.g., "input_cost_per_token") - service_tier: The service tier ("flex", "priority", "fast", "ultrafast", or None for standard) + service_tier: The service tier ("flex", "balanced", "priority", "fast", "ultrafast", or None for standard) Returns: str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token") diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 7decf1b4186..948a9bc6852 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -255,6 +255,15 @@ class BaseConfig(ABC): ) -> dict: pass + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + return extra_body + def sign_request( self, headers: dict, diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index 3834d19ec2b..1b1ea75572e 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -1,5 +1,6 @@ import types from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, cast import httpx @@ -364,6 +365,15 @@ class BaseResponsesAPIConfig(ABC): out.append(item) return cast(ResponseInputParam, out) + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: GenericLiteLLMParams, + ) -> Mapping[str, object]: + return extra_body + @staticmethod def normalize_responses_api_request_dict(data: dict[str, Any]) -> dict[str, Any]: """Apply provider-agnostic fixes to an outbound Responses API request dict.""" diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9eccfe12e71..2f97e306437 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -649,7 +649,16 @@ class BaseLLMHTTPHandler: def sign_and_log( transformed: dict[str, object], # mutable-ok: async_completion takes dict ) -> tuple[dict[str, object], dict[str, object], bytes | None]: # mutable-ok: async_completion takes dict - data: Final = {**transformed, **extra_body} if extra_body is not None else transformed + data: Final = ( + { + **transformed, + **provider_config.transform_extra_body( + extra_body=extra_body, request=transformed, model=model, litellm_params=litellm_params + ), + } + if extra_body is not None + else transformed + ) signed: Final = cast( # cast-ok: sign_request is declared as a bare dict "tuple[dict[str, object], bytes | None]", provider_config.sign_request( @@ -2421,7 +2430,11 @@ class BaseLLMHTTPHandler: data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) if extra_body: - data.update(extra_body) + data.update( + responses_api_provider_config.transform_extra_body( + extra_body=extra_body, request=data, model=model, litellm_params=litellm_params + ) + ) stream = bool(stream or data.get("stream")) # Preserve the OpenAI-style request context (not sent to the provider) for streaming @@ -2609,7 +2622,11 @@ class BaseLLMHTTPHandler: data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data) if extra_body: - data.update(extra_body) + data.update( + responses_api_provider_config.transform_extra_body( + extra_body=extra_body, request=data, model=model, litellm_params=litellm_params + ) + ) stream = bool(stream or data.get("stream")) # Preserve the OpenAI-style request context (not sent to the provider) for streaming diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index 19e29bcdcb2..07d3d078180 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -3,7 +3,7 @@ Dynamic configuration class generator for JSON-based providers. """ from collections.abc import Coroutine -from typing import Any, Final, Literal, overload +from typing import TYPE_CHECKING, Any, Final, Literal, overload from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -16,6 +16,9 @@ from litellm.types.llms.openai import AllMessageValues from .json_loader import SimpleProviderConfig +if TYPE_CHECKING: + from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig + def create_config_class(provider: SimpleProviderConfig): """Generate config class dynamically from JSON configuration""" @@ -173,7 +176,7 @@ def create_config_class(provider: SimpleProviderConfig): _responses_config_cache: Final[dict] = {} -def create_responses_config_class(provider: SimpleProviderConfig): +def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]": """Generate a Responses API config class dynamically from JSON configuration. Parallel to create_config_class() but for /v1/responses endpoints. diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe10293c420..ae09b48bd1e 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -200,5 +200,11 @@ "temperature_max": 1.99 }, "supported_endpoints": ["/v1/chat/completions"] + }, + "sail": { + "base_url": "https://api.sailresearch.com/v1", + "api_key_env": "SAIL_API_KEY", + "api_base_env": "SAIL_API_BASE", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"] } } diff --git a/litellm/llms/sail/chat/transformation.py b/litellm/llms/sail/chat/transformation.py new file mode 100644 index 00000000000..f50ed6de962 --- /dev/null +++ b/litellm/llms/sail/chat/transformation.py @@ -0,0 +1,72 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.llms.sail.common_utils import ( + chat_request_for_sail, + completion_window_for_service_tier, + extra_body_for_sail, + json_body, +) +from litellm.types.llms.openai import AllMessageValues + +_REJECTED_BY_SAIL: Final = frozenset( + {"stop", "seed", "frequency_penalty", "presence_penalty", "logit_bias", "logprobs", "top_logprobs"} +) +_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user") + + +class SailChatConfig(OpenAIGPTConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: return type fixed by the base interface + inherited: Final = tuple( + param for param in super().get_supported_openai_params(model) if param not in _REJECTED_BY_SAIL + ) + added: Final = tuple(param for param in _ACCEPTED_BY_SAIL if param not in inherited) + return [*inherited, *added] # mutable-ok: the base interface returns a list + + def map_openai_params( + self, + non_default_params: dict, # mutable-ok: signature fixed by the base interface + optional_params: dict, # mutable-ok: signature fixed by the base interface + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: return type fixed by the base interface + completion_window_for_service_tier(non_default_params.get("service_tier"), model=model, drop_params=drop_params) + return super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ) + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: signature fixed by the base interface + optional_params: dict, # mutable-ok: signature fixed by the base interface + litellm_params: dict, # mutable-ok: signature fixed by the base interface + headers: dict, # mutable-ok: signature fixed by the base interface + ) -> dict: # mutable-ok: return type fixed by the base interface + request: Final = chat_request_for_sail( + super().transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ), + model=model, + drop_params=bool(litellm_params.get("drop_params")), + ) + return json_body(request) + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + return extra_body_for_sail( + extra_body, request.get("metadata"), model=model, drop_params=bool(litellm_params.get("drop_params")) + ) diff --git a/litellm/llms/sail/common_utils.py b/litellm/llms/sail/common_utils.py new file mode 100644 index 00000000000..a5e9f5e34a1 --- /dev/null +++ b/litellm/llms/sail/common_utils.py @@ -0,0 +1,177 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +import litellm +from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig +from litellm.types.utils import LlmProviders + +SAIL: Final = LlmProviders.SAIL.value + +CompletionWindow: TypeAlias = Literal["asap", "balanced", "flex"] + +_WINDOW_FOR_SERVICE_TIER: Final[Mapping[str, CompletionWindow | None]] = MappingProxyType( + {"auto": None, "default": "asap", "priority": "asap", "flex": "flex", "balanced": "balanced"} +) +_BILLED_TIER_FOR_WINDOW: Final[Mapping[str, str | None]] = MappingProxyType( + {"asap": None, "balanced": "balanced", "standard": "balanced", "flex": "flex"} +) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_DROP_PARAMS_HINT: Final = ( + "To drop it, set `litellm.drop_params=True` or for proxy: `litellm_settings: drop_params: true`" +) + + +def sail_provider_config() -> SimpleProviderConfig: + provider: Final = JSONProviderRegistry.get(SAIL) + assert provider is not None, "litellm/llms/openai_like/providers.json ships a 'sail' entry" + return provider + + +def _unsupported(message: str, model: str) -> litellm.UnsupportedParamsError: + return litellm.UnsupportedParamsError(message=f"{message} {_DROP_PARAMS_HINT}", llm_provider=SAIL, model=model) + + +def _dropping(drop_params: bool) -> bool: + return drop_params or bool(litellm.drop_params) + + +def without_keys(mapping: Mapping[str, object], keys: frozenset[str]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in mapping.items() if key not in keys}) + + +def _entry(key: str, value: object) -> Mapping[str, object]: + return MappingProxyType({key: value}) + + +def json_body(mapping: Mapping[str, object]) -> dict[str, object]: # mutable-ok: HTTP bodies are plain dicts + return {key: _json_value(value) for key, value in mapping.items()} # mutable-ok: HTTP bodies are plain dicts + + +def _json_value(value: object) -> object: + return json_body(value) if isinstance(value, MappingProxyType) else value + + +def completion_window_for_service_tier( + service_tier: object, *, model: str, drop_params: bool +) -> CompletionWindow | None: + """Sail picks speed and price by ``metadata.completion_window`` and rejects + ``service_tier``, so the tier is translated.""" + if service_tier is None: + return None + tier: Final = service_tier.lower() if isinstance(service_tier, str) else None + if tier in _WINDOW_FOR_SERVICE_TIER: + return _WINDOW_FOR_SERVICE_TIER[tier] + if _dropping(drop_params): + return None + raise _unsupported( + f"sail does not support service_tier={service_tier!r}. Supported values: {', '.join(_WINDOW_FOR_SERVICE_TIER)}.", + model, + ) + + +def _metadata_without_caller_window( + metadata: object, *, field: str, model: str, drop_params: bool +) -> Mapping[str, object]: + """Chat bills from ``service_tier``, so a window written into metadata would + run on Sail at a price LiteLLM never charges.""" + if not isinstance(metadata, Mapping): + return _EMPTY + if "completion_window" in metadata and not _dropping(drop_params): + raise _unsupported(f"sail does not accept {field}.completion_window. Send service_tier instead.", model) + return without_keys(metadata, frozenset({"completion_window"})) + + +def extra_body_for_sail( + extra_body: Mapping[str, object], request_metadata: object, *, model: str, drop_params: bool +) -> Mapping[str, object]: + """``extra_body`` keys are sent over the request body, so its ``metadata`` + would replace the metadata carrying the window. The two are merged, and a + tier or window set in ``extra_body`` is rejected because billing cannot see it.""" + if "service_tier" in extra_body and not _dropping(drop_params): + raise _unsupported("sail does not accept service_tier inside extra_body. Send service_tier instead.", model) + caller_metadata: Final = _metadata_without_caller_window( + extra_body.get("metadata"), field="extra_body.metadata", model=model, drop_params=drop_params + ) + merged_metadata: Final = MappingProxyType( + {**caller_metadata, **(request_metadata if isinstance(request_metadata, Mapping) else _EMPTY)} + ) + rest: Final = without_keys(extra_body, frozenset({"service_tier", "metadata"})) + raw_metadata: Final = extra_body.get("metadata") + if merged_metadata: + return json_body(MappingProxyType({**rest, "metadata": merged_metadata})) + if isinstance(raw_metadata, Mapping) or "metadata" not in extra_body: + return json_body(rest) + return json_body(MappingProxyType({**rest, "metadata": raw_metadata})) + + +def chat_request_for_sail(request: Mapping[str, object], *, model: str, drop_params: bool) -> Mapping[str, object]: + raw_tier: Final = request.get("service_tier") + window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params) + caller_metadata: Final = _metadata_without_caller_window( + request.get("metadata"), field="metadata", model=model, drop_params=drop_params + ) + metadata: Final = MappingProxyType({**caller_metadata, "completion_window": window}) if window else caller_metadata + extra_body: Final = request.get("extra_body") + return MappingProxyType( + { + **without_keys(request, frozenset({"service_tier", "metadata", "extra_body"})), + **(_entry("metadata", metadata) if metadata else _EMPTY), + **( + _entry("extra_body", extra_body_for_sail(extra_body, metadata, model=model, drop_params=drop_params)) + if isinstance(extra_body, Mapping) + else _EMPTY + ), + } + ) + + +def _caller_completion_window(window: object, *, model: str, drop_params: bool) -> str | None: + if isinstance(window, str) and window.lower() in _BILLED_TIER_FOR_WINDOW: + return window.lower() + if _dropping(drop_params): + return None + raise _unsupported( + f"sail does not support metadata.completion_window={window!r}. Supported values: " + f"{', '.join(_BILLED_TIER_FOR_WINDOW)}.", + model, + ) + + +def responses_params_with_completion_window( + params: Mapping[str, object], *, model: str, drop_params: bool +) -> Mapping[str, object]: + """Responses billing reads these mapped params, so ``service_tier`` is kept + as the tier whose price columns match the window and stripped from the body later.""" + raw_tier: Final = params.get("service_tier") + raw_metadata: Final = params.get("metadata") + metadata: Final[Mapping[str, object]] = raw_metadata if isinstance(raw_metadata, Mapping) else _EMPTY + tier_window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params) + caller_window: Final = ( + _caller_completion_window(metadata["completion_window"], model=model, drop_params=drop_params) + if "completion_window" in metadata + else None + ) + if ( + caller_window is not None + and tier_window is not None + and _BILLED_TIER_FOR_WINDOW[caller_window] != _BILLED_TIER_FOR_WINDOW[tier_window] + ): + raise _unsupported( + f"sail got service_tier={raw_tier!r} and metadata.completion_window={caller_window!r}, which " + "select different completion windows. Send one of them.", + model, + ) + window: Final = caller_window or tier_window + other_metadata: Final = without_keys(metadata, frozenset({"completion_window"})) + wire_metadata: Final = ( + MappingProxyType({**other_metadata, "completion_window": window}) if window else other_metadata + ) + billed_tier: Final = _BILLED_TIER_FOR_WINDOW[window] if window else None + return MappingProxyType( + { + **without_keys(params, frozenset({"service_tier", "metadata"})), + **(_entry("metadata", wire_metadata) if wire_metadata or raw_metadata is not None else _EMPTY), + **(_entry("service_tier", billed_tier) if billed_tier else _EMPTY), + } + ) diff --git a/litellm/llms/sail/responses/transformation.py b/litellm/llms/sail/responses/transformation.py new file mode 100644 index 00000000000..c3c39a125f5 --- /dev/null +++ b/litellm/llms/sail/responses/transformation.py @@ -0,0 +1,58 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.openai_like.dynamic_config import create_responses_config_class +from litellm.llms.sail.common_utils import ( + extra_body_for_sail, + json_body, + responses_params_with_completion_window, + sail_provider_config, + without_keys, +) +from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams +from litellm.types.router import GenericLiteLLMParams + + +class SailResponsesAPIConfig(create_responses_config_class(sail_provider_config())): + def map_openai_params( + self, + response_api_optional_params: ResponsesAPIOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: return type fixed by the base interface + params: Final = responses_params_with_completion_window( + super().map_openai_params( + response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params + ), + model=model, + drop_params=drop_params, + ) + return json_body(params) + + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict, # mutable-ok: signature fixed by the base interface + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: signature fixed by the base interface + ) -> dict: # mutable-ok: return type fixed by the base interface + request: Final[Mapping[str, object]] = super().transform_responses_api_request( + model=model, + input=input, + response_api_optional_request_params=response_api_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + return json_body(without_keys(request, frozenset({"service_tier"}))) + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: GenericLiteLLMParams, + ) -> Mapping[str, object]: + return extra_body_for_sail( + extra_body, request.get("metadata"), model=model, drop_params=bool(litellm_params.drop_params) + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4b88072b479..82bb1c84dbe 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22229,6 +22229,265 @@ "litellm_provider": "perplexity", "mode": "search" }, + "sail/moonshotai/Kimi-K3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1.25e-05, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token_balanced": 2e-06, + "output_cost_per_token_balanced": 1e-05, + "cache_read_input_token_cost_balanced": 2e-07, + "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_flex": 6.25e-06, + "cache_read_input_token_cost_flex": 1.5e-07, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/zai-org/GLM-5.3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 9.8e-07, + "output_cost_per_token": 3.08e-06, + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token_balanced": 5e-07, + "output_cost_per_token_balanced": 2.5e-06, + "cache_read_input_token_cost_balanced": 1.2e-07, + "input_cost_per_token_flex": 4e-07, + "output_cost_per_token_flex": 1.8e-06, + "cache_read_input_token_cost_flex": 8e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/zai-org/GLM-5.3-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 3.5e-07, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_balanced": 8e-08, + "output_cost_per_token_balanced": 2.8e-07, + "cache_read_input_token_cost_balanced": 2e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 1.8e-07, + "cache_read_input_token_cost_flex": 1e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/deepseek-ai/DeepSeek-V4-Pro-0813": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 9.2e-07, + "output_cost_per_token": 2.77e-06, + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token_balanced": 7.4e-07, + "output_cost_per_token_balanced": 2.22e-06, + "cache_read_input_token_cost_balanced": 3e-08, + "input_cost_per_token_flex": 4.6e-07, + "output_cost_per_token_flex": 1.39e-06, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_balanced": 7e-08, + "output_cost_per_token_balanced": 1.4e-07, + "cache_read_input_token_cost_balanced": 2e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 9e-08, + "cache_read_input_token_cost_flex": 1e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/deepseek-ai/DeepSeek-V4.1-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token_balanced": 1.2e-07, + "output_cost_per_token_balanced": 4.8e-07, + "cache_read_input_token_cost_balanced": 5e-09, + "input_cost_per_token_flex": 8e-08, + "output_cost_per_token_flex": 3e-07, + "cache_read_input_token_cost_flex": 4e-09, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_balanced": 4.5e-07, + "output_cost_per_token_balanced": 3e-06, + "cache_read_input_token_cost_balanced": 2e-07, + "input_cost_per_token_flex": 3.5e-07, + "output_cost_per_token_flex": 2e-06, + "cache_read_input_token_cost_flex": 1e-07, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/google/gemma-4-31B-it": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_balanced": 1.2e-07, + "output_cost_per_token_balanced": 6e-07, + "cache_read_input_token_cost_balanced": 8e-08, + "input_cost_per_token_flex": 6e-08, + "output_cost_per_token_flex": 3e-07, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/nvidia/Gemma-4-31B-IT-NVFP4": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token_balanced": 1.1e-07, + "output_cost_per_token_balanced": 3.2e-07, + "cache_read_input_token_cost_balanced": 6e-08, + "input_cost_per_token_flex": 7e-08, + "output_cost_per_token_flex": 2e-07, + "cache_read_input_token_cost_flex": 4e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/google/gemma-4-12B-it": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token_balanced": 1e-07, + "output_cost_per_token_balanced": 2e-06, + "cache_read_input_token_cost_balanced": 7e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 1e-06, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/openai/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/Qwen/Qwen3.6-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 4e-07, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, "searxng/search": { "litellm_provider": "searxng", "mode": "search", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 87d38606aba..8bd7ed81583 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2967,6 +2967,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "Sail", + "provider_display_name": "Sail", + "litellm_provider": "sail", + "credential_fields": [ + { + "key": "api_key", + "label": "Sail API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + }, + { + "key": "api_base", + "label": "API Base", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "sail/openai/gpt-oss-120b" + }, { "provider": "Sambanova", "provider_display_name": "Sambanova", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2e518af4da4..749ef229fbe 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -276,6 +276,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): input_cost_per_token: Required[float | None] input_cost_per_token_flex: float | None # OpenAI flex service tier pricing input_cost_per_token_priority: float | None # OpenAI priority service tier pricing + input_cost_per_token_balanced: ReadOnly[float | None] input_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_creation_input_token_cost: float | None cache_creation_input_token_cost_above_200k_tokens: float | None @@ -291,6 +292,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_image_token_cost: ReadOnly[float | None] cache_read_input_token_cost_flex: float | None # OpenAI flex service tier pricing cache_read_input_token_cost_priority: float | None # OpenAI priority service tier pricing + cache_read_input_token_cost_balanced: ReadOnly[float | None] cache_read_input_token_cost_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing cache_read_input_token_cost_above_200k_tokens: float | None cache_read_input_token_cost_above_200k_tokens_priority: float | None @@ -337,6 +339,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_token: Required[float | None] output_cost_per_token_flex: float | None # OpenAI flex service tier pricing output_cost_per_token_priority: float | None # OpenAI priority service tier pricing + output_cost_per_token_balanced: ReadOnly[float | None] output_cost_per_token_ultrafast: ReadOnly[float | None] # OpenAI ultrafast service tier pricing regional_processing_uplift_multiplier_eu: ( float | None @@ -3715,6 +3718,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): # This allows any model_info parameter to be set in litellm_params input_cost_per_token_flex: float | None = None input_cost_per_token_priority: float | None = None + input_cost_per_token_balanced: float | None = None input_cost_per_token_ultrafast: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None cache_creation_input_token_cost_above_200k_tokens: float | None = None @@ -3727,6 +3731,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_creation_input_audio_token_cost: float | None = None cache_read_input_token_cost_flex: float | None = None cache_read_input_token_cost_priority: float | None = None + cache_read_input_token_cost_balanced: float | None = None cache_read_input_token_cost_ultrafast: float | None = None cache_read_input_token_cost_above_200k_tokens: float | None = None cache_read_input_token_cost_above_200k_tokens_priority: float | None = None @@ -3766,6 +3771,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): output_cost_per_token_batches: float | None = None output_cost_per_token_flex: float | None = None output_cost_per_token_priority: float | None = None + output_cost_per_token_balanced: float | None = None output_cost_per_token_ultrafast: float | None = None output_cost_per_audio_token: float | None = None output_cost_per_token_above_128k_tokens: float | None = None @@ -4126,6 +4132,7 @@ class LlmProviders(str, Enum): SCX_AI = "scx-ai" DARKBLOOM = "darkbloom" META = "meta" + SAIL = "sail" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" @@ -4386,6 +4393,7 @@ class ServiceTier(Enum): AUTO = "auto" FLEX = "flex" + BALANCED = "balanced" PRIORITY = "priority" FAST = "fast" ULTRAFAST = "ultrafast" diff --git a/litellm/utils.py b/litellm/utils.py index 42da2e2a7b7..e5eea562c11 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6114,6 +6114,7 @@ def _get_model_info_helper( input_cost_per_token=_input_cost_per_token, input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), + input_cost_per_token_balanced=_model_info.get("input_cost_per_token_balanced", None), input_cost_per_token_ultrafast=_model_info.get("input_cost_per_token_ultrafast", None), cache_creation_input_token_cost=_model_info.get("cache_creation_input_token_cost", None), cache_creation_input_token_cost_above_200k_tokens=_model_info.get( @@ -6158,6 +6159,7 @@ def _get_model_info_helper( ), cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), + cache_read_input_token_cost_balanced=_model_info.get("cache_read_input_token_cost_balanced", None), cache_read_input_token_cost_ultrafast=_model_info.get("cache_read_input_token_cost_ultrafast", None), cache_read_input_token_cost_batches=_model_info.get("cache_read_input_token_cost_batches"), cache_read_input_token_cost_above_200k_tokens_batches=_model_info.get( @@ -6219,6 +6221,7 @@ def _get_model_info_helper( output_cost_per_token=_output_cost_per_token, output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), + output_cost_per_token_balanced=_model_info.get("output_cost_per_token_balanced", None), output_cost_per_token_ultrafast=_model_info.get("output_cost_per_token_ultrafast", None), regional_processing_uplift_multiplier_eu=_model_info.get( "regional_processing_uplift_multiplier_eu", None @@ -8575,6 +8578,7 @@ class ProviderConfigManager: lambda: ProviderConfigManager._get_langgraph_config(), False, ), + LlmProviders.SAIL: (ProviderConfigManager._get_sail_chat_config, False), LlmProviders.LANGFLOW: ( lambda: ProviderConfigManager._get_langflow_config(), False, @@ -8647,6 +8651,12 @@ class ProviderConfigManager: return litellm.CohereV2ChatConfig() return litellm.CohereChatConfig() + @staticmethod + def _get_sail_chat_config() -> BaseConfig: + from litellm.llms.sail.chat.transformation import SailChatConfig + + return SailChatConfig() + @staticmethod def _get_langgraph_config() -> BaseConfig: """Get LangGraph config.""" @@ -9115,6 +9125,10 @@ class ProviderConfigManager: return None elif litellm.LlmProviders.XAI == provider: return litellm.XAIResponsesAPIConfig() + elif litellm.LlmProviders.SAIL == provider: + from litellm.llms.sail.responses.transformation import SailResponsesAPIConfig + + return SailResponsesAPIConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: from litellm.llms.github_copilot.responses.transformation import ( github_copilot_supports_responses_api, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4b88072b479..82bb1c84dbe 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -22229,6 +22229,265 @@ "litellm_provider": "perplexity", "mode": "search" }, + "sail/moonshotai/Kimi-K3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 1.25e-05, + "cache_read_input_token_cost": 2.5e-07, + "input_cost_per_token_balanced": 2e-06, + "output_cost_per_token_balanced": 1e-05, + "cache_read_input_token_cost_balanced": 2e-07, + "input_cost_per_token_flex": 1.25e-06, + "output_cost_per_token_flex": 6.25e-06, + "cache_read_input_token_cost_flex": 1.5e-07, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/zai-org/GLM-5.3": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 9.8e-07, + "output_cost_per_token": 3.08e-06, + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token_balanced": 5e-07, + "output_cost_per_token_balanced": 2.5e-06, + "cache_read_input_token_cost_balanced": 1.2e-07, + "input_cost_per_token_flex": 4e-07, + "output_cost_per_token_flex": 1.8e-06, + "cache_read_input_token_cost_flex": 8e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/zai-org/GLM-5.3-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.1e-07, + "output_cost_per_token": 3.5e-07, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_balanced": 8e-08, + "output_cost_per_token_balanced": 2.8e-07, + "cache_read_input_token_cost_balanced": 2e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 1.8e-07, + "cache_read_input_token_cost_flex": 1e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/deepseek-ai/DeepSeek-V4-Pro-0813": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 9.2e-07, + "output_cost_per_token": 2.77e-06, + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token_balanced": 7.4e-07, + "output_cost_per_token_balanced": 2.22e-06, + "cache_read_input_token_cost_balanced": 3e-08, + "input_cost_per_token_flex": 4.6e-07, + "output_cost_per_token_flex": 1.39e-06, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/deepseek-ai/DeepSeek-V4-Flash-0731": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 9e-08, + "output_cost_per_token": 1.8e-07, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_balanced": 7e-08, + "output_cost_per_token_balanced": 1.4e-07, + "cache_read_input_token_cost_balanced": 2e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 9e-08, + "cache_read_input_token_cost_flex": 1e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/deepseek-ai/DeepSeek-V4.1-Flash": { + "max_tokens": 1048576, + "max_input_tokens": 1048576, + "max_output_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 6e-09, + "input_cost_per_token_balanced": 1.2e-07, + "output_cost_per_token_balanced": 4.8e-07, + "cache_read_input_token_cost_balanced": 5e-09, + "input_cost_per_token_flex": 8e-08, + "output_cost_per_token_flex": 3e-07, + "cache_read_input_token_cost_flex": 4e-09, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/moonshotai/Kimi-K2.6": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_balanced": 4.5e-07, + "output_cost_per_token_balanced": 3e-06, + "cache_read_input_token_cost_balanced": 2e-07, + "input_cost_per_token_flex": 3.5e-07, + "output_cost_per_token_flex": 2e-06, + "cache_read_input_token_cost_flex": 1e-07, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/google/gemma-4-31B-it": { + "max_tokens": 256000, + "max_input_tokens": 256000, + "max_output_tokens": 256000, + "input_cost_per_token": 4e-07, + "output_cost_per_token": 6e-07, + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token_balanced": 1.2e-07, + "output_cost_per_token_balanced": 6e-07, + "cache_read_input_token_cost_balanced": 8e-08, + "input_cost_per_token_flex": 6e-08, + "output_cost_per_token_flex": 3e-07, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/nvidia/Gemma-4-31B-IT-NVFP4": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 1.4e-07, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token_balanced": 1.1e-07, + "output_cost_per_token_balanced": 3.2e-07, + "cache_read_input_token_cost_balanced": 6e-08, + "input_cost_per_token_flex": 7e-08, + "output_cost_per_token_flex": 2e-07, + "cache_read_input_token_cost_flex": 4e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/google/gemma-4-12B-it": { + "max_tokens": 16384, + "max_input_tokens": 16384, + "max_output_tokens": 16384, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token_balanced": 1e-07, + "output_cost_per_token_balanced": 2e-06, + "cache_read_input_token_cost_balanced": 7e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 1e-06, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/openai/gpt-oss-120b": { + "max_tokens": 131072, + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "input_cost_per_token": 6e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 3e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "source": "https://docs.sailresearch.com/models" + }, + "sail/Qwen/Qwen3.6-35B-A3B": { + "max_tokens": 262144, + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "input_cost_per_token": 5e-08, + "output_cost_per_token": 4e-07, + "cache_read_input_token_cost": 2e-08, + "input_cost_per_token_flex": 5e-08, + "output_cost_per_token_flex": 4e-07, + "cache_read_input_token_cost_flex": 2e-08, + "litellm_provider": "sail", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_vision": true, + "source": "https://docs.sailresearch.com/models" + }, "searxng/search": { "litellm_provider": "searxng", "mode": "search", diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index c4048cac905..e893b6265fa 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -220,6 +220,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "cache_read_input_token_cost_balanced": { + "type": "number", + "minimum": 0 + }, "cache_read_input_token_cost_batches": { "type": "number", "minimum": 0 @@ -406,6 +410,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "input_cost_per_token_balanced": { + "type": "number", + "minimum": 0 + }, "input_cost_per_token_batches": { "type": "number", "minimum": 0, @@ -768,6 +776,10 @@ "minimum": 0, "description": "Rate applied once the prompt exceeds the token threshold in the field name." }, + "output_cost_per_token_balanced": { + "type": "number", + "minimum": 0 + }, "output_cost_per_token_batches": { "type": "number", "minimum": 0, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index e6cb0592a15..790a050a878 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2075,6 +2075,23 @@ "interactions": true } }, + "sail": { + "display_name": "Sail (`sail`)", + "url": "https://docs.litellm.ai/docs/providers/sail", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "meta": { "display_name": "Meta Model API (`meta`)", "url": "https://docs.litellm.ai/docs/providers/meta", diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 9cfe6e33ed6..cd52e563d69 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -100,6 +100,10 @@ - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"} - {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"} - {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"} +- {id: llm.chat_completions.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "service_tier flex, balanced and auto map to Sail completion windows and bill the matching price columns"} +- {id: llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: sail, capability: service_tier, streaming: nonstream, assertions: [rejects_unknown_tier], source: "llm_translation/test_sail_e2e.py", rationale: "A service_tier Sail has no completion window for is a 400 without drop_params"} +- {id: llm.responses.sail.service_tier.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: responses, route: sail, capability: service_tier, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_sail_e2e.py", rationale: "A caller metadata.completion_window of flex on /v1/responses bills Sail flex rates"} +- {id: llm.messages.sail.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: sail, capability: basic, streaming: nonstream, assertions: [works], source: "llm_translation/test_sail_e2e.py", rationale: "Sail over /v1/messages"} - {id: llm.chat_completions.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic over /chat/completions: cost header and spend row agree"} - {id: llm.chat_completions.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /chat/completions"} - {id: llm.messages.anthropic.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_conversational_matrix_e2e.py", rationale: "Anthropic tool result round trip over /v1/messages"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index 8417b51360e..d089b9c1ed8 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -56,6 +56,7 @@ LlmRoute = Literal[ "gemini", "hosted_vllm", "openai", + "sail", "together_ai", "vertex", "xiaomi_mimo", diff --git a/tests/e2e/llm_translation/test_sail_e2e.py b/tests/e2e/llm_translation/test_sail_e2e.py new file mode 100644 index 00000000000..9c714544d6e --- /dev/null +++ b/tests/e2e/llm_translation/test_sail_e2e.py @@ -0,0 +1,209 @@ +"""Live e2e: Sail through the gateway, where LiteLLM turns ``service_tier`` into Sail's +``metadata.completion_window`` and bills the price columns of the window it sent. + +The deployment carries its own base, balanced and flex rates, each distinct, so a bill at +the wrong tier cannot pass. They are registered on the deployment instead of read from the +proxy's cost map, because a stack that loads the published map has no ``sail/`` rows until +this provider ships. Requires SAIL_API_KEY on the proxy; no skip gate. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final, Literal + +import openai +import pytest +from e2e_config import SLOW_PROVIDER_TIMEOUT_SECONDS, unique_marker +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, SpendLogRow +from openai import OpenAI +from proxy_client import ProxyClient +from sdk_clients import NO_PROXY_CACHE, SdkClients, response_header + +pytestmark = pytest.mark.e2e + +BACKEND: Final = "sail/zai-org/GLM-5.3" +PricedTier = Literal["base", "balanced", "flex"] +PRICED_TIERS: Final[tuple[PricedTier, ...]] = ("base", "balanced", "flex") +PROMPT: Final = "Reply with one word." +MAX_TOKENS: Final = 512 + + +@dataclass(frozen=True, slots=True) +class _Rates: + input: float + output: float + cache_read: float + + +RATES: Final[Mapping[PricedTier, _Rates]] = { + "base": _Rates(input=3e-06, output=9e-06, cache_read=1e-06), + "balanced": _Rates(input=2e-06, output=6e-06, cache_read=7e-07), + "flex": _Rates(input=1e-06, output=3e-06, cache_read=4e-07), +} + + +@dataclass(frozen=True, slots=True) +class _Tokens: + prompt: int + cached: int + completion: int + + +def _approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-12, abs(expected) * 1e-2) + + +def _cost(rates: _Rates, tokens: _Tokens) -> float: + return ( + (tokens.prompt - tokens.cached) * rates.input + + tokens.cached * rates.cache_read + + tokens.completion * rates.output + ) + + +def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: + model: Final = f"e2e-sail-{unique_marker()}" + model_id: Final = proxy.create_model( + model, + LiteLLMParamsBody( + model=BACKEND, + api_key="os.environ/SAIL_API_KEY", + input_cost_per_token=RATES["base"].input, + output_cost_per_token=RATES["base"].output, + cache_read_input_token_cost=RATES["base"].cache_read, + input_cost_per_token_balanced=RATES["balanced"].input, + output_cost_per_token_balanced=RATES["balanced"].output, + cache_read_input_token_cost_balanced=RATES["balanced"].cache_read, + input_cost_per_token_flex=RATES["flex"].input, + output_cost_per_token_flex=RATES["flex"].output, + cache_read_input_token_cost_flex=RATES["flex"].cache_read, + ), + ) + resources.defer(lambda: proxy.delete_model(model_id)) + return model, resources.key() + + +def _openai(sdk: SdkClients, key: str) -> OpenAI: + return sdk.openai(key).with_options(timeout=SLOW_PROVIDER_TIMEOUT_SECONDS) + + +def _assert_billed_at(tier: PricedTier, tokens: _Tokens, header_cost: str | None) -> float: + assert tokens.prompt > 0 and tokens.completion > 0, f"Sail reported no usage, so no cost is real: {tokens}" + assert header_cost is not None, "x-litellm-response-cost header missing" + costs: Final = {priced: _cost(rates, tokens) for priced, rates in RATES.items()} + assert not any(_approx_equal(costs[other], costs[tier]) for other in PRICED_TIERS if other != tier), ( + f"{BACKEND} tier rates too close together to tell {tier} apart at {tokens}: {costs}" + ) + assert _approx_equal(float(header_cost), costs[tier]), ( + f"header cost {header_cost} is not the {tier} price at {tokens}: expected {costs[tier]}, all tiers {costs}" + ) + return float(header_cost) + + +def _assert_spend_row_matches(proxy: ProxyClient, key: str, header_cost: float) -> None: + def priced(rows: list[SpendLogRow]) -> bool: + return any((row.spend or 0) > 0 for row in rows) + + rows: Final = [row for row in proxy.poll_logs_for_key(key, predicate=priced) if (row.spend or 0) > 0] + assert rows, f"no priced spend row landed for key {key}" + assert rows[0].custom_llm_provider == "sail", f"spend row misattributed: {rows[0]}" + assert rows[0].spend is not None and _approx_equal(rows[0].spend, header_cost), ( + f"logged spend {rows[0].spend} disagrees with the x-litellm-response-cost header {header_cost}" + ) + + +class TestSailChatCompletions: + @pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.cost_logged") + @pytest.mark.parametrize( + ("service_tier", "billed_tier"), [("flex", "flex"), ("balanced", "balanced"), ("auto", "base")] + ) + def test_service_tier_bills_the_matching_completion_window( + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + service_tier: str, + billed_tier: PricedTier, + ) -> None: + model, key = _register(proxy, resources) + + raw: Final = _openai(sdk, key).chat.completions.with_raw_response.create( + model=model, + messages=[{"role": "user", "content": f"{PROMPT} {unique_marker()}"}], + max_completion_tokens=MAX_TOKENS, + extra_body={**NO_PROXY_CACHE, "service_tier": service_tier}, + ) + usage: Final = raw.parse().usage + assert usage is not None, "chat response carries no usage" + details: Final = usage.prompt_tokens_details + tokens: Final = _Tokens( + prompt=usage.prompt_tokens, + cached=(details.cached_tokens or 0) if details else 0, + completion=usage.completion_tokens, + ) + + header_cost: Final = _assert_billed_at( + billed_tier, tokens, response_header(raw.headers, "x-litellm-response-cost") + ) + _assert_spend_row_matches(proxy, key, header_cost) + + @pytest.mark.covers("llm.chat_completions.sail.service_tier.nonstream.rejects_unknown_tier") + def test_unknown_service_tier_is_rejected( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register(proxy, resources) + + with pytest.raises(openai.BadRequestError) as raised: + _ = _openai(sdk, key).chat.completions.create( + model=model, + messages=[{"role": "user", "content": PROMPT}], + max_completion_tokens=MAX_TOKENS, + extra_body={**NO_PROXY_CACHE, "service_tier": "bogus"}, + ) + assert "service_tier" in raised.value.message, f"400 does not name service_tier: {raised.value.message}" + + +class TestSailResponses: + @pytest.mark.covers("llm.responses.sail.service_tier.nonstream.cost_logged") + def test_flex_completion_window_bills_flex_rates( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register(proxy, resources) + + raw: Final = _openai(sdk, key).responses.with_raw_response.create( + model=model, + input=f"{PROMPT} {unique_marker()}", + max_output_tokens=MAX_TOKENS, + metadata={"completion_window": "flex"}, + extra_body=NO_PROXY_CACHE, + ) + usage: Final = raw.parse().usage + assert usage is not None, "responses answer carries no usage" + tokens: Final = _Tokens( + prompt=usage.input_tokens, + cached=usage.input_tokens_details.cached_tokens, + completion=usage.output_tokens, + ) + + header_cost: Final = _assert_billed_at("flex", tokens, response_header(raw.headers, "x-litellm-response-cost")) + _assert_spend_row_matches(proxy, key, header_cost) + + +class TestSailMessages: + @pytest.mark.covers("llm.messages.sail.basic.nonstream.works") + def test_plain_call_returns_a_message( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model, key = _register(proxy, resources) + + message: Final = sdk.anthropic(key).messages.create( + model=model, + max_tokens=MAX_TOKENS, + messages=[{"role": "user", "content": PROMPT}], + extra_body=NO_PROXY_CACHE, + ) + assert message.role == "assistant" and message.content, f"/v1/messages returned no content: {message}" + assert message.usage.output_tokens > 0, f"/v1/messages reported no output usage: {message.usage}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 84399fd6155..6e66529ec8f 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1207,7 +1207,7 @@ class LiteLLMParamsBody(BaseModel): """POST /model/new litellm_params: `model` is the only required field; `api_key` et al may be an `os.environ/FOO` reference the proxy resolves at call time. The `*_cost_per_token` / `*_token_cost` fields register a per-deployment custom - pricing override (the cache and `_priority` rates only apply when both base + pricing override (the cache and service-tier rates only apply when both base rates are set, which is what makes the proxy register the deployment's full pricing entry); left None (and dropped from the body) the deployment keeps the backend's canonical rate.""" @@ -1243,6 +1243,12 @@ class LiteLLMParamsBody(BaseModel): cache_creation_input_token_cost: float | None = None input_cost_per_token_priority: float | None = None output_cost_per_token_priority: float | None = None + input_cost_per_token_balanced: float | None = None + output_cost_per_token_balanced: float | None = None + cache_read_input_token_cost_balanced: float | None = None + input_cost_per_token_flex: float | None = None + output_cost_per_token_flex: float | None = None + cache_read_input_token_cost_flex: float | None = None extra_headers: dict[str, str] | None = None use_in_pass_through: bool | None = None complexity_router_config: dict[str, object] | None = None diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py new file mode 100644 index 00000000000..aeee67677f3 --- /dev/null +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_utils.py @@ -0,0 +1,181 @@ +import asyncio +import uuid +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage + +TIER_MODEL: Final = "tier-priced-test-model" +TIER_ROW: Final[Mapping[str, float]] = MappingProxyType( + { + "input_cost_per_token": 4e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token_flex": 1e-06, + "output_cost_per_token_flex": 2e-06, + "cache_read_input_token_cost_flex": 2.5e-07, + "input_cost_per_token_balanced": 2e-06, + "output_cost_per_token_balanced": 4e-06, + "cache_read_input_token_cost_balanced": 5e-07, + } +) +PROMPT_TOKENS: Final = 1000 +CACHED_TOKENS: Final = 200 +COMPLETION_TOKENS: Final = 500 +TIER_API_BASE: Final = "https://tier-pricing.invalid/v1" + + +def _cost_at(prices: Mapping[str, float], column_suffix: str) -> float: + return ( + (PROMPT_TOKENS - CACHED_TOKENS) * prices[f"input_cost_per_token{column_suffix}"] + + CACHED_TOKENS * prices[f"cache_read_input_token_cost{column_suffix}"] + + COMPLETION_TOKENS * prices[f"output_cost_per_token{column_suffix}"] + ) + + +def _register_tier_model() -> None: + litellm.register_model({TIER_MODEL: {"litellm_provider": "openai", "mode": "chat", **TIER_ROW}}) + + +@pytest.mark.parametrize( + ("service_tier", "column_suffix"), + [ + pytest.param(None, "", id="no-tier-bills-base"), + pytest.param("auto", "", id="auto-bills-base"), + pytest.param("default", "", id="default-bills-base"), + pytest.param("priority", "", id="tier-without-columns-bills-base"), + pytest.param("flex", "_flex", id="flex"), + pytest.param("balanced", "_balanced", id="balanced"), + pytest.param("BALANCED", "_balanced", id="balanced-any-case"), + ], +) +def test_completion_cost_bills_the_price_columns_of_the_service_tier( + local_model_cost_map: None, service_tier: str | None, column_suffix: str +) -> None: + _register_tier_model() + response: Final = ModelResponse( + model=TIER_MODEL, + usage=Usage( + prompt_tokens=PROMPT_TOKENS, + completion_tokens=COMPLETION_TOKENS, + total_tokens=PROMPT_TOKENS + COMPLETION_TOKENS, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=CACHED_TOKENS), + ), + ) + + cost: Final = litellm.completion_cost( + completion_response=response, model=TIER_MODEL, custom_llm_provider="openai", service_tier=service_tier + ) + + assert cost == pytest.approx(_cost_at(TIER_ROW, column_suffix)) + + +class _CostRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.cost_by_model_group: Mapping[str, float] = MappingProxyType({}) + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + payload: Final = kwargs.get("standard_logging_object") + cost: Final = kwargs.get("response_cost") + if isinstance(payload, dict) and isinstance(cost, float): + self.cost_by_model_group = MappingProxyType( + {**self.cost_by_model_group, str(payload.get("model_group")): cost} + ) + + +async def _logged_cost(recorder: _CostRecorder, model_group: str) -> float: + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + assert model_group in recorder.cost_by_model_group, recorder.cost_by_model_group + return recorder.cost_by_model_group[model_group] + + +def _chat_completion_body() -> dict[str, object]: + return { + "id": "chatcmpl-tier", + "object": "chat.completion", + "created": 0, + "model": TIER_MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + "prompt_tokens_details": {"cached_tokens": CACHED_TOKENS}, + }, + } + + +DEPLOYMENT_OVERRIDE: Final = 9e-06 +PRICE_COLUMNS: Final = ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost") +PARITY_ROW: Final[Mapping[str, float]] = MappingProxyType( + { + "input_cost_per_token": 4e-06, + "output_cost_per_token": 8e-06, + "cache_read_input_token_cost": 1e-06, + **{ + f"{column}_{tier}": price + for tier in ("flex", "balanced") + for column, price in zip(PRICE_COLUMNS, (1e-06, 2e-06, 2.5e-07), strict=True) + }, + } +) + + +@pytest.mark.parametrize( + "overridden_columns", + [ + pytest.param((), id="catalog-only"), + *(pytest.param((column,), id=f"deployment-overrides-{column}") for column in PRICE_COLUMNS), + pytest.param(PRICE_COLUMNS, id="deployment-overrides-all"), + ], +) +@pytest.mark.asyncio +async def test_router_prices_balanced_columns_by_the_same_rules_as_flex( + local_model_cost_map: None, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + overridden_columns: tuple[str, ...], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _CostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + litellm.register_model({TIER_MODEL: {"litellm_provider": "openai", "mode": "chat", **PARITY_ROW}}) + respx_mock.post(f"{TIER_API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=_chat_completion_body()) + ) + group: Final = {tier: f"{tier}-{uuid.uuid4().hex}" for tier in ("flex", "balanced")} + router: Final = litellm.Router( + model_list=[ + { + "model_name": group[tier], + "litellm_params": { + "model": f"openai/{TIER_MODEL}", + "api_key": "sk-test", + "api_base": TIER_API_BASE, + **{f"{column}_{tier}": DEPLOYMENT_OVERRIDE for column in overridden_columns}, + }, + } + for tier in ("flex", "balanced") + ] + ) + + for tier in ("flex", "balanced"): + await router.acompletion(model=group[tier], messages=[{"role": "user", "content": "hi"}], service_tier=tier) + flex_cost: Final = await _logged_cost(recorder, group["flex"]) + balanced_cost: Final = await _logged_cost(recorder, group["balanced"]) + + assert balanced_cost == pytest.approx(flex_cost) + if not overridden_columns: + assert balanced_cost == pytest.approx(_cost_at(PARITY_ROW, "_balanced")) diff --git a/tests/unit/llms/base_llm/chat/__init__.py b/tests/unit/llms/base_llm/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/base_llm/chat/test_transformation.py b/tests/unit/llms/base_llm/chat/test_transformation.py new file mode 100644 index 00000000000..5af54390e27 --- /dev/null +++ b/tests/unit/llms/base_llm/chat/test_transformation.py @@ -0,0 +1,32 @@ +import json + +import httpx +import pytest +import respx + +import litellm + + +def test_base_http_handler_sends_a_caller_extra_body_over_the_request_unchanged( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", "True") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post(url__regex=r"https://api\.deepseek\.com/.*chat/completions").mock( + return_value=httpx.Response( + 200, json={"id": "c", "object": "chat.completion", "created": 0, "model": "m", "choices": []} + ) + ) + + litellm.completion( + model="deepseek/deepseek-chat", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-test", + temperature=0.5, + extra_body={"foo": 1, "temperature": 0.9, "metadata": {"b": "2"}}, + ) + + body = json.loads(route.calls.last.request.content) + assert body["foo"] == 1 + assert body["temperature"] == 0.9 + assert body["metadata"] == {"b": "2"} diff --git a/tests/unit/llms/base_llm/responses/test_transformation.py b/tests/unit/llms/base_llm/responses/test_transformation.py index c6142685661..82e979e7777 100644 --- a/tests/unit/llms/base_llm/responses/test_transformation.py +++ b/tests/unit/llms/base_llm/responses/test_transformation.py @@ -1,7 +1,11 @@ """The shared Responses API config contract.""" +import json + +import httpx import pytest +import litellm from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.router import GenericLiteLLMParams @@ -33,3 +37,32 @@ async def test_default_async_transform_delegates_to_the_sync_transform(): ) assert async_body == sync_body assert "cache_control" not in async_body["input"][0]["content"][0] + + +def test_responses_sends_a_caller_extra_body_over_the_request_unchanged(respx_mock, monkeypatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={ + "id": "resp", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "m", + "output": [], + }, + ) + ) + + litellm.responses( + model="openai/gpt-5", + input="hi", + api_key="sk-test", + metadata={"a": "1"}, + extra_body={"foo": 1, "metadata": {"b": "2"}}, + ) + + body = json.loads(route.calls.last.request.content) + assert body["foo"] == 1 + assert body["metadata"] == {"b": "2"} diff --git a/tests/unit/llms/sail/__init__.py b/tests/unit/llms/sail/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/sail/chat/__init__.py b/tests/unit/llms/sail/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py new file mode 100644 index 00000000000..a42fb1074a0 --- /dev/null +++ b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py @@ -0,0 +1,353 @@ +import re +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from tests.unit.llms.sail.helpers import ( + MODEL, + SAIL_API_BASE, + SpendCapture, + chat_completion_stream, + cost_at, + sent_body, +) + +MESSAGES: Final = [{"role": "user", "content": "hi"}] +TIER_CASES: Final = [ + pytest.param(None, None, "", id="no-tier"), + pytest.param("auto", None, "", id="auto"), + pytest.param("default", "asap", "", id="default"), + pytest.param("priority", "asap", "", id="priority"), + pytest.param("flex", "flex", "_flex", id="flex"), + pytest.param("balanced", "balanced", "_balanced", id="balanced"), + pytest.param("FLEX", "flex", "_flex", id="flex-any-case"), +] + + +def _window(body: dict[str, object]) -> object: + metadata: Final = body.get("metadata") + return metadata.get("completion_window") if isinstance(metadata, dict) else None + + +@pytest.mark.parametrize(("service_tier", "window", "column_suffix"), TIER_CASES) +@pytest.mark.asyncio +async def test_sail_chat_sends_the_tier_window_and_bills_its_price_columns( + sail_env: None, + chat_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + window: str | None, + column_suffix: str, +) -> None: + await litellm.acompletion( + model=MODEL, messages=MESSAGES, service_tier=service_tier, litellm_call_id=spend_capture.call_id + ) + + body: Final = sent_body(chat_route) + assert "service_tier" not in body + assert _window(body) == window + assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) + + +@pytest.mark.parametrize(("service_tier", "window", "column_suffix"), TIER_CASES) +@pytest.mark.asyncio +async def test_sail_chat_stream_sends_the_tier_window_and_bills_its_price_columns( + sail_env: None, + respx_mock: respx.MockRouter, + spend_capture: SpendCapture, + service_tier: str | None, + window: str | None, + column_suffix: str, +) -> None: + route: Final = respx_mock.post(f"{SAIL_API_BASE}/chat/completions").mock( + return_value=httpx.Response( + 200, content=chat_completion_stream(), headers={"content-type": "text/event-stream"} + ) + ) + + stream: Final = await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier=service_tier, + stream=True, + stream_options={"include_usage": True}, + litellm_call_id=spend_capture.call_id, + ) + async for _ in stream: + pass + + body: Final = sent_body(route) + assert "service_tier" not in body + assert _window(body) == window + assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) + + +@pytest.mark.parametrize( + ("service_tier", "window"), [pytest.param(*case.values[:2], id=case.id) for case in TIER_CASES] +) +def test_sail_sync_chat_sends_the_tier_window( + sail_env: None, chat_route: respx.Route, service_tier: str | None, window: str | None +) -> None: + litellm.completion(model=MODEL, messages=MESSAGES, service_tier=service_tier) + + body: Final = sent_body(chat_route) + assert "service_tier" not in body + assert _window(body) == window + + +@pytest.mark.parametrize("service_tier", ["scale", "standard", "asap", 5, ["flex"]]) +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_tier_with_no_window_before_sending( + sail_env: None, chat_route: respx.Route, service_tier: object +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape(f"service_tier={service_tier!r}")) as error: + await litellm.acompletion(model=MODEL, messages=MESSAGES, service_tier=service_tier) + + assert error.value.status_code == 400 + assert not chat_route.called + + +@pytest.mark.parametrize("service_tier", ["scale", 5]) +@pytest.mark.asyncio +async def test_sail_chat_drops_an_unknown_tier_under_drop_params_and_bills_asap( + sail_env: None, chat_route: respx.Route, spend_capture: SpendCapture, service_tier: object +) -> None: + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier=service_tier, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(chat_route) + assert "service_tier" not in body + assert "metadata" not in body + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + +@pytest.fixture(params=["openai-sdk", "base-http-handler"]) +def chat_http_path(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER", str(request.param == "base-http-handler")) + + +@pytest.mark.parametrize( + ("service_tier", "wire_metadata", "column_suffix"), + [ + pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"), + pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), + ], +) +@pytest.mark.asyncio +async def test_sail_chat_merges_caller_extra_body_metadata_with_the_tier_window( + sail_env: None, + chat_http_path: None, + chat_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + wire_metadata: dict[str, str], + column_suffix: str, +) -> None: + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier=service_tier, + extra_body={"metadata": {"trace_id": "t-1"}, "foo": 1}, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(chat_route) + assert body["metadata"] == wire_metadata + assert body["foo"] == 1 + assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) + + +@pytest.mark.parametrize( + ("extra_body", "message"), + [ + pytest.param( + {"metadata": {"completion_window": "flex"}}, + "extra_body.metadata.completion_window", + id="extra-body-window", + ), + pytest.param({"service_tier": "flex"}, "service_tier inside extra_body", id="extra-body-tier"), + ], +) +@pytest.mark.parametrize("service_tier", [None, "balanced"]) +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_window_billing_cannot_see_before_sending( + sail_env: None, + chat_http_path: None, + chat_route: respx.Route, + service_tier: str | None, + extra_body: dict[str, object], + message: str, +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match=message) as error: + await litellm.acompletion(model=MODEL, messages=MESSAGES, service_tier=service_tier, extra_body=extra_body) + + assert error.value.status_code == 400 + assert not chat_route.called + + +@pytest.mark.parametrize( + ("service_tier", "wire_metadata", "column_suffix"), + [ + pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"), + pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), + ], +) +@pytest.mark.asyncio +async def test_sail_chat_drops_a_window_billing_cannot_see_under_drop_params( + sail_env: None, + chat_http_path: None, + chat_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + wire_metadata: dict[str, str], + column_suffix: str, +) -> None: + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier=service_tier, + extra_body={"service_tier": "flex", "metadata": {"trace_id": "t-1", "completion_window": "flex"}}, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(chat_route) + assert "service_tier" not in body + assert body["metadata"] == wire_metadata + assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) + + +@pytest.mark.asyncio +async def test_sail_chat_drops_a_lone_caller_window_under_drop_params_and_bills_asap( + sail_env: None, chat_http_path: None, chat_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + extra_body={"metadata": {"completion_window": "flex"}}, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + assert "completion_window" not in (sent_body(chat_route).get("metadata") or {}) + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + +@pytest.mark.asyncio +async def test_sail_chat_passes_a_non_mapping_extra_body_metadata_through_untouched( + sail_env: None, chat_http_path: None, chat_route: respx.Route +) -> None: + await litellm.acompletion(model=MODEL, messages=MESSAGES, extra_body={"metadata": None, "foo": 1}) + + body: Final = sent_body(chat_route) + assert "metadata" in body + assert body["metadata"] is None + assert body["foo"] == 1 + + +def test_sail_sync_chat_rejects_an_unknown_tier_as_unsupported_params(sail_env: None, chat_route: respx.Route) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="service_tier='scale'"): + litellm.completion(model=MODEL, messages=MESSAGES, service_tier="scale") + + assert not chat_route.called + + +@pytest.mark.asyncio +async def test_sail_chat_keeps_the_window_when_preview_features_forward_caller_metadata( + sail_env: None, + chat_http_path: None, + chat_route: respx.Route, + spend_capture: SpendCapture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "enable_preview_features", True) + + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier="flex", + metadata={"requester_metadata": {"trace_id": "t-1"}}, + litellm_call_id=spend_capture.call_id, + ) + + assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "flex"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_flex")) + + +@pytest.mark.parametrize( + "rejected", + [ + pytest.param({"stop": ["x"]}, id="stop"), + pytest.param({"seed": 1}, id="seed"), + pytest.param({"frequency_penalty": 0.5}, id="frequency_penalty"), + pytest.param({"presence_penalty": 0.5}, id="presence_penalty"), + pytest.param({"logit_bias": {"1": 1}}, id="logit_bias"), + pytest.param({"logprobs": True}, id="logprobs"), + pytest.param({"top_logprobs": 2}, id="top_logprobs"), + ], +) +def test_sail_chat_rejects_params_sail_rejects_unless_dropped( + sail_env: None, chat_route: respx.Route, rejected: dict[str, object] +) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + litellm.completion(model=MODEL, messages=MESSAGES, **rejected) + assert not chat_route.called + + litellm.completion(model=MODEL, messages=MESSAGES, drop_params=True, **rejected) + assert set(rejected).isdisjoint(sent_body(chat_route)) + + +def test_sail_chat_forwards_params_sail_accepts(sail_env: None, chat_route: respx.Route) -> None: + tools: Final = [{"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}] + + litellm.completion( + model=MODEL, + messages=MESSAGES, + max_tokens=64, + tools=tools, + tool_choice="auto", + response_format={"type": "json_object"}, + reasoning_effort="low", + user="user-1", + ) + + body: Final = sent_body(chat_route) + assert body["max_tokens"] == 64 + assert body["tools"] == tools + assert body["tool_choice"] == "auto" + assert body["response_format"] == {"type": "json_object"} + assert body["reasoning_effort"] == "low" + assert body["user"] == "user-1" + + +def test_sail_chat_passes_max_tokens_and_max_completion_tokens_through_as_sent( + sail_env: None, chat_route: respx.Route +) -> None: + litellm.completion(model=MODEL, messages=MESSAGES, max_tokens=64, max_completion_tokens=32) + + body: Final = sent_body(chat_route) + assert body["max_tokens"] == 64 + assert body["max_completion_tokens"] == 32 + + +def test_sail_chat_uses_sail_api_base_env_and_key( + sail_env: None, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SAIL_API_BASE", "https://sail-gateway.invalid/v1") + route: Final = respx_mock.post("https://sail-gateway.invalid/v1/chat/completions").mock( + return_value=httpx.Response( + 200, json={"id": "c", "object": "chat.completion", "created": 0, "model": "m", "choices": []} + ) + ) + + litellm.completion(model=MODEL, messages=MESSAGES) + + assert route.calls.last.request.headers["Authorization"] == "Bearer sail-test-key" diff --git a/tests/unit/llms/sail/conftest.py b/tests/unit/llms/sail/conftest.py new file mode 100644 index 00000000000..2b2a6e5cae0 --- /dev/null +++ b/tests/unit/llms/sail/conftest.py @@ -0,0 +1,41 @@ +import uuid +from collections.abc import Iterator +from typing import Final + +import httpx +import pytest +import pytest_asyncio +import respx + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from tests.unit.llms.sail.helpers import SAIL_API_BASE, SpendCapture, chat_completion_body + + +@pytest.fixture +def sail_env(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("SAIL_API_KEY", "sail-test-key") + monkeypatch.delenv("SAIL_API_BASE", raising=False) + monkeypatch.setattr( + litellm, + "disable_aiohttp_transport", + True, + ) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest_asyncio.fixture +async def spend_capture(monkeypatch: pytest.MonkeyPatch) -> SpendCapture: + GLOBAL_LOGGING_WORKER.start() + capture: Final = SpendCapture(call_id=f"sail-{uuid.uuid4()}") + monkeypatch.setattr(litellm, "callbacks", [capture]) + return capture + + +@pytest.fixture +def chat_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{SAIL_API_BASE}/chat/completions").mock( + return_value=httpx.Response(200, json=chat_completion_body()) + ) diff --git a/tests/unit/llms/sail/helpers.py b/tests/unit/llms/sail/helpers.py new file mode 100644 index 00000000000..2684310d94a --- /dev/null +++ b/tests/unit/llms/sail/helpers.py @@ -0,0 +1,119 @@ +import asyncio +import json +from collections.abc import Mapping +from typing import Final + +import respx + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + +SAIL_API_BASE: Final = "https://api.sailresearch.com/v1" +MODEL: Final = "sail/zai-org/GLM-5.3" +PROMPT_TOKENS: Final = 1000 +CACHED_TOKENS: Final = 200 +COMPLETION_TOKENS: Final = 500 + + +def cost_at(column_suffix: str) -> float: + prices: Final[Mapping[str, object]] = litellm.model_cost[MODEL] + return ( + (PROMPT_TOKENS - CACHED_TOKENS) * float(prices[f"input_cost_per_token{column_suffix}"]) + + CACHED_TOKENS * float(prices[f"cache_read_input_token_cost{column_suffix}"]) + + COMPLETION_TOKENS * float(prices[f"output_cost_per_token{column_suffix}"]) + ) + + +def sent_body(route: respx.Route) -> dict[str, object]: + return json.loads(route.calls.last.request.content) + + +def chat_completion_body() -> dict[str, object]: + return { + "id": "chatcmpl-sail", + "object": "chat.completion", + "created": 0, + "model": "zai-org/GLM-5.3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + "prompt_tokens_details": {"cached_tokens": CACHED_TOKENS}, + }, + } + + +def chat_completion_stream() -> bytes: + chunk: Final = {"id": "chatcmpl-sail", "object": "chat.completion.chunk", "created": 0, "model": "zai-org/GLM-5.3"} + events: Final = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**chunk, "choices": [], "usage": chat_completion_body()["usage"]}, + ) + return "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + b"data: [DONE]\n\n" + + +def responses_body() -> dict[str, object]: + return { + "id": "resp_sail", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "zai-org/GLM-5.3", + "output": [ + { + "type": "message", + "id": "msg_sail", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": { + "input_tokens": PROMPT_TOKENS, + "input_tokens_details": {"cached_tokens": CACHED_TOKENS}, + "output_tokens": COMPLETION_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + }, + } + + +def messages_body() -> dict[str, object]: + return { + "id": "msg_sail", + "type": "message", + "role": "assistant", + "model": "zai-org/GLM-5.3", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": PROMPT_TOKENS - CACHED_TOKENS, + "cache_read_input_tokens": CACHED_TOKENS, + "output_tokens": COMPLETION_TOKENS, + }, + } + + +class SpendCapture(CustomLogger): + """Records the cost the spend logs would store for one call, matched by its call id.""" + + def __init__(self, call_id: str) -> None: + super().__init__() + self.call_id = call_id + self.costs: tuple[object, ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + if kwargs.get("litellm_call_id") == self.call_id: + payload: Final = kwargs.get("standard_logging_object") + self.costs = (*self.costs, payload.get("response_cost") if isinstance(payload, dict) else None) + + async def settled_cost(self) -> object: + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + assert len(self.costs) == 1, self.costs + return self.costs[0] diff --git a/tests/unit/llms/sail/messages/__init__.py b/tests/unit/llms/sail/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py new file mode 100644 index 00000000000..c6e74534791 --- /dev/null +++ b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py @@ -0,0 +1,31 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, messages_body, sent_body + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +@pytest.fixture +def messages_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=httpx.Response(200, json=messages_body())) + + +@pytest.mark.parametrize("service_tier", [None, "auto", "priority", "flex", "balanced", "scale"]) +@pytest.mark.asyncio +async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier( + sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture, service_tier: str | None +) -> None: + await litellm.anthropic_messages( + model=MODEL, messages=MESSAGES, max_tokens=16, service_tier=service_tier, litellm_call_id=spend_capture.call_id + ) + + body: Final = sent_body(messages_route) + assert body["messages"] == MESSAGES + assert "service_tier" not in body + assert "completion_window" not in (body.get("metadata") or {}) + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) diff --git a/tests/unit/llms/sail/responses/__init__.py b/tests/unit/llms/sail/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/sail/responses/test_sail_responses_transformation.py b/tests/unit/llms/sail/responses/test_sail_responses_transformation.py new file mode 100644 index 00000000000..3384b79bdec --- /dev/null +++ b/tests/unit/llms/sail/responses/test_sail_responses_transformation.py @@ -0,0 +1,217 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, responses_body, sent_body + +INPUT: Final = "hi" + + +@pytest.fixture +def responses_route(respx_mock: respx.MockRouter) -> respx.Route: + return respx_mock.post(f"{SAIL_API_BASE}/responses").mock(return_value=httpx.Response(200, json=responses_body())) + + +@pytest.mark.parametrize( + ("service_tier", "metadata", "wire_metadata", "column_suffix"), + [ + pytest.param(None, None, None, "", id="no-tier"), + pytest.param("auto", None, None, "", id="auto"), + pytest.param("default", None, {"completion_window": "asap"}, "", id="default"), + pytest.param("priority", None, {"completion_window": "asap"}, "", id="priority"), + pytest.param("flex", None, {"completion_window": "flex"}, "_flex", id="flex"), + pytest.param("balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced"), + pytest.param("Balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"), + pytest.param( + "flex", {"user_tag": "a"}, {"user_tag": "a", "completion_window": "flex"}, "_flex", id="tier-keeps-metadata" + ), + pytest.param(None, {"completion_window": "flex"}, {"completion_window": "flex"}, "_flex", id="caller-window"), + pytest.param( + None, + {"completion_window": "standard"}, + {"completion_window": "standard"}, + "_balanced", + id="standard-window", + ), + pytest.param(None, {"completion_window": "FLEX"}, {"completion_window": "flex"}, "_flex", id="window-any-case"), + pytest.param( + "priority", {"completion_window": "asap"}, {"completion_window": "asap"}, "", id="agreeing-tier-and-window" + ), + pytest.param(None, {"user_tag": "a"}, {"user_tag": "a"}, "", id="metadata-without-window"), + ], +) +@pytest.mark.asyncio +async def test_sail_responses_send_the_window_and_bill_its_price_columns( + sail_env: None, + responses_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + metadata: dict[str, str] | None, + wire_metadata: dict[str, str] | None, + column_suffix: str, +) -> None: + await litellm.aresponses( + model=MODEL, + input=INPUT, + service_tier=service_tier, + metadata=metadata, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(responses_route) + assert "service_tier" not in body + assert body.get("metadata") == wire_metadata + assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) + + +@pytest.mark.parametrize( + ("service_tier", "metadata", "message"), + [ + pytest.param("scale", None, "service_tier='scale'", id="unknown-tier"), + pytest.param(5, None, "service_tier=5", id="non-string-tier"), + pytest.param(None, {"completion_window": "soon"}, "completion_window='soon'", id="unknown-window"), + pytest.param("flex", {"completion_window": "asap"}, "select different completion windows", id="conflict"), + ], +) +@pytest.mark.asyncio +async def test_sail_responses_reject_before_sending( + sail_env: None, + responses_route: respx.Route, + service_tier: object, + metadata: dict[str, str] | None, + message: str, +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match=message): + await litellm.aresponses(model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata) + + assert not responses_route.called + + +@pytest.mark.asyncio +async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params( + sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.aresponses( + model=MODEL, + input=INPUT, + service_tier="scale", + metadata={"completion_window": "soon", "user_tag": "a"}, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(responses_route) + assert "service_tier" not in body + assert body["metadata"] == {"user_tag": "a"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + +@pytest.mark.parametrize( + ("service_tier", "wire_metadata", "column_suffix"), + [ + pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"), + pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), + ], +) +@pytest.mark.asyncio +async def test_sail_responses_merge_caller_extra_body_metadata_with_the_tier_window( + sail_env: None, + responses_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + wire_metadata: dict[str, str], + column_suffix: str, +) -> None: + await litellm.aresponses( + model=MODEL, + input=INPUT, + service_tier=service_tier, + extra_body={"metadata": {"trace_id": "t-1"}, "foo": 1}, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(responses_route) + assert body["metadata"] == wire_metadata + assert body["foo"] == 1 + assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) + + +@pytest.mark.parametrize( + ("extra_body", "message"), + [ + pytest.param( + {"metadata": {"completion_window": "flex"}}, + "extra_body.metadata.completion_window", + id="extra-body-window", + ), + pytest.param({"service_tier": "flex"}, "service_tier inside extra_body", id="extra-body-tier"), + ], +) +@pytest.mark.asyncio +async def test_sail_responses_reject_a_window_billing_cannot_see_before_sending( + sail_env: None, responses_route: respx.Route, extra_body: dict[str, object], message: str +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match=message): + await litellm.aresponses(model=MODEL, input=INPUT, extra_body=extra_body) + + assert not responses_route.called + + +def test_sail_sync_responses_drop_a_window_billing_cannot_see_under_drop_params( + sail_env: None, responses_route: respx.Route +) -> None: + litellm.responses( + model=MODEL, + input=INPUT, + service_tier="balanced", + extra_body={"service_tier": "flex", "metadata": {"trace_id": "t-1", "completion_window": "flex"}}, + drop_params=True, + ) + + body: Final = sent_body(responses_route) + assert "service_tier" not in body + assert body["metadata"] == {"trace_id": "t-1", "completion_window": "balanced"} + + +@pytest.mark.asyncio +async def test_sail_responses_drop_a_lone_caller_window_under_drop_params_and_bill_asap( + sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.aresponses( + model=MODEL, + input=INPUT, + extra_body={"metadata": {"completion_window": "flex"}}, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + assert "completion_window" not in (sent_body(responses_route).get("metadata") or {}) + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + +def test_sail_responses_pass_a_non_mapping_extra_body_metadata_through_untouched( + sail_env: None, responses_route: respx.Route +) -> None: + litellm.responses(model=MODEL, input=INPUT, extra_body={"metadata": None, "foo": 1}) + + body: Final = sent_body(responses_route) + assert "metadata" in body + assert body["metadata"] is None + assert body["foo"] == 1 + + +@pytest.mark.asyncio +async def test_sail_responses_use_sail_api_base_env_and_key( + sail_env: None, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SAIL_API_BASE", "https://sail-gateway.invalid/v1") + route: Final = respx_mock.post("https://sail-gateway.invalid/v1/responses").mock( + return_value=httpx.Response(200, json=responses_body()) + ) + + await litellm.aresponses(model=MODEL, input=INPUT) + + assert route.calls.last.request.headers["Authorization"] == "Bearer sail-test-key" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 5b327305e31..2c612aa350c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -805,10 +805,12 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_token_above_512k_tokens": {"type": "number"}, "cache_read_input_token_cost_flex": {"type": "number"}, "cache_read_input_token_cost_priority": {"type": "number"}, + "cache_read_input_token_cost_balanced": {"type": "number"}, "cache_read_input_token_cost_above_200k_tokens_priority": {"type": "number"}, "cache_read_input_token_cost_above_272k_tokens_priority": {"type": "number"}, "input_cost_per_token_flex": {"type": "number"}, "input_cost_per_token_priority": {"type": "number"}, + "input_cost_per_token_balanced": {"type": "number"}, "input_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "input_cost_per_token_above_272k_tokens_priority": {"type": "number"}, "input_cost_per_token_above_272k_tokens_batches": {"type": "number"}, @@ -816,6 +818,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "input_cost_per_audio_token_priority": {"type": "number"}, "output_cost_per_token_flex": {"type": "number"}, "output_cost_per_token_priority": {"type": "number"}, + "output_cost_per_token_balanced": {"type": "number"}, "output_cost_per_token_above_200k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_priority": {"type": "number"}, "output_cost_per_token_above_272k_tokens_batches": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index d6b8d00a023..29ce2d4865a 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -194,6 +194,7 @@ describe("provider_info_helpers", () => { Providers.PETALS, Providers.PG_VECTOR, Providers.PREDIBASE, + Providers.Sail, Providers.WANDB, Providers.ZAI, ]; @@ -403,6 +404,14 @@ describe("provider_info_helpers", () => { expect(result).not.toContain("anthropic-native"); }); + it("should list sail models when called with the 'Sail' provider key", () => { + const modelMap = { + "sail/openai/gpt-oss-120b": { litellm_provider: "sail" }, + "sagemaker-model": { litellm_provider: "sagemaker" }, + }; + expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); + }); + it("should include bedrock converse but exclude standalone bedrock_mantle when called with 'Bedrock' provider key", () => { const modelMap = { "bedrock-base": { litellm_provider: "bedrock" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index eab55503da8..b0e33338bab 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -160,6 +160,7 @@ export enum Providers { REPLICATE = "Replicate", RunwayML = "RunwayML", SAGEMAKER_LEGACY = "Sagemaker", + Sail = "Sail", Sambanova = "Sambanova", SAP = "SAP Generative AI Hub", SCX_AI = "SCX.ai", @@ -278,6 +279,7 @@ export const provider_map: Record = { RunwayML: "runwayml", SAGEMAKER_LEGACY: "sagemaker", SageMaker: "sagemaker_chat", + Sail: "sail", Sambanova: "sambanova", SAP: "sap", SCX_AI: "scx-ai", @@ -448,6 +450,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Oracle]: "oci/xai.grok-4", [Providers.RunwayML]: "runwayml/gen4_turbo", [Providers.SageMaker]: "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b", + [Providers.Sail]: "sail/openai/gpt-oss-120b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Snowflake]: "snowflake/mistral-7b", [Providers.Vertex_AI]: "gemini-pro", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 513cad9a714..c77f7d84bd8 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -33002,6 +33002,8 @@ export interface components { cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Balanced */ + cache_read_input_token_cost_balanced?: number | null; /** Cache Read Input Token Cost Batches */ cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ @@ -33082,6 +33084,8 @@ export interface components { input_cost_per_token_above_272k_tokens_priority?: number | null; /** Input Cost Per Token Above 512K Tokens */ input_cost_per_token_above_512k_tokens?: number | null; + /** Input Cost Per Token Balanced */ + input_cost_per_token_balanced?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -33207,6 +33211,8 @@ export interface components { output_cost_per_token_above_272k_tokens_priority?: number | null; /** Output Cost Per Token Above 512K Tokens */ output_cost_per_token_above_512k_tokens?: number | null; + /** Output Cost Per Token Balanced */ + output_cost_per_token_balanced?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */ @@ -46797,6 +46803,8 @@ export interface components { cache_read_input_token_cost_above_272k_tokens_priority?: number | null; /** Cache Read Input Token Cost Above 512K Tokens */ cache_read_input_token_cost_above_512k_tokens?: number | null; + /** Cache Read Input Token Cost Balanced */ + cache_read_input_token_cost_balanced?: number | null; /** Cache Read Input Token Cost Batches */ cache_read_input_token_cost_batches?: number | null; /** Cache Read Input Token Cost Flex */ @@ -46877,6 +46885,8 @@ export interface components { input_cost_per_token_above_272k_tokens_priority?: number | null; /** Input Cost Per Token Above 512K Tokens */ input_cost_per_token_above_512k_tokens?: number | null; + /** Input Cost Per Token Balanced */ + input_cost_per_token_balanced?: number | null; /** Input Cost Per Token Batches */ input_cost_per_token_batches?: number | null; /** Input Cost Per Token Cache Hit */ @@ -47002,6 +47012,8 @@ export interface components { output_cost_per_token_above_272k_tokens_priority?: number | null; /** Output Cost Per Token Above 512K Tokens */ output_cost_per_token_above_512k_tokens?: number | null; + /** Output Cost Per Token Balanced */ + output_cost_per_token_balanced?: number | null; /** Output Cost Per Token Batches */ output_cost_per_token_batches?: number | null; /** Output Cost Per Token Flex */