mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
3a6744cd02
Register Sail (providers.json, LlmProviders.SAIL, OpenAI-compatible lists, ProviderConfigManager) for chat, Responses and /v1/messages, and add its 12 models to both cost maps with asap, balanced and flex price columns. Sail picks speed and price with metadata.completion_window and rejects service_tier, so the Sail chat and Responses configs translate the tier: default and priority to asap, flex to flex, balanced to balanced, auto to no window. Billing prices the window that was sent. A tier Sail has no window for, or a window or tier set where billing cannot see it (request metadata, extra_body), is a 400 unless drop_params is set. Add balanced to ServiceTier and its _balanced price columns to the model info types, the Rust catalog and the dashboard schema. A transform_extra_body hook on the chat and Responses base configs, which returns extra_body unchanged by default, lets Sail keep the window when a caller also sends extra_body.metadata. Sail is listed in the Add Model form and model picker. Co-authored-by: shrey kharbanda <shrey@berri.ai>
119 lines
4.2 KiB
Python
119 lines
4.2 KiB
Python
import asyncio
|
|
import json
|
|
from collections.abc import Mapping
|
|
from typing import Final
|
|
|
|
import respx
|
|
|
|
import litellm
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
|
|
|
SAIL_API_BASE: Final = "https://api.sailresearch.com/v1"
|
|
MODEL: Final = "sail/zai-org/GLM-5.3"
|
|
PROMPT_TOKENS: Final = 1000
|
|
CACHED_TOKENS: Final = 200
|
|
COMPLETION_TOKENS: Final = 500
|
|
|
|
|
|
def cost_at(column_suffix: str) -> float:
|
|
prices: Final[Mapping[str, object]] = litellm.model_cost[MODEL]
|
|
return (
|
|
(PROMPT_TOKENS - CACHED_TOKENS) * float(prices[f"input_cost_per_token{column_suffix}"])
|
|
+ CACHED_TOKENS * float(prices[f"cache_read_input_token_cost{column_suffix}"])
|
|
+ COMPLETION_TOKENS * float(prices[f"output_cost_per_token{column_suffix}"])
|
|
)
|
|
|
|
|
|
def sent_body(route: respx.Route) -> dict[str, object]:
|
|
return json.loads(route.calls.last.request.content)
|
|
|
|
|
|
def chat_completion_body() -> dict[str, object]:
|
|
return {
|
|
"id": "chatcmpl-sail",
|
|
"object": "chat.completion",
|
|
"created": 0,
|
|
"model": "zai-org/GLM-5.3",
|
|
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
|
"usage": {
|
|
"prompt_tokens": PROMPT_TOKENS,
|
|
"completion_tokens": COMPLETION_TOKENS,
|
|
"total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
|
|
"prompt_tokens_details": {"cached_tokens": CACHED_TOKENS},
|
|
},
|
|
}
|
|
|
|
|
|
def chat_completion_stream() -> bytes:
|
|
chunk: Final = {"id": "chatcmpl-sail", "object": "chat.completion.chunk", "created": 0, "model": "zai-org/GLM-5.3"}
|
|
events: Final = (
|
|
{**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}]},
|
|
{**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]},
|
|
{**chunk, "choices": [], "usage": chat_completion_body()["usage"]},
|
|
)
|
|
return "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + b"data: [DONE]\n\n"
|
|
|
|
|
|
def responses_body() -> dict[str, object]:
|
|
return {
|
|
"id": "resp_sail",
|
|
"object": "response",
|
|
"created_at": 0,
|
|
"status": "completed",
|
|
"model": "zai-org/GLM-5.3",
|
|
"output": [
|
|
{
|
|
"type": "message",
|
|
"id": "msg_sail",
|
|
"status": "completed",
|
|
"role": "assistant",
|
|
"content": [{"type": "output_text", "text": "ok", "annotations": []}],
|
|
}
|
|
],
|
|
"usage": {
|
|
"input_tokens": PROMPT_TOKENS,
|
|
"input_tokens_details": {"cached_tokens": CACHED_TOKENS},
|
|
"output_tokens": COMPLETION_TOKENS,
|
|
"output_tokens_details": {"reasoning_tokens": 0},
|
|
"total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
|
|
},
|
|
}
|
|
|
|
|
|
def messages_body() -> dict[str, object]:
|
|
return {
|
|
"id": "msg_sail",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"model": "zai-org/GLM-5.3",
|
|
"content": [{"type": "text", "text": "ok"}],
|
|
"stop_reason": "end_turn",
|
|
"usage": {
|
|
"input_tokens": PROMPT_TOKENS - CACHED_TOKENS,
|
|
"cache_read_input_tokens": CACHED_TOKENS,
|
|
"output_tokens": COMPLETION_TOKENS,
|
|
},
|
|
}
|
|
|
|
|
|
class SpendCapture(CustomLogger):
|
|
"""Records the cost the spend logs would store for one call, matched by its call id."""
|
|
|
|
def __init__(self, call_id: str) -> None:
|
|
super().__init__()
|
|
self.call_id = call_id
|
|
self.costs: tuple[object, ...] = ()
|
|
|
|
async def async_log_success_event(
|
|
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
|
) -> None:
|
|
if kwargs.get("litellm_call_id") == self.call_id:
|
|
payload: Final = kwargs.get("standard_logging_object")
|
|
self.costs = (*self.costs, payload.get("response_cost") if isinstance(payload, dict) else None)
|
|
|
|
async def settled_cost(self) -> object:
|
|
await asyncio.sleep(0)
|
|
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0)
|
|
assert len(self.costs) == 1, self.costs
|
|
return self.costs[0]
|