This commit is contained in:
Kavin Anand 2026-10-29 18:34:54 +00:00 • committed by GitHub
commit ac88eed297
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 510 additions and 66 deletions

View file

@ -1647,8 +1647,17 @@ def completion_cost(
if explicit_tier is not None:
service_tier = explicit_tier
else:
from litellm.llms.sail.common_utils import billed_service_tier
requested_tier: Final[object] = (
billed_service_tier(optional_params)
if custom_llm_provider == "sail" and optional_params is not None
else optional_params.get("service_tier")
if optional_params is not None
else None
)
service_tier = _resolve_billable_service_tier( # rebind-ok: resolved from request then response
requested=optional_params.get("service_tier") if optional_params is not None else None,
requested=requested_tier,
served=_served_service_tier(completion_response, cost_per_token_usage_object),
)

View file

@ -1207,6 +1207,7 @@ class ChunkProcessor:
for field, value in (
("inference_geo", calculated_usage_per_chunk["inference_geo"]),
("speed", calculated_usage_per_chunk["speed"]),
("service_tier", self._last_provider_pricing_field(chunks, "service_tier")),
)
if value is not None
}

View file

@ -410,7 +410,7 @@ DROP_UNSUPPORTED_SERVICE_TIER_WARNING: Final = (
"Anthropic only accepts 'auto' and 'standard_only'."
)
MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle"}
MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle", "sail"}
DROP_SERVICE_TIER_PROVIDER_WARNING: Final = (
"Dropping `service_tier` for model=%s on provider=%s. Only the Anthropic API routes accept it."

View file

@ -600,9 +600,8 @@ def anthropic_messages_handler(
presanitized=bool(kwargs.pop("_litellm_messages_presanitized", False)),
)
metadata = validate_anthropic_api_metadata(metadata)
local_vars: Final = locals()
anthropic_metadata: Final = validate_anthropic_api_metadata(metadata)
is_async: Final = kwargs.pop("is_async", False)
# Use provided client or create a new one
litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj")
@ -667,7 +666,7 @@ def anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=model,
metadata=metadata,
metadata=anthropic_metadata,
stop_sequences=stop_sequences,
stream=stream,
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
@ -699,7 +698,7 @@ def anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=original_model,
metadata=metadata,
metadata=anthropic_metadata,
stop_sequences=stop_sequences,
stream=stream,
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
@ -729,7 +728,7 @@ def anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,
model=original_model,
metadata=metadata,
metadata=anthropic_metadata,
stop_sequences=stop_sequences,
stream=stream,
system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower
@ -760,10 +759,16 @@ def anthropic_messages_handler(
)
if marked_tools is not None:
AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1)
local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})})
request_metadata: Final = anthropic_messages_provider_config.request_metadata(metadata)
request_params: Final = {
**local_vars,
**request_kwargs,
**({"tools": list(marked_tools)} if marked_tools is not None else {}),
"metadata": request_metadata,
}
anthropic_messages_optional_request_params: Final = (
AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param(
params=local_vars,
params=request_params,
model=model,
drop_params=litellm_params.get("drop_params") is True,
custom_llm_provider=custom_llm_provider,

View file

@ -111,7 +111,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
def _resolved_provider(self) -> str:
return self.custom_llm_provider or "anthropic"
def get_supported_anthropic_messages_params(self, model: str) -> list:
def get_supported_anthropic_messages_params(self, model: str) -> list[str]:
return [
"messages",
"model",
@ -666,7 +666,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
anthropic_messages_optional_request_params: dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> dict:
) -> dict[str, object]:
"""
No transformation is needed for Anthropic messages

View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -19,6 +20,11 @@ else:
class BaseAnthropicMessagesConfig(ABC):
def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None:
if metadata is None:
return None
return AnthropicMetadata.model_validate(metadata).model_dump(exclude_none=True)
@abstractmethod
def validate_anthropic_messages_environment( # use different name because return type is different from base config's validate_environment
self,

View file

@ -2605,14 +2605,14 @@ class BaseLLMHTTPHandler:
logging_obj.update_from_kwargs(
kwargs=kwargs,
model=model,
optional_params=dict(anthropic_messages_optional_request_params),
optional_params=dict(request_params),
litellm_params={
"preset_cache_key": None,
"stream_response": {},
"model_info": kwargs.get("model_info"),
**litellm_params.per_second_pricing(),
**vertex_location_params,
**anthropic_messages_optional_request_params,
**request_params,
},
custom_llm_provider=custom_llm_provider,
)

View file

@ -83,7 +83,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig):
anthropic_messages_optional_request_params: dict, # mutable-ok: matches dict-typed base signature
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: matches dict-typed base signature
) -> dict: # mutable-ok: matches dict-typed base signature
) -> dict[str, object]: # mutable-ok: matches dict-typed base signature
"""
Anthropic ignores prompt-caching hints it cannot honor, but strict
non-Anthropic implementations of the Messages API 400 the whole request

View file

@ -4,16 +4,16 @@ from typing import Final
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.llms.sail.common_utils import (
chat_request_for_sail,
completion_window_for_service_tier,
extra_body_for_sail,
json_body,
params_with_completion_window,
)
from litellm.types.llms.openai import AllMessageValues
_REJECTED_BY_SAIL: Final = frozenset(
{"stop", "seed", "frequency_penalty", "presence_penalty", "logit_bias", "logprobs", "top_logprobs"}
)
_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user")
_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user", "metadata")
class SailChatConfig(OpenAIGPTConfig):
@ -31,13 +31,15 @@ class SailChatConfig(OpenAIGPTConfig):
model: str,
drop_params: bool,
) -> dict: # mutable-ok: return type fixed by the base interface
completion_window_for_service_tier(non_default_params.get("service_tier"), model=model, drop_params=drop_params)
return super().map_openai_params(
mapped: Final = super().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=drop_params,
)
return json_body(
params_with_completion_window(mapped, model=model, drop_params=drop_params, allow_drop_window=False)
)
def transform_request(
self,

View file

@ -3,6 +3,7 @@ from types import MappingProxyType
from typing import Final, Literal, TypeAlias
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig
from litellm.types.utils import LlmProviders
@ -70,6 +71,41 @@ def completion_window_for_service_tier(
)
def _unserved_pairing(window: str | None, *, background: bool, source: str) -> str | None:
if window == "flex" and not background:
return (
f"sail serves {source} only to Responses requests with background=true. "
"Send 'balanced' for a synchronous request."
)
if window == "asap" and background:
return (
f"sail does not serve {source} to Responses requests with background=true. "
"Send 'balanced' or 'flex' for a background request."
)
return None
def _window_sail_serves(
window: str | None, *, background: bool, source: str, model: str, drop_params: bool | None
) -> str | None:
"""Sail serves ``flex`` only to Responses requests with ``background=true``
and ``asap`` only to requests without it, and answers the other pairings
with a 400, so they are rejected before the call. ``drop_params=None``
means the pairing cannot be dropped."""
message: Final = _unserved_pairing(window, background=background, source=source)
if message is None:
return window
if drop_params is None:
raise litellm.UnsupportedParamsError(message=message, llm_provider=SAIL, model=model)
if _dropping(drop_params):
return None
raise _unsupported(message, model)
def _tier_source(service_tier: object, window: str | None) -> str:
return f"service_tier={service_tier!r} (completion_window {window!r})"
def _metadata_without_caller_window(
metadata: object, *, field: str, model: str, drop_params: bool
) -> Mapping[str, object]:
@ -106,19 +142,18 @@ def extra_body_for_sail(
def chat_request_for_sail(request: Mapping[str, object], *, model: str, drop_params: bool) -> Mapping[str, object]:
raw_tier: Final = request.get("service_tier")
window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params)
caller_metadata: Final = _metadata_without_caller_window(
request.get("metadata"), field="metadata", model=model, drop_params=drop_params
normalized: Final = params_with_completion_window(
request, model=model, drop_params=drop_params, allow_drop_window=False
)
metadata: Final = MappingProxyType({**caller_metadata, "completion_window": window}) if window else caller_metadata
extra_body: Final = request.get("extra_body")
extra_body: Final = normalized.get("extra_body")
return MappingProxyType(
{
**without_keys(request, frozenset({"service_tier", "metadata", "extra_body"})),
**(_entry("metadata", metadata) if metadata else _EMPTY),
**without_keys(normalized, frozenset({"service_tier", "extra_body"})),
**(
_entry("extra_body", extra_body_for_sail(extra_body, metadata, model=model, drop_params=drop_params))
_entry(
"extra_body",
extra_body_for_sail(extra_body, normalized.get("metadata"), model=model, drop_params=drop_params),
)
if isinstance(extra_body, Mapping)
else _EMPTY
),
@ -138,11 +173,9 @@ def _caller_completion_window(window: object, *, model: str, drop_params: bool)
)
def responses_params_with_completion_window(
params: Mapping[str, object], *, model: str, drop_params: bool
def params_with_completion_window(
params: Mapping[str, object], *, model: str, drop_params: bool, allow_drop_window: bool = True
) -> Mapping[str, object]:
"""Responses billing reads these mapped params, so ``service_tier`` is kept
as the tier whose price columns match the window and stripped from the body later."""
raw_tier: Final = params.get("service_tier")
raw_metadata: Final = params.get("metadata")
metadata: Final[Mapping[str, object]] = raw_metadata if isinstance(raw_metadata, Mapping) else _EMPTY
@ -162,7 +195,15 @@ def responses_params_with_completion_window(
"select different completion windows. Send one of them.",
model,
)
window: Final = caller_window or tier_window
window: Final = _window_sail_serves(
caller_window or tier_window,
background=allow_drop_window and params.get("background") is True,
source=(
f"metadata.completion_window={caller_window!r}" if caller_window else _tier_source(raw_tier, tier_window)
),
model=model,
drop_params=drop_params if allow_drop_window else None,
)
other_metadata: Final = without_keys(metadata, frozenset({"completion_window"}))
wire_metadata: Final = (
MappingProxyType({**other_metadata, "completion_window": window}) if window else other_metadata
@ -175,3 +216,13 @@ def responses_params_with_completion_window(
**(_entry("service_tier", billed_tier) if billed_tier else _EMPTY),
}
)
def billed_service_tier(params: Mapping[str, object]) -> str | None:
metadata: Final = as_str_mapping(params.get("metadata"))
if metadata is not None:
window: Final = metadata.get("completion_window")
if isinstance(window, str) and window.lower() in _BILLED_TIER_FOR_WINDOW:
return _BILLED_TIER_FOR_WINDOW[window.lower()]
service_tier: Final = params.get("service_tier")
return service_tier if isinstance(service_tier, str) else None

View file

View file

@ -0,0 +1,40 @@
from collections.abc import Mapping
from typing import Final
from litellm.llms.openai_like.messages.transformation import JSONProviderAnthropicMessagesConfig
from litellm.llms.sail.common_utils import json_body, params_with_completion_window, without_keys
from litellm.types.router import GenericLiteLLMParams
class SailAnthropicMessagesConfig(JSONProviderAnthropicMessagesConfig):
def get_supported_anthropic_messages_params(
self, model: str
) -> list[str]: # mutable-ok: matches the provider interface
return [*super().get_supported_anthropic_messages_params(model), "metadata"]
def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None:
if metadata is None:
return None
return {key: value for key, value in metadata.items() if isinstance(value, str)} or None
def transform_anthropic_messages_request(
self,
model: str,
messages: list[dict[str, object]], # mutable-ok: matches the native Messages provider interface
anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches the provider interface
litellm_params: GenericLiteLLMParams,
headers: dict[str, str], # mutable-ok: matches the native Messages provider interface
) -> dict[str, object]: # mutable-ok: HTTP bodies and the provider interface require plain dicts
normalized: Final = params_with_completion_window(
anthropic_messages_optional_request_params,
model=model,
drop_params=bool(litellm_params.drop_params),
allow_drop_window=False,
)
return super().transform_anthropic_messages_request(
model=model,
messages=messages,
anthropic_messages_optional_request_params=json_body(without_keys(normalized, frozenset({"service_tier"}))),
litellm_params=litellm_params,
headers=headers,
)

View file

@ -5,7 +5,7 @@ from litellm.llms.openai_like.dynamic_config import create_responses_config_clas
from litellm.llms.sail.common_utils import (
extra_body_for_sail,
json_body,
responses_params_with_completion_window,
params_with_completion_window,
sail_provider_config,
without_keys,
)
@ -20,7 +20,7 @@ class SailResponsesAPIConfig(create_responses_config_class(sail_provider_config(
model: str,
drop_params: bool,
) -> dict: # mutable-ok: return type fixed by the base interface
params: Final = responses_params_with_completion_window(
params: Final = params_with_completion_window(
super().map_openai_params(
response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params
),

View file

@ -180,6 +180,7 @@ from litellm.utils import (
TextCompletionResponse,
TranscriptionResponse,
Usage,
add_openai_metadata,
add_provider_specific_params_to_optional_params,
async_mock_completion_streaming_obj,
convert_to_model_response_object,
@ -5487,6 +5488,20 @@ def _complete_databricks_agent(ctx: CompletionDispatchContext) -> _CompletionDis
)
def _caller_metadata_param(custom_llm_provider: str | None, metadata: object) -> Mapping[str, object]:
"""Sail reads the completion window from request metadata, so a Sail chat
request carries the caller's own body metadata (the proxy's
``requester_metadata`` copy, string values only). Kept out of
``completion`` because that function is at basedpyright's complexity limit."""
if custom_llm_provider != "sail":
return MappingProxyType({})
metadata_mapping: Final = as_str_mapping(metadata)
requester_metadata: Final = (
as_str_mapping(metadata_mapping.get("requester_metadata")) if metadata_mapping is not None else None
)
return MappingProxyType({"metadata": add_openai_metadata(requester_metadata)})
@tracer.wrap()
@client
def completion(
@ -6044,6 +6059,7 @@ def completion(
"prompt_cache_retention": prompt_cache_retention,
"allowed_openai_params": allowed_openai_params,
"base_model": base_model,
**_caller_metadata_param(custom_llm_provider, metadata),
}
provider_reasoning_effort: Final = reasoning_effort_with_summary_alias(
resolved_reasoning_effort,

View file

@ -10071,6 +10071,10 @@ class ProviderConfigManager:
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
json_provider: Final = JSONProviderRegistry.get(provider.value)
if provider == LlmProviders.SAIL and json_provider is not None:
from litellm.llms.sail.messages.transformation import SailAnthropicMessagesConfig
return SailAnthropicMessagesConfig(json_provider)
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
from litellm.llms.openai_like.messages.transformation import (
JSONProviderAnthropicMessagesConfig,

View file

@ -16,14 +16,62 @@ from tests.unit.llms.sail.helpers import (
)
MESSAGES: Final = [{"role": "user", "content": "hi"}]
@pytest.mark.asyncio
async def test_sail_chat_forwards_the_callers_window_without_proxy_metadata(
sail_env: None, chat_route: respx.Route, spend_capture: SpendCapture
) -> None:
await litellm.acompletion(
model=MODEL,
messages=MESSAGES,
metadata={
"requester_metadata": {"completion_window": "balanced", "trace_id": "caller-trace"},
"user_api_key_hash": "proxy-private",
},
litellm_call_id=spend_capture.call_id,
)
assert sent_body(chat_route)["metadata"] == {"completion_window": "balanced", "trace_id": "caller-trace"}
assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced"))
@pytest.mark.parametrize("drop_params", [False, True])
@pytest.mark.asyncio
async def test_sail_chat_rejects_a_caller_metadata_flex_window(
sail_env: None, chat_route: respx.Route, drop_params: bool
) -> None:
with pytest.raises(litellm.UnsupportedParamsError, match="background=true"):
await litellm.acompletion(
model=MODEL,
messages=MESSAGES,
metadata={"requester_metadata": {"completion_window": "flex"}},
drop_params=drop_params,
)
assert not chat_route.called
@pytest.mark.asyncio
async def test_sail_chat_rejects_a_conflicting_caller_window_and_tier(sail_env: None, chat_route: respx.Route) -> None:
with pytest.raises(litellm.UnsupportedParamsError, match="different completion windows"):
await litellm.acompletion(
model=MODEL,
messages=MESSAGES,
service_tier="priority",
metadata={"requester_metadata": {"completion_window": "balanced"}},
)
assert not chat_route.called
TIER_CASES: Final = [
pytest.param(None, None, "", id="no-tier"),
pytest.param("auto", None, "", id="auto"),
pytest.param("default", "asap", "", id="default"),
pytest.param("priority", "asap", "", id="priority"),
pytest.param("flex", "flex", "_flex", id="flex"),
pytest.param("balanced", "balanced", "_balanced", id="balanced"),
pytest.param("FLEX", "flex", "_flex", id="flex-any-case"),
pytest.param("BALANCED", "balanced", "_balanced", id="balanced-any-case"),
]
@ -110,6 +158,25 @@ async def test_sail_chat_rejects_a_tier_with_no_window_before_sending(
assert not chat_route.called
@pytest.mark.parametrize("service_tier", ["flex", "FLEX"])
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("drop_params", [False, True])
@pytest.mark.asyncio
async def test_sail_chat_rejects_a_flex_tier_before_sending_even_under_drop_params(
sail_env: None, chat_route: respx.Route, service_tier: str, stream: bool, drop_params: bool
) -> None:
with pytest.raises(
litellm.UnsupportedParamsError, match="only to Responses requests with background=true"
) as error:
await litellm.acompletion(
model=MODEL, messages=MESSAGES, service_tier=service_tier, stream=stream, drop_params=drop_params
)
assert error.value.status_code == 400
assert "drop_params" not in str(error.value)
assert not chat_route.called
@pytest.mark.parametrize("service_tier", ["bogus", "scale", 5])
@pytest.mark.parametrize(("global_drop", "request_drop"), [(False, True), (True, False)])
@pytest.mark.asyncio
@ -145,7 +212,7 @@ def chat_http_path(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPat
@pytest.mark.parametrize(
("service_tier", "wire_metadata", "column_suffix"),
[
pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"),
pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"),
pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"),
],
)
@ -281,13 +348,13 @@ async def test_sail_chat_keeps_the_window_when_preview_features_forward_caller_m
await litellm.acompletion(
model=MODEL,
messages=MESSAGES,
service_tier="flex",
service_tier="balanced",
metadata={"requester_metadata": {"trace_id": "t-1"}},
litellm_call_id=spend_capture.call_id,
)
assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "flex"}
assert await spend_capture.settled_cost() == pytest.approx(cost_at("_flex"))
assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "balanced"}
assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced"))
@pytest.mark.parametrize(

View file

@ -1,3 +1,5 @@
import json
from collections.abc import Mapping
from typing import Final
import httpx
@ -5,20 +7,65 @@ import pytest
import respx
import litellm
from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, messages_body, sent_body
from litellm.utils import ProviderConfigManager
from tests.unit.llms.sail.helpers import (
CACHED_TOKENS,
COMPLETION_TOKENS,
MODEL,
PROMPT_TOKENS,
SAIL_API_BASE,
SpendCapture,
cost_at,
messages_body,
sent_body,
)
MESSAGES: Final = [{"role": "user", "content": "hi"}]
def test_sail_messages_advertise_window_metadata_support() -> None:
config: Final = ProviderConfigManager.get_provider_anthropic_messages_config(
model=MODEL, provider=litellm.LlmProviders.SAIL
)
assert config is not None
assert "metadata" in config.get_supported_anthropic_messages_params(model=MODEL)
@pytest.mark.asyncio
async def test_sail_messages_forward_the_callers_window_and_bill_its_price(
sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture
) -> None:
await litellm.anthropic_messages(
model=MODEL,
messages=MESSAGES,
max_tokens=16,
metadata={"completion_window": "balanced", "trace_id": "caller-trace"},
litellm_metadata={"user_api_key_hash": "proxy-private"},
litellm_call_id=spend_capture.call_id,
)
assert sent_body(messages_route)["metadata"] == {"completion_window": "balanced", "trace_id": "caller-trace"}
assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced"))
@pytest.fixture
def messages_route(respx_mock: respx.MockRouter) -> respx.Route:
return respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=httpx.Response(200, json=messages_body()))
@pytest.mark.parametrize("service_tier", [None, "auto", "priority", "flex", "balanced", "scale"])
@pytest.mark.parametrize(
("service_tier", "window", "suffix"),
[(None, None, ""), ("auto", None, ""), ("priority", "asap", ""), ("balanced", "balanced", "_balanced")],
)
@pytest.mark.asyncio
async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier(
sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture, service_tier: str | None
async def test_sail_messages_send_and_bill_the_selected_window(
sail_env: None,
messages_route: respx.Route,
spend_capture: SpendCapture,
service_tier: str | None,
window: str | None,
suffix: str,
) -> None:
await litellm.anthropic_messages(
model=MODEL, messages=MESSAGES, max_tokens=16, service_tier=service_tier, litellm_call_id=spend_capture.call_id
@ -27,5 +74,133 @@ async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier(
body: Final = sent_body(messages_route)
assert body["messages"] == MESSAGES
assert "service_tier" not in body
assert "completion_window" not in (body.get("metadata") or {})
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
assert body.get("metadata") == ({"completion_window": window} if window else None)
assert await spend_capture.settled_cost() == pytest.approx(cost_at(suffix))
@pytest.mark.parametrize("drop_params", [False, True])
@pytest.mark.parametrize("as_metadata", [False, True])
@pytest.mark.asyncio
async def test_sail_messages_reject_flex_before_sending(
sail_env: None, messages_route: respx.Route, drop_params: bool, as_metadata: bool
) -> None:
params: Final = {"metadata": {"completion_window": "flex"}} if as_metadata else {"service_tier": "flex"}
with pytest.raises(litellm.UnsupportedParamsError, match="background=true"):
await litellm.anthropic_messages(
model=MODEL, messages=MESSAGES, max_tokens=16, drop_params=drop_params, **params
)
assert not messages_route.called
@pytest.mark.asyncio
async def test_sail_messages_reject_conflicting_window_and_tier(sail_env: None, messages_route: respx.Route) -> None:
with pytest.raises(litellm.UnsupportedParamsError, match="different completion windows"):
await litellm.anthropic_messages(
model=MODEL,
messages=MESSAGES,
max_tokens=16,
service_tier="priority",
metadata={"completion_window": "balanced"},
)
assert not messages_route.called
@pytest.mark.asyncio
async def test_sail_streamed_messages_bill_the_reported_window(
sail_env: None, respx_mock: respx.MockRouter, spend_capture: SpendCapture
) -> None:
respx_mock.post(f"{SAIL_API_BASE}/messages").mock(
return_value=httpx.Response(
200, content=_messages_stream("balanced"), headers={"content-type": "text/event-stream"}
)
)
response: Final = await litellm.anthropic_messages(
model=MODEL, messages=MESSAGES, max_tokens=16, stream=True, litellm_call_id=spend_capture.call_id
)
chunks: Final = tuple([chunk async for chunk in response])
assert chunks
assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced"))
def _messages_stream(service_tier: str | None = None) -> str:
body: Final = messages_body()
usage: Final = {
"input_tokens": PROMPT_TOKENS - CACHED_TOKENS,
"cache_read_input_tokens": CACHED_TOKENS,
"output_tokens": COMPLETION_TOKENS,
**({"service_tier": service_tier} if service_tier else {}),
}
events: Final = (
{"type": "message_start", "message": {**body, "content": [], "usage": {**usage, "output_tokens": 0}}},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}},
{"type": "content_block_stop", "index": 0},
{"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": usage},
{"type": "message_stop"},
)
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(
("params", "drop_path", "wire_metadata", "suffix"),
[
({"service_tier": "flex"}, "service_tier", None, ""),
({"service_tier": "balanced"}, "service_tier", None, ""),
({"metadata": {"completion_window": "balanced"}}, "metadata", None, ""),
(
{"metadata": {"completion_window": "balanced", "trace_id": "caller"}},
"metadata.completion_window",
{"trace_id": "caller"},
"",
),
(
{"service_tier": "flex", "metadata": {"completion_window": "balanced"}},
"service_tier",
{"completion_window": "balanced"},
"_balanced",
),
(
{"service_tier": "balanced", "metadata": {"completion_window": "flex", "trace_id": "caller"}},
"metadata.completion_window",
{"completion_window": "balanced", "trace_id": "caller"},
"_balanced",
),
],
)
@pytest.mark.asyncio
async def test_sail_messages_bill_the_window_sent_after_parameter_drops(
sail_env: None,
respx_mock: respx.MockRouter,
spend_capture: SpendCapture,
params: Mapping[str, object],
drop_path: str,
wire_metadata: Mapping[str, str] | None,
suffix: str,
stream: bool,
) -> None:
upstream: Final = (
httpx.Response(200, content=_messages_stream(), headers={"content-type": "text/event-stream"})
if stream
else httpx.Response(200, json=messages_body())
)
route: Final = respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=upstream)
response: Final = await litellm.anthropic_messages(
model=MODEL,
messages=MESSAGES,
max_tokens=16,
stream=stream,
additional_drop_params=[drop_path],
litellm_call_id=spend_capture.call_id,
**params,
)
if stream:
async for _ in response:
pass
assert sent_body(route).get("metadata") == wire_metadata
assert await spend_capture.settled_cost() == pytest.approx(cost_at(suffix))

View file

@ -16,31 +16,47 @@ def responses_route(respx_mock: respx.MockRouter) -> respx.Route:
@pytest.mark.parametrize(
("service_tier", "metadata", "wire_metadata", "column_suffix"),
("service_tier", "metadata", "background", "wire_metadata", "column_suffix"),
[
pytest.param(None, None, None, "", id="no-tier"),
pytest.param("auto", None, None, "", id="auto"),
pytest.param("default", None, {"completion_window": "asap"}, "", id="default"),
pytest.param("priority", None, {"completion_window": "asap"}, "", id="priority"),
pytest.param("flex", None, {"completion_window": "flex"}, "_flex", id="flex"),
pytest.param("balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced"),
pytest.param("Balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"),
pytest.param(None, None, None, None, "", id="no-tier"),
pytest.param("auto", None, None, None, "", id="auto"),
pytest.param("default", None, None, {"completion_window": "asap"}, "", id="default"),
pytest.param("priority", None, None, {"completion_window": "asap"}, "", id="priority"),
pytest.param("flex", None, True, {"completion_window": "flex"}, "_flex", id="flex-background"),
pytest.param("balanced", None, None, {"completion_window": "balanced"}, "_balanced", id="balanced"),
pytest.param("balanced", None, True, {"completion_window": "balanced"}, "_balanced", id="balanced-background"),
pytest.param("Balanced", None, None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"),
pytest.param(
"flex", {"user_tag": "a"}, {"user_tag": "a", "completion_window": "flex"}, "_flex", id="tier-keeps-metadata"
"flex",
{"user_tag": "a"},
True,
{"user_tag": "a", "completion_window": "flex"},
"_flex",
id="tier-keeps-metadata",
),
pytest.param(
None, {"completion_window": "flex"}, True, {"completion_window": "flex"}, "_flex", id="caller-window"
),
pytest.param(None, {"completion_window": "flex"}, {"completion_window": "flex"}, "_flex", id="caller-window"),
pytest.param(
None,
{"completion_window": "standard"},
None,
{"completion_window": "standard"},
"_balanced",
id="standard-window",
),
pytest.param(None, {"completion_window": "FLEX"}, {"completion_window": "flex"}, "_flex", id="window-any-case"),
pytest.param(
"priority", {"completion_window": "asap"}, {"completion_window": "asap"}, "", id="agreeing-tier-and-window"
None, {"completion_window": "FLEX"}, True, {"completion_window": "flex"}, "_flex", id="window-any-case"
),
pytest.param(None, {"user_tag": "a"}, {"user_tag": "a"}, "", id="metadata-without-window"),
pytest.param(
"priority",
{"completion_window": "asap"},
False,
{"completion_window": "asap"},
"",
id="agreeing-tier-and-window",
),
pytest.param(None, {"user_tag": "a"}, None, {"user_tag": "a"}, "", id="metadata-without-window"),
],
)
@pytest.mark.asyncio
@ -50,6 +66,7 @@ async def test_sail_responses_send_the_window_and_bill_its_price_columns(
spend_capture: SpendCapture,
service_tier: str | None,
metadata: dict[str, str] | None,
background: bool | None,
wire_metadata: dict[str, str] | None,
column_suffix: str,
) -> None:
@ -58,22 +75,36 @@ async def test_sail_responses_send_the_window_and_bill_its_price_columns(
input=INPUT,
service_tier=service_tier,
metadata=metadata,
background=background,
litellm_call_id=spend_capture.call_id,
)
body: Final = sent_body(responses_route)
assert "service_tier" not in body
assert body.get("metadata") == wire_metadata
assert body.get("background") == background
assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix))
FLEX_NEEDS_BACKGROUND: Final = "only to Responses requests with background=true"
ASAP_CANNOT_RUN_IN_BACKGROUND: Final = "does not serve .* to Responses requests with background=true"
@pytest.mark.parametrize(
("service_tier", "metadata", "message"),
("service_tier", "metadata", "background", "message"),
[
pytest.param("scale", None, "service_tier='scale'", id="unknown-tier"),
pytest.param(5, None, "service_tier=5", id="non-string-tier"),
pytest.param(None, {"completion_window": "soon"}, "completion_window='soon'", id="unknown-window"),
pytest.param("flex", {"completion_window": "asap"}, "select different completion windows", id="conflict"),
pytest.param("scale", None, None, "service_tier='scale'", id="unknown-tier"),
pytest.param(5, None, None, "service_tier=5", id="non-string-tier"),
pytest.param(None, {"completion_window": "soon"}, None, "completion_window='soon'", id="unknown-window"),
pytest.param("flex", {"completion_window": "asap"}, None, "select different completion windows", id="conflict"),
pytest.param("flex", None, None, FLEX_NEEDS_BACKGROUND, id="flex-tier-in-foreground"),
pytest.param("FLEX", None, False, FLEX_NEEDS_BACKGROUND, id="flex-tier-with-background-false"),
pytest.param(None, {"completion_window": "flex"}, None, FLEX_NEEDS_BACKGROUND, id="flex-window-in-foreground"),
pytest.param("priority", None, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="priority-tier-in-background"),
pytest.param("default", None, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="default-tier-in-background"),
pytest.param(
None, {"completion_window": "asap"}, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="asap-window-in-background"
),
],
)
@pytest.mark.asyncio
@ -82,14 +113,51 @@ async def test_sail_responses_reject_before_sending(
responses_route: respx.Route,
service_tier: object,
metadata: dict[str, str] | None,
background: bool | None,
message: str,
) -> None:
with pytest.raises(litellm.UnsupportedParamsError, match=message):
await litellm.aresponses(model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata)
await litellm.aresponses(
model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata, background=background
)
assert not responses_route.called
@pytest.mark.parametrize(
("service_tier", "metadata", "background", "wire_metadata"),
[
pytest.param("flex", None, None, None, id="flex-tier-in-foreground"),
pytest.param(None, {"completion_window": "flex", "user_tag": "a"}, None, {"user_tag": "a"}, id="flex-window"),
pytest.param("priority", None, True, None, id="priority-tier-in-background"),
],
)
@pytest.mark.asyncio
async def test_sail_responses_drop_a_window_sail_cannot_serve_under_drop_params_and_bill_base(
sail_env: None,
responses_route: respx.Route,
spend_capture: SpendCapture,
service_tier: str | None,
metadata: dict[str, str] | None,
background: bool | None,
wire_metadata: dict[str, str] | None,
) -> None:
await litellm.aresponses(
model=MODEL,
input=INPUT,
service_tier=service_tier,
metadata=metadata,
background=background,
drop_params=True,
litellm_call_id=spend_capture.call_id,
)
body: Final = sent_body(responses_route)
assert "service_tier" not in body
assert body.get("metadata") == wire_metadata
assert await spend_capture.settled_cost() == pytest.approx(cost_at(""))
@pytest.mark.asyncio
async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params(
sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture
@ -112,7 +180,7 @@ async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params(
@pytest.mark.parametrize(
("service_tier", "wire_metadata", "column_suffix"),
[
pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"),
pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"),
pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"),
],
)