diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a4a9a5bee39..f2a23c8e5bc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1647,8 +1647,17 @@ def completion_cost( if explicit_tier is not None: service_tier = explicit_tier else: + from litellm.llms.sail.common_utils import billed_service_tier + + requested_tier: Final[object] = ( + billed_service_tier(optional_params) + if custom_llm_provider == "sail" and optional_params is not None + else optional_params.get("service_tier") + if optional_params is not None + else None + ) service_tier = _resolve_billable_service_tier( # rebind-ok: resolved from request then response - requested=optional_params.get("service_tier") if optional_params is not None else None, + requested=requested_tier, served=_served_service_tier(completion_response, cost_per_token_usage_object), ) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 8e1791c3c92..66d880b1ba8 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1207,6 +1207,7 @@ class ChunkProcessor: for field, value in ( ("inference_geo", calculated_usage_per_chunk["inference_geo"]), ("speed", calculated_usage_per_chunk["speed"]), + ("service_tier", self._last_provider_pricing_field(chunks, "service_tier")), ) if value is not None } diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 865247afc63..12fe74e03e4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -410,7 +410,7 @@ DROP_UNSUPPORTED_SERVICE_TIER_WARNING: Final = ( "Anthropic only accepts 'auto' and 'standard_only'." ) -MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle"} +MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle", "sail"} DROP_SERVICE_TIER_PROVIDER_WARNING: Final = ( "Dropping `service_tier` for model=%s on provider=%s. Only the Anthropic API routes accept it." diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..3b745294133 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -600,9 +600,8 @@ def anthropic_messages_handler( presanitized=bool(kwargs.pop("_litellm_messages_presanitized", False)), ) - metadata = validate_anthropic_api_metadata(metadata) - local_vars: Final = locals() + anthropic_metadata: Final = validate_anthropic_api_metadata(metadata) is_async: Final = kwargs.pop("is_async", False) # Use provided client or create a new one litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") @@ -667,7 +666,7 @@ def anthropic_messages_handler( max_tokens=max_tokens, messages=messages, model=model, - metadata=metadata, + metadata=anthropic_metadata, stop_sequences=stop_sequences, stream=stream, system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower @@ -699,7 +698,7 @@ def anthropic_messages_handler( max_tokens=max_tokens, messages=messages, model=original_model, - metadata=metadata, + metadata=anthropic_metadata, stop_sequences=stop_sequences, stream=stream, system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower @@ -729,7 +728,7 @@ def anthropic_messages_handler( max_tokens=max_tokens, messages=messages, model=original_model, - metadata=metadata, + metadata=anthropic_metadata, stop_sequences=stop_sequences, stream=stream, system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower @@ -760,10 +759,16 @@ def anthropic_messages_handler( ) if marked_tools is not None: AnthropicCacheControlHook.record_gateway_injection(request_kwargs, 1) - local_vars.update({**request_kwargs, **({"tools": list(marked_tools)} if marked_tools is not None else {})}) + request_metadata: Final = anthropic_messages_provider_config.request_metadata(metadata) + request_params: Final = { + **local_vars, + **request_kwargs, + **({"tools": list(marked_tools)} if marked_tools is not None else {}), + "metadata": request_metadata, + } anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( - params=local_vars, + params=request_params, model=model, drop_params=litellm_params.get("drop_params") is True, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index c086618e829..610a5476b7e 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -111,7 +111,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _resolved_provider(self) -> str: return self.custom_llm_provider or "anthropic" - def get_supported_anthropic_messages_params(self, model: str) -> list: + def get_supported_anthropic_messages_params(self, model: str) -> list[str]: return [ "messages", "model", @@ -666,7 +666,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): anthropic_messages_optional_request_params: dict, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> dict: + ) -> dict[str, object]: """ No transformation is needed for Anthropic messages diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bda7015ab2b..f23ed695a66 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -19,6 +20,11 @@ else: class BaseAnthropicMessagesConfig(ABC): + def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None: + if metadata is None: + return None + return AnthropicMetadata.model_validate(metadata).model_dump(exclude_none=True) + @abstractmethod def validate_anthropic_messages_environment( # use different name because return type is different from base config's validate_environment self, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4178095c9ef..143106208af 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2605,14 +2605,14 @@ class BaseLLMHTTPHandler: logging_obj.update_from_kwargs( kwargs=kwargs, model=model, - optional_params=dict(anthropic_messages_optional_request_params), + optional_params=dict(request_params), litellm_params={ "preset_cache_key": None, "stream_response": {}, "model_info": kwargs.get("model_info"), **litellm_params.per_second_pricing(), **vertex_location_params, - **anthropic_messages_optional_request_params, + **request_params, }, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index a5d825a39d7..4c2d419a05f 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -83,7 +83,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): anthropic_messages_optional_request_params: dict, # mutable-ok: matches dict-typed base signature litellm_params: GenericLiteLLMParams, headers: dict, # mutable-ok: matches dict-typed base signature - ) -> dict: # mutable-ok: matches dict-typed base signature + ) -> dict[str, object]: # mutable-ok: matches dict-typed base signature """ Anthropic ignores prompt-caching hints it cannot honor, but strict non-Anthropic implementations of the Messages API 400 the whole request diff --git a/litellm/llms/sail/chat/transformation.py b/litellm/llms/sail/chat/transformation.py index 64f69af06fa..6eb88e820cd 100644 --- a/litellm/llms/sail/chat/transformation.py +++ b/litellm/llms/sail/chat/transformation.py @@ -4,16 +4,16 @@ from typing import Final from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.sail.common_utils import ( chat_request_for_sail, - completion_window_for_service_tier, extra_body_for_sail, json_body, + params_with_completion_window, ) from litellm.types.llms.openai import AllMessageValues _REJECTED_BY_SAIL: Final = frozenset( {"stop", "seed", "frequency_penalty", "presence_penalty", "logit_bias", "logprobs", "top_logprobs"} ) -_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user") +_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user", "metadata") class SailChatConfig(OpenAIGPTConfig): @@ -31,13 +31,15 @@ class SailChatConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: # mutable-ok: return type fixed by the base interface - completion_window_for_service_tier(non_default_params.get("service_tier"), model=model, drop_params=drop_params) - return super().map_openai_params( + mapped: Final = super().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, drop_params=drop_params, ) + return json_body( + params_with_completion_window(mapped, model=model, drop_params=drop_params, allow_drop_window=False) + ) def transform_request( self, diff --git a/litellm/llms/sail/common_utils.py b/litellm/llms/sail/common_utils.py index cb00ed14214..f2f3e948376 100644 --- a/litellm/llms/sail/common_utils.py +++ b/litellm/llms/sail/common_utils.py @@ -3,6 +3,7 @@ from types import MappingProxyType from typing import Final, Literal, TypeAlias import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig from litellm.types.utils import LlmProviders @@ -70,6 +71,41 @@ def completion_window_for_service_tier( ) +def _unserved_pairing(window: str | None, *, background: bool, source: str) -> str | None: + if window == "flex" and not background: + return ( + f"sail serves {source} only to Responses requests with background=true. " + "Send 'balanced' for a synchronous request." + ) + if window == "asap" and background: + return ( + f"sail does not serve {source} to Responses requests with background=true. " + "Send 'balanced' or 'flex' for a background request." + ) + return None + + +def _window_sail_serves( + window: str | None, *, background: bool, source: str, model: str, drop_params: bool | None +) -> str | None: + """Sail serves ``flex`` only to Responses requests with ``background=true`` + and ``asap`` only to requests without it, and answers the other pairings + with a 400, so they are rejected before the call. ``drop_params=None`` + means the pairing cannot be dropped.""" + message: Final = _unserved_pairing(window, background=background, source=source) + if message is None: + return window + if drop_params is None: + raise litellm.UnsupportedParamsError(message=message, llm_provider=SAIL, model=model) + if _dropping(drop_params): + return None + raise _unsupported(message, model) + + +def _tier_source(service_tier: object, window: str | None) -> str: + return f"service_tier={service_tier!r} (completion_window {window!r})" + + def _metadata_without_caller_window( metadata: object, *, field: str, model: str, drop_params: bool ) -> Mapping[str, object]: @@ -106,19 +142,18 @@ def extra_body_for_sail( def chat_request_for_sail(request: Mapping[str, object], *, model: str, drop_params: bool) -> Mapping[str, object]: - raw_tier: Final = request.get("service_tier") - window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params) - caller_metadata: Final = _metadata_without_caller_window( - request.get("metadata"), field="metadata", model=model, drop_params=drop_params + normalized: Final = params_with_completion_window( + request, model=model, drop_params=drop_params, allow_drop_window=False ) - metadata: Final = MappingProxyType({**caller_metadata, "completion_window": window}) if window else caller_metadata - extra_body: Final = request.get("extra_body") + extra_body: Final = normalized.get("extra_body") return MappingProxyType( { - **without_keys(request, frozenset({"service_tier", "metadata", "extra_body"})), - **(_entry("metadata", metadata) if metadata else _EMPTY), + **without_keys(normalized, frozenset({"service_tier", "extra_body"})), **( - _entry("extra_body", extra_body_for_sail(extra_body, metadata, model=model, drop_params=drop_params)) + _entry( + "extra_body", + extra_body_for_sail(extra_body, normalized.get("metadata"), model=model, drop_params=drop_params), + ) if isinstance(extra_body, Mapping) else _EMPTY ), @@ -138,11 +173,9 @@ def _caller_completion_window(window: object, *, model: str, drop_params: bool) ) -def responses_params_with_completion_window( - params: Mapping[str, object], *, model: str, drop_params: bool +def params_with_completion_window( + params: Mapping[str, object], *, model: str, drop_params: bool, allow_drop_window: bool = True ) -> Mapping[str, object]: - """Responses billing reads these mapped params, so ``service_tier`` is kept - as the tier whose price columns match the window and stripped from the body later.""" raw_tier: Final = params.get("service_tier") raw_metadata: Final = params.get("metadata") metadata: Final[Mapping[str, object]] = raw_metadata if isinstance(raw_metadata, Mapping) else _EMPTY @@ -162,7 +195,15 @@ def responses_params_with_completion_window( "select different completion windows. Send one of them.", model, ) - window: Final = caller_window or tier_window + window: Final = _window_sail_serves( + caller_window or tier_window, + background=allow_drop_window and params.get("background") is True, + source=( + f"metadata.completion_window={caller_window!r}" if caller_window else _tier_source(raw_tier, tier_window) + ), + model=model, + drop_params=drop_params if allow_drop_window else None, + ) other_metadata: Final = without_keys(metadata, frozenset({"completion_window"})) wire_metadata: Final = ( MappingProxyType({**other_metadata, "completion_window": window}) if window else other_metadata @@ -175,3 +216,13 @@ def responses_params_with_completion_window( **(_entry("service_tier", billed_tier) if billed_tier else _EMPTY), } ) + + +def billed_service_tier(params: Mapping[str, object]) -> str | None: + metadata: Final = as_str_mapping(params.get("metadata")) + if metadata is not None: + window: Final = metadata.get("completion_window") + if isinstance(window, str) and window.lower() in _BILLED_TIER_FOR_WINDOW: + return _BILLED_TIER_FOR_WINDOW[window.lower()] + service_tier: Final = params.get("service_tier") + return service_tier if isinstance(service_tier, str) else None diff --git a/litellm/llms/sail/messages/__init__.py b/litellm/llms/sail/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/sail/messages/transformation.py b/litellm/llms/sail/messages/transformation.py new file mode 100644 index 00000000000..3e0dd9a4b24 --- /dev/null +++ b/litellm/llms/sail/messages/transformation.py @@ -0,0 +1,40 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.openai_like.messages.transformation import JSONProviderAnthropicMessagesConfig +from litellm.llms.sail.common_utils import json_body, params_with_completion_window, without_keys +from litellm.types.router import GenericLiteLLMParams + + +class SailAnthropicMessagesConfig(JSONProviderAnthropicMessagesConfig): + def get_supported_anthropic_messages_params( + self, model: str + ) -> list[str]: # mutable-ok: matches the provider interface + return [*super().get_supported_anthropic_messages_params(model), "metadata"] + + def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None: + if metadata is None: + return None + return {key: value for key, value in metadata.items() if isinstance(value, str)} or None + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict[str, object]], # mutable-ok: matches the native Messages provider interface + anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches the provider interface + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: matches the native Messages provider interface + ) -> dict[str, object]: # mutable-ok: HTTP bodies and the provider interface require plain dicts + normalized: Final = params_with_completion_window( + anthropic_messages_optional_request_params, + model=model, + drop_params=bool(litellm_params.drop_params), + allow_drop_window=False, + ) + return super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=json_body(without_keys(normalized, frozenset({"service_tier"}))), + litellm_params=litellm_params, + headers=headers, + ) diff --git a/litellm/llms/sail/responses/transformation.py b/litellm/llms/sail/responses/transformation.py index c3c39a125f5..9dd86569084 100644 --- a/litellm/llms/sail/responses/transformation.py +++ b/litellm/llms/sail/responses/transformation.py @@ -5,7 +5,7 @@ from litellm.llms.openai_like.dynamic_config import create_responses_config_clas from litellm.llms.sail.common_utils import ( extra_body_for_sail, json_body, - responses_params_with_completion_window, + params_with_completion_window, sail_provider_config, without_keys, ) @@ -20,7 +20,7 @@ class SailResponsesAPIConfig(create_responses_config_class(sail_provider_config( model: str, drop_params: bool, ) -> dict: # mutable-ok: return type fixed by the base interface - params: Final = responses_params_with_completion_window( + params: Final = params_with_completion_window( super().map_openai_params( response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params ), diff --git a/litellm/main.py b/litellm/main.py index 50ab274a060..dc16767266e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -180,6 +180,7 @@ from litellm.utils import ( TextCompletionResponse, TranscriptionResponse, Usage, + add_openai_metadata, add_provider_specific_params_to_optional_params, async_mock_completion_streaming_obj, convert_to_model_response_object, @@ -5487,6 +5488,20 @@ def _complete_databricks_agent(ctx: CompletionDispatchContext) -> _CompletionDis ) +def _caller_metadata_param(custom_llm_provider: str | None, metadata: object) -> Mapping[str, object]: + """Sail reads the completion window from request metadata, so a Sail chat + request carries the caller's own body metadata (the proxy's + ``requester_metadata`` copy, string values only). Kept out of + ``completion`` because that function is at basedpyright's complexity limit.""" + if custom_llm_provider != "sail": + return MappingProxyType({}) + metadata_mapping: Final = as_str_mapping(metadata) + requester_metadata: Final = ( + as_str_mapping(metadata_mapping.get("requester_metadata")) if metadata_mapping is not None else None + ) + return MappingProxyType({"metadata": add_openai_metadata(requester_metadata)}) + + @tracer.wrap() @client def completion( @@ -6044,6 +6059,7 @@ def completion( "prompt_cache_retention": prompt_cache_retention, "allowed_openai_params": allowed_openai_params, "base_model": base_model, + **_caller_metadata_param(custom_llm_provider, metadata), } provider_reasoning_effort: Final = reasoning_effort_with_summary_alias( resolved_reasoning_effort, diff --git a/litellm/utils.py b/litellm/utils.py index be82e65d1b9..6670a836f48 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10071,6 +10071,10 @@ class ProviderConfigManager: from litellm.llms.openai_like.json_loader import JSONProviderRegistry json_provider: Final = JSONProviderRegistry.get(provider.value) + if provider == LlmProviders.SAIL and json_provider is not None: + from litellm.llms.sail.messages.transformation import SailAnthropicMessagesConfig + + return SailAnthropicMessagesConfig(json_provider) if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints: from litellm.llms.openai_like.messages.transformation import ( JSONProviderAnthropicMessagesConfig, diff --git a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py index c7b1a77343a..b7138a60873 100644 --- a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py +++ b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py @@ -16,14 +16,62 @@ from tests.unit.llms.sail.helpers import ( ) MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +@pytest.mark.asyncio +async def test_sail_chat_forwards_the_callers_window_without_proxy_metadata( + sail_env: None, chat_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + metadata={ + "requester_metadata": {"completion_window": "balanced", "trace_id": "caller-trace"}, + "user_api_key_hash": "proxy-private", + }, + litellm_call_id=spend_capture.call_id, + ) + + assert sent_body(chat_route)["metadata"] == {"completion_window": "balanced", "trace_id": "caller-trace"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_caller_metadata_flex_window( + sail_env: None, chat_route: respx.Route, drop_params: bool +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="background=true"): + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + metadata={"requester_metadata": {"completion_window": "flex"}}, + drop_params=drop_params, + ) + + assert not chat_route.called + + +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_conflicting_caller_window_and_tier(sail_env: None, chat_route: respx.Route) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="different completion windows"): + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier="priority", + metadata={"requester_metadata": {"completion_window": "balanced"}}, + ) + + assert not chat_route.called + + TIER_CASES: Final = [ pytest.param(None, None, "", id="no-tier"), pytest.param("auto", None, "", id="auto"), pytest.param("default", "asap", "", id="default"), pytest.param("priority", "asap", "", id="priority"), - pytest.param("flex", "flex", "_flex", id="flex"), pytest.param("balanced", "balanced", "_balanced", id="balanced"), - pytest.param("FLEX", "flex", "_flex", id="flex-any-case"), + pytest.param("BALANCED", "balanced", "_balanced", id="balanced-any-case"), ] @@ -110,6 +158,25 @@ async def test_sail_chat_rejects_a_tier_with_no_window_before_sending( assert not chat_route.called +@pytest.mark.parametrize("service_tier", ["flex", "FLEX"]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_flex_tier_before_sending_even_under_drop_params( + sail_env: None, chat_route: respx.Route, service_tier: str, stream: bool, drop_params: bool +) -> None: + with pytest.raises( + litellm.UnsupportedParamsError, match="only to Responses requests with background=true" + ) as error: + await litellm.acompletion( + model=MODEL, messages=MESSAGES, service_tier=service_tier, stream=stream, drop_params=drop_params + ) + + assert error.value.status_code == 400 + assert "drop_params" not in str(error.value) + assert not chat_route.called + + @pytest.mark.parametrize("service_tier", ["bogus", "scale", 5]) @pytest.mark.parametrize(("global_drop", "request_drop"), [(False, True), (True, False)]) @pytest.mark.asyncio @@ -145,7 +212,7 @@ def chat_http_path(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPat @pytest.mark.parametrize( ("service_tier", "wire_metadata", "column_suffix"), [ - pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"), + pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"), pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), ], ) @@ -281,13 +348,13 @@ async def test_sail_chat_keeps_the_window_when_preview_features_forward_caller_m await litellm.acompletion( model=MODEL, messages=MESSAGES, - service_tier="flex", + service_tier="balanced", metadata={"requester_metadata": {"trace_id": "t-1"}}, litellm_call_id=spend_capture.call_id, ) - assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "flex"} - assert await spend_capture.settled_cost() == pytest.approx(cost_at("_flex")) + assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "balanced"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) @pytest.mark.parametrize( diff --git a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py index c6e74534791..9fac963f2d7 100644 --- a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py +++ b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py @@ -1,3 +1,5 @@ +import json +from collections.abc import Mapping from typing import Final import httpx @@ -5,20 +7,65 @@ import pytest import respx import litellm -from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, messages_body, sent_body +from litellm.utils import ProviderConfigManager +from tests.unit.llms.sail.helpers import ( + CACHED_TOKENS, + COMPLETION_TOKENS, + MODEL, + PROMPT_TOKENS, + SAIL_API_BASE, + SpendCapture, + cost_at, + messages_body, + sent_body, +) MESSAGES: Final = [{"role": "user", "content": "hi"}] +def test_sail_messages_advertise_window_metadata_support() -> None: + config: Final = ProviderConfigManager.get_provider_anthropic_messages_config( + model=MODEL, provider=litellm.LlmProviders.SAIL + ) + + assert config is not None + assert "metadata" in config.get_supported_anthropic_messages_params(model=MODEL) + + +@pytest.mark.asyncio +async def test_sail_messages_forward_the_callers_window_and_bill_its_price( + sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.anthropic_messages( + model=MODEL, + messages=MESSAGES, + max_tokens=16, + metadata={"completion_window": "balanced", "trace_id": "caller-trace"}, + litellm_metadata={"user_api_key_hash": "proxy-private"}, + litellm_call_id=spend_capture.call_id, + ) + + assert sent_body(messages_route)["metadata"] == {"completion_window": "balanced", "trace_id": "caller-trace"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) + + @pytest.fixture def messages_route(respx_mock: respx.MockRouter) -> respx.Route: return respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=httpx.Response(200, json=messages_body())) -@pytest.mark.parametrize("service_tier", [None, "auto", "priority", "flex", "balanced", "scale"]) +@pytest.mark.parametrize( + ("service_tier", "window", "suffix"), + [(None, None, ""), ("auto", None, ""), ("priority", "asap", ""), ("balanced", "balanced", "_balanced")], +) @pytest.mark.asyncio -async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier( - sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture, service_tier: str | None +async def test_sail_messages_send_and_bill_the_selected_window( + sail_env: None, + messages_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + window: str | None, + suffix: str, ) -> None: await litellm.anthropic_messages( model=MODEL, messages=MESSAGES, max_tokens=16, service_tier=service_tier, litellm_call_id=spend_capture.call_id @@ -27,5 +74,133 @@ async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier( body: Final = sent_body(messages_route) assert body["messages"] == MESSAGES assert "service_tier" not in body - assert "completion_window" not in (body.get("metadata") or {}) - assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + assert body.get("metadata") == ({"completion_window": window} if window else None) + assert await spend_capture.settled_cost() == pytest.approx(cost_at(suffix)) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("as_metadata", [False, True]) +@pytest.mark.asyncio +async def test_sail_messages_reject_flex_before_sending( + sail_env: None, messages_route: respx.Route, drop_params: bool, as_metadata: bool +) -> None: + params: Final = {"metadata": {"completion_window": "flex"}} if as_metadata else {"service_tier": "flex"} + with pytest.raises(litellm.UnsupportedParamsError, match="background=true"): + await litellm.anthropic_messages( + model=MODEL, messages=MESSAGES, max_tokens=16, drop_params=drop_params, **params + ) + + assert not messages_route.called + + +@pytest.mark.asyncio +async def test_sail_messages_reject_conflicting_window_and_tier(sail_env: None, messages_route: respx.Route) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="different completion windows"): + await litellm.anthropic_messages( + model=MODEL, + messages=MESSAGES, + max_tokens=16, + service_tier="priority", + metadata={"completion_window": "balanced"}, + ) + + assert not messages_route.called + + +@pytest.mark.asyncio +async def test_sail_streamed_messages_bill_the_reported_window( + sail_env: None, respx_mock: respx.MockRouter, spend_capture: SpendCapture +) -> None: + respx_mock.post(f"{SAIL_API_BASE}/messages").mock( + return_value=httpx.Response( + 200, content=_messages_stream("balanced"), headers={"content-type": "text/event-stream"} + ) + ) + + response: Final = await litellm.anthropic_messages( + model=MODEL, messages=MESSAGES, max_tokens=16, stream=True, litellm_call_id=spend_capture.call_id + ) + chunks: Final = tuple([chunk async for chunk in response]) + + assert chunks + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) + + +def _messages_stream(service_tier: str | None = None) -> str: + body: Final = messages_body() + usage: Final = { + "input_tokens": PROMPT_TOKENS - CACHED_TOKENS, + "cache_read_input_tokens": CACHED_TOKENS, + "output_tokens": COMPLETION_TOKENS, + **({"service_tier": service_tier} if service_tier else {}), + } + events: Final = ( + {"type": "message_start", "message": {**body, "content": [], "usage": {**usage, "output_tokens": 0}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": usage}, + {"type": "message_stop"}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("params", "drop_path", "wire_metadata", "suffix"), + [ + ({"service_tier": "flex"}, "service_tier", None, ""), + ({"service_tier": "balanced"}, "service_tier", None, ""), + ({"metadata": {"completion_window": "balanced"}}, "metadata", None, ""), + ( + {"metadata": {"completion_window": "balanced", "trace_id": "caller"}}, + "metadata.completion_window", + {"trace_id": "caller"}, + "", + ), + ( + {"service_tier": "flex", "metadata": {"completion_window": "balanced"}}, + "service_tier", + {"completion_window": "balanced"}, + "_balanced", + ), + ( + {"service_tier": "balanced", "metadata": {"completion_window": "flex", "trace_id": "caller"}}, + "metadata.completion_window", + {"completion_window": "balanced", "trace_id": "caller"}, + "_balanced", + ), + ], +) +@pytest.mark.asyncio +async def test_sail_messages_bill_the_window_sent_after_parameter_drops( + sail_env: None, + respx_mock: respx.MockRouter, + spend_capture: SpendCapture, + params: Mapping[str, object], + drop_path: str, + wire_metadata: Mapping[str, str] | None, + suffix: str, + stream: bool, +) -> None: + upstream: Final = ( + httpx.Response(200, content=_messages_stream(), headers={"content-type": "text/event-stream"}) + if stream + else httpx.Response(200, json=messages_body()) + ) + route: Final = respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=upstream) + response: Final = await litellm.anthropic_messages( + model=MODEL, + messages=MESSAGES, + max_tokens=16, + stream=stream, + additional_drop_params=[drop_path], + litellm_call_id=spend_capture.call_id, + **params, + ) + if stream: + async for _ in response: + pass + + assert sent_body(route).get("metadata") == wire_metadata + assert await spend_capture.settled_cost() == pytest.approx(cost_at(suffix)) diff --git a/tests/unit/llms/sail/responses/test_sail_responses_transformation.py b/tests/unit/llms/sail/responses/test_sail_responses_transformation.py index 3384b79bdec..cc687e74b57 100644 --- a/tests/unit/llms/sail/responses/test_sail_responses_transformation.py +++ b/tests/unit/llms/sail/responses/test_sail_responses_transformation.py @@ -16,31 +16,47 @@ def responses_route(respx_mock: respx.MockRouter) -> respx.Route: @pytest.mark.parametrize( - ("service_tier", "metadata", "wire_metadata", "column_suffix"), + ("service_tier", "metadata", "background", "wire_metadata", "column_suffix"), [ - pytest.param(None, None, None, "", id="no-tier"), - pytest.param("auto", None, None, "", id="auto"), - pytest.param("default", None, {"completion_window": "asap"}, "", id="default"), - pytest.param("priority", None, {"completion_window": "asap"}, "", id="priority"), - pytest.param("flex", None, {"completion_window": "flex"}, "_flex", id="flex"), - pytest.param("balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced"), - pytest.param("Balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"), + pytest.param(None, None, None, None, "", id="no-tier"), + pytest.param("auto", None, None, None, "", id="auto"), + pytest.param("default", None, None, {"completion_window": "asap"}, "", id="default"), + pytest.param("priority", None, None, {"completion_window": "asap"}, "", id="priority"), + pytest.param("flex", None, True, {"completion_window": "flex"}, "_flex", id="flex-background"), + pytest.param("balanced", None, None, {"completion_window": "balanced"}, "_balanced", id="balanced"), + pytest.param("balanced", None, True, {"completion_window": "balanced"}, "_balanced", id="balanced-background"), + pytest.param("Balanced", None, None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"), pytest.param( - "flex", {"user_tag": "a"}, {"user_tag": "a", "completion_window": "flex"}, "_flex", id="tier-keeps-metadata" + "flex", + {"user_tag": "a"}, + True, + {"user_tag": "a", "completion_window": "flex"}, + "_flex", + id="tier-keeps-metadata", + ), + pytest.param( + None, {"completion_window": "flex"}, True, {"completion_window": "flex"}, "_flex", id="caller-window" ), - pytest.param(None, {"completion_window": "flex"}, {"completion_window": "flex"}, "_flex", id="caller-window"), pytest.param( None, {"completion_window": "standard"}, + None, {"completion_window": "standard"}, "_balanced", id="standard-window", ), - pytest.param(None, {"completion_window": "FLEX"}, {"completion_window": "flex"}, "_flex", id="window-any-case"), pytest.param( - "priority", {"completion_window": "asap"}, {"completion_window": "asap"}, "", id="agreeing-tier-and-window" + None, {"completion_window": "FLEX"}, True, {"completion_window": "flex"}, "_flex", id="window-any-case" ), - pytest.param(None, {"user_tag": "a"}, {"user_tag": "a"}, "", id="metadata-without-window"), + pytest.param( + "priority", + {"completion_window": "asap"}, + False, + {"completion_window": "asap"}, + "", + id="agreeing-tier-and-window", + ), + pytest.param(None, {"user_tag": "a"}, None, {"user_tag": "a"}, "", id="metadata-without-window"), ], ) @pytest.mark.asyncio @@ -50,6 +66,7 @@ async def test_sail_responses_send_the_window_and_bill_its_price_columns( spend_capture: SpendCapture, service_tier: str | None, metadata: dict[str, str] | None, + background: bool | None, wire_metadata: dict[str, str] | None, column_suffix: str, ) -> None: @@ -58,22 +75,36 @@ async def test_sail_responses_send_the_window_and_bill_its_price_columns( input=INPUT, service_tier=service_tier, metadata=metadata, + background=background, litellm_call_id=spend_capture.call_id, ) body: Final = sent_body(responses_route) assert "service_tier" not in body assert body.get("metadata") == wire_metadata + assert body.get("background") == background assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) +FLEX_NEEDS_BACKGROUND: Final = "only to Responses requests with background=true" +ASAP_CANNOT_RUN_IN_BACKGROUND: Final = "does not serve .* to Responses requests with background=true" + + @pytest.mark.parametrize( - ("service_tier", "metadata", "message"), + ("service_tier", "metadata", "background", "message"), [ - pytest.param("scale", None, "service_tier='scale'", id="unknown-tier"), - pytest.param(5, None, "service_tier=5", id="non-string-tier"), - pytest.param(None, {"completion_window": "soon"}, "completion_window='soon'", id="unknown-window"), - pytest.param("flex", {"completion_window": "asap"}, "select different completion windows", id="conflict"), + pytest.param("scale", None, None, "service_tier='scale'", id="unknown-tier"), + pytest.param(5, None, None, "service_tier=5", id="non-string-tier"), + pytest.param(None, {"completion_window": "soon"}, None, "completion_window='soon'", id="unknown-window"), + pytest.param("flex", {"completion_window": "asap"}, None, "select different completion windows", id="conflict"), + pytest.param("flex", None, None, FLEX_NEEDS_BACKGROUND, id="flex-tier-in-foreground"), + pytest.param("FLEX", None, False, FLEX_NEEDS_BACKGROUND, id="flex-tier-with-background-false"), + pytest.param(None, {"completion_window": "flex"}, None, FLEX_NEEDS_BACKGROUND, id="flex-window-in-foreground"), + pytest.param("priority", None, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="priority-tier-in-background"), + pytest.param("default", None, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="default-tier-in-background"), + pytest.param( + None, {"completion_window": "asap"}, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="asap-window-in-background" + ), ], ) @pytest.mark.asyncio @@ -82,14 +113,51 @@ async def test_sail_responses_reject_before_sending( responses_route: respx.Route, service_tier: object, metadata: dict[str, str] | None, + background: bool | None, message: str, ) -> None: with pytest.raises(litellm.UnsupportedParamsError, match=message): - await litellm.aresponses(model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata) + await litellm.aresponses( + model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata, background=background + ) assert not responses_route.called +@pytest.mark.parametrize( + ("service_tier", "metadata", "background", "wire_metadata"), + [ + pytest.param("flex", None, None, None, id="flex-tier-in-foreground"), + pytest.param(None, {"completion_window": "flex", "user_tag": "a"}, None, {"user_tag": "a"}, id="flex-window"), + pytest.param("priority", None, True, None, id="priority-tier-in-background"), + ], +) +@pytest.mark.asyncio +async def test_sail_responses_drop_a_window_sail_cannot_serve_under_drop_params_and_bill_base( + sail_env: None, + responses_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + metadata: dict[str, str] | None, + background: bool | None, + wire_metadata: dict[str, str] | None, +) -> None: + await litellm.aresponses( + model=MODEL, + input=INPUT, + service_tier=service_tier, + metadata=metadata, + background=background, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(responses_route) + assert "service_tier" not in body + assert body.get("metadata") == wire_metadata + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + @pytest.mark.asyncio async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params( sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture @@ -112,7 +180,7 @@ async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params( @pytest.mark.parametrize( ("service_tier", "wire_metadata", "column_suffix"), [ - pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"), + pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"), pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), ], )