From 9f60d01990fa1ac79d3f544c7b3f2c3b683dfd39 Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Wed, 14 Oct 2026 13:50:09 -0700 Subject: [PATCH 1/7] fix(sail): reject completion windows Sail does not serve for the request Sail serves the flex window only to Responses requests with background=true, and the asap window only to requests without it. It answers every other pairing with a 400. The Sail config still mapped service_tier "flex" to the flex window on Chat Completions and on foreground Responses, so those calls reached Sail and failed, even under drop_params. Chat Completions now rejects a tier that selects flex before calling Sail. drop_params does not drop it, because chat billing reads the caller's service_tier and would bill flex for a request Sail runs at asap. Responses rejects flex without background=true and asap with it, from either service_tier or metadata.completion_window. Under drop_params the window is dropped and the request bills at the base price. Co-Authored-By: Claude Opus 5.5 --- litellm/llms/sail/chat/transformation.py | 4 +- litellm/llms/sail/common_utils.py | 58 +++++++++- .../chat/test_sail_chat_transformation.py | 30 ++++- .../test_sail_responses_transformation.py | 108 ++++++++++++++---- 4 files changed, 170 insertions(+), 30 deletions(-) diff --git a/litellm/llms/sail/chat/transformation.py b/litellm/llms/sail/chat/transformation.py index 64f69af06fa..b2d39f24a57 100644 --- a/litellm/llms/sail/chat/transformation.py +++ b/litellm/llms/sail/chat/transformation.py @@ -3,8 +3,8 @@ from typing import Final from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.sail.common_utils import ( + chat_completion_window, chat_request_for_sail, - completion_window_for_service_tier, extra_body_for_sail, json_body, ) @@ -31,7 +31,7 @@ class SailChatConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: # mutable-ok: return type fixed by the base interface - completion_window_for_service_tier(non_default_params.get("service_tier"), model=model, drop_params=drop_params) + chat_completion_window(non_default_params.get("service_tier"), model=model, drop_params=drop_params) return super().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, diff --git a/litellm/llms/sail/common_utils.py b/litellm/llms/sail/common_utils.py index cb00ed14214..e6e4f6448ca 100644 --- a/litellm/llms/sail/common_utils.py +++ b/litellm/llms/sail/common_utils.py @@ -70,6 +70,52 @@ def completion_window_for_service_tier( ) +def _unserved_pairing(window: str | None, *, background: bool, source: str) -> str | None: + if window == "flex" and not background: + return ( + f"sail serves {source} only to Responses requests with background=true. " + "Send 'balanced' for a synchronous request." + ) + if window == "asap" and background: + return ( + f"sail does not serve {source} to Responses requests with background=true. " + "Send 'balanced' or 'flex' for a background request." + ) + return None + + +def _window_sail_serves( + window: str | None, *, background: bool, source: str, model: str, drop_params: bool | None +) -> str | None: + """Sail serves ``flex`` only to Responses requests with ``background=true`` + and ``asap`` only to requests without it, and answers the other pairings + with a 400, so they are rejected before the call. ``drop_params=None`` + means the pairing cannot be dropped.""" + message: Final = _unserved_pairing(window, background=background, source=source) + if message is None: + return window + if drop_params is None: + raise litellm.UnsupportedParamsError(message=message, llm_provider=SAIL, model=model) + if _dropping(drop_params): + return None + raise _unsupported(message, model) + + +def _tier_source(service_tier: object, window: str | None) -> str: + return f"service_tier={service_tier!r} (completion_window {window!r})" + + +def chat_completion_window(service_tier: object, *, model: str, drop_params: bool) -> str | None: + """Chat Completions requests are always synchronous, so a tier that selects + ``flex`` is rejected. Chat billing reads the caller's ``service_tier``, not + the mapped params, so dropping it would bill flex for a request Sail runs + at asap, and ``drop_params`` does not drop it.""" + window: Final = completion_window_for_service_tier(service_tier, model=model, drop_params=drop_params) + return _window_sail_serves( + window, background=False, source=_tier_source(service_tier, window), model=model, drop_params=None + ) + + def _metadata_without_caller_window( metadata: object, *, field: str, model: str, drop_params: bool ) -> Mapping[str, object]: @@ -107,7 +153,7 @@ def extra_body_for_sail( def chat_request_for_sail(request: Mapping[str, object], *, model: str, drop_params: bool) -> Mapping[str, object]: raw_tier: Final = request.get("service_tier") - window: Final = completion_window_for_service_tier(raw_tier, model=model, drop_params=drop_params) + window: Final = chat_completion_window(raw_tier, model=model, drop_params=drop_params) caller_metadata: Final = _metadata_without_caller_window( request.get("metadata"), field="metadata", model=model, drop_params=drop_params ) @@ -162,7 +208,15 @@ def responses_params_with_completion_window( "select different completion windows. Send one of them.", model, ) - window: Final = caller_window or tier_window + window: Final = _window_sail_serves( + caller_window or tier_window, + background=params.get("background") is True, + source=( + f"metadata.completion_window={caller_window!r}" if caller_window else _tier_source(raw_tier, tier_window) + ), + model=model, + drop_params=drop_params, + ) other_metadata: Final = without_keys(metadata, frozenset({"completion_window"})) wire_metadata: Final = ( MappingProxyType({**other_metadata, "completion_window": window}) if window else other_metadata diff --git a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py index c7b1a77343a..e61f6382040 100644 --- a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py +++ b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py @@ -21,9 +21,8 @@ TIER_CASES: Final = [ pytest.param("auto", None, "", id="auto"), pytest.param("default", "asap", "", id="default"), pytest.param("priority", "asap", "", id="priority"), - pytest.param("flex", "flex", "_flex", id="flex"), pytest.param("balanced", "balanced", "_balanced", id="balanced"), - pytest.param("FLEX", "flex", "_flex", id="flex-any-case"), + pytest.param("BALANCED", "balanced", "_balanced", id="balanced-any-case"), ] @@ -110,6 +109,25 @@ async def test_sail_chat_rejects_a_tier_with_no_window_before_sending( assert not chat_route.called +@pytest.mark.parametrize("service_tier", ["flex", "FLEX"]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_flex_tier_before_sending_even_under_drop_params( + sail_env: None, chat_route: respx.Route, service_tier: str, stream: bool, drop_params: bool +) -> None: + with pytest.raises( + litellm.UnsupportedParamsError, match="only to Responses requests with background=true" + ) as error: + await litellm.acompletion( + model=MODEL, messages=MESSAGES, service_tier=service_tier, stream=stream, drop_params=drop_params + ) + + assert error.value.status_code == 400 + assert "drop_params" not in str(error.value) + assert not chat_route.called + + @pytest.mark.parametrize("service_tier", ["bogus", "scale", 5]) @pytest.mark.parametrize(("global_drop", "request_drop"), [(False, True), (True, False)]) @pytest.mark.asyncio @@ -145,7 +163,7 @@ def chat_http_path(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPat @pytest.mark.parametrize( ("service_tier", "wire_metadata", "column_suffix"), [ - pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"), + pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"), pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), ], ) @@ -281,13 +299,13 @@ async def test_sail_chat_keeps_the_window_when_preview_features_forward_caller_m await litellm.acompletion( model=MODEL, messages=MESSAGES, - service_tier="flex", + service_tier="balanced", metadata={"requester_metadata": {"trace_id": "t-1"}}, litellm_call_id=spend_capture.call_id, ) - assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "flex"} - assert await spend_capture.settled_cost() == pytest.approx(cost_at("_flex")) + assert sent_body(chat_route)["metadata"] == {"trace_id": "t-1", "completion_window": "balanced"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) @pytest.mark.parametrize( diff --git a/tests/unit/llms/sail/responses/test_sail_responses_transformation.py b/tests/unit/llms/sail/responses/test_sail_responses_transformation.py index 3384b79bdec..cc687e74b57 100644 --- a/tests/unit/llms/sail/responses/test_sail_responses_transformation.py +++ b/tests/unit/llms/sail/responses/test_sail_responses_transformation.py @@ -16,31 +16,47 @@ def responses_route(respx_mock: respx.MockRouter) -> respx.Route: @pytest.mark.parametrize( - ("service_tier", "metadata", "wire_metadata", "column_suffix"), + ("service_tier", "metadata", "background", "wire_metadata", "column_suffix"), [ - pytest.param(None, None, None, "", id="no-tier"), - pytest.param("auto", None, None, "", id="auto"), - pytest.param("default", None, {"completion_window": "asap"}, "", id="default"), - pytest.param("priority", None, {"completion_window": "asap"}, "", id="priority"), - pytest.param("flex", None, {"completion_window": "flex"}, "_flex", id="flex"), - pytest.param("balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced"), - pytest.param("Balanced", None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"), + pytest.param(None, None, None, None, "", id="no-tier"), + pytest.param("auto", None, None, None, "", id="auto"), + pytest.param("default", None, None, {"completion_window": "asap"}, "", id="default"), + pytest.param("priority", None, None, {"completion_window": "asap"}, "", id="priority"), + pytest.param("flex", None, True, {"completion_window": "flex"}, "_flex", id="flex-background"), + pytest.param("balanced", None, None, {"completion_window": "balanced"}, "_balanced", id="balanced"), + pytest.param("balanced", None, True, {"completion_window": "balanced"}, "_balanced", id="balanced-background"), + pytest.param("Balanced", None, None, {"completion_window": "balanced"}, "_balanced", id="balanced-any-case"), pytest.param( - "flex", {"user_tag": "a"}, {"user_tag": "a", "completion_window": "flex"}, "_flex", id="tier-keeps-metadata" + "flex", + {"user_tag": "a"}, + True, + {"user_tag": "a", "completion_window": "flex"}, + "_flex", + id="tier-keeps-metadata", + ), + pytest.param( + None, {"completion_window": "flex"}, True, {"completion_window": "flex"}, "_flex", id="caller-window" ), - pytest.param(None, {"completion_window": "flex"}, {"completion_window": "flex"}, "_flex", id="caller-window"), pytest.param( None, {"completion_window": "standard"}, + None, {"completion_window": "standard"}, "_balanced", id="standard-window", ), - pytest.param(None, {"completion_window": "FLEX"}, {"completion_window": "flex"}, "_flex", id="window-any-case"), pytest.param( - "priority", {"completion_window": "asap"}, {"completion_window": "asap"}, "", id="agreeing-tier-and-window" + None, {"completion_window": "FLEX"}, True, {"completion_window": "flex"}, "_flex", id="window-any-case" ), - pytest.param(None, {"user_tag": "a"}, {"user_tag": "a"}, "", id="metadata-without-window"), + pytest.param( + "priority", + {"completion_window": "asap"}, + False, + {"completion_window": "asap"}, + "", + id="agreeing-tier-and-window", + ), + pytest.param(None, {"user_tag": "a"}, None, {"user_tag": "a"}, "", id="metadata-without-window"), ], ) @pytest.mark.asyncio @@ -50,6 +66,7 @@ async def test_sail_responses_send_the_window_and_bill_its_price_columns( spend_capture: SpendCapture, service_tier: str | None, metadata: dict[str, str] | None, + background: bool | None, wire_metadata: dict[str, str] | None, column_suffix: str, ) -> None: @@ -58,22 +75,36 @@ async def test_sail_responses_send_the_window_and_bill_its_price_columns( input=INPUT, service_tier=service_tier, metadata=metadata, + background=background, litellm_call_id=spend_capture.call_id, ) body: Final = sent_body(responses_route) assert "service_tier" not in body assert body.get("metadata") == wire_metadata + assert body.get("background") == background assert await spend_capture.settled_cost() == pytest.approx(cost_at(column_suffix)) +FLEX_NEEDS_BACKGROUND: Final = "only to Responses requests with background=true" +ASAP_CANNOT_RUN_IN_BACKGROUND: Final = "does not serve .* to Responses requests with background=true" + + @pytest.mark.parametrize( - ("service_tier", "metadata", "message"), + ("service_tier", "metadata", "background", "message"), [ - pytest.param("scale", None, "service_tier='scale'", id="unknown-tier"), - pytest.param(5, None, "service_tier=5", id="non-string-tier"), - pytest.param(None, {"completion_window": "soon"}, "completion_window='soon'", id="unknown-window"), - pytest.param("flex", {"completion_window": "asap"}, "select different completion windows", id="conflict"), + pytest.param("scale", None, None, "service_tier='scale'", id="unknown-tier"), + pytest.param(5, None, None, "service_tier=5", id="non-string-tier"), + pytest.param(None, {"completion_window": "soon"}, None, "completion_window='soon'", id="unknown-window"), + pytest.param("flex", {"completion_window": "asap"}, None, "select different completion windows", id="conflict"), + pytest.param("flex", None, None, FLEX_NEEDS_BACKGROUND, id="flex-tier-in-foreground"), + pytest.param("FLEX", None, False, FLEX_NEEDS_BACKGROUND, id="flex-tier-with-background-false"), + pytest.param(None, {"completion_window": "flex"}, None, FLEX_NEEDS_BACKGROUND, id="flex-window-in-foreground"), + pytest.param("priority", None, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="priority-tier-in-background"), + pytest.param("default", None, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="default-tier-in-background"), + pytest.param( + None, {"completion_window": "asap"}, True, ASAP_CANNOT_RUN_IN_BACKGROUND, id="asap-window-in-background" + ), ], ) @pytest.mark.asyncio @@ -82,14 +113,51 @@ async def test_sail_responses_reject_before_sending( responses_route: respx.Route, service_tier: object, metadata: dict[str, str] | None, + background: bool | None, message: str, ) -> None: with pytest.raises(litellm.UnsupportedParamsError, match=message): - await litellm.aresponses(model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata) + await litellm.aresponses( + model=MODEL, input=INPUT, service_tier=service_tier, metadata=metadata, background=background + ) assert not responses_route.called +@pytest.mark.parametrize( + ("service_tier", "metadata", "background", "wire_metadata"), + [ + pytest.param("flex", None, None, None, id="flex-tier-in-foreground"), + pytest.param(None, {"completion_window": "flex", "user_tag": "a"}, None, {"user_tag": "a"}, id="flex-window"), + pytest.param("priority", None, True, None, id="priority-tier-in-background"), + ], +) +@pytest.mark.asyncio +async def test_sail_responses_drop_a_window_sail_cannot_serve_under_drop_params_and_bill_base( + sail_env: None, + responses_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + metadata: dict[str, str] | None, + background: bool | None, + wire_metadata: dict[str, str] | None, +) -> None: + await litellm.aresponses( + model=MODEL, + input=INPUT, + service_tier=service_tier, + metadata=metadata, + background=background, + drop_params=True, + litellm_call_id=spend_capture.call_id, + ) + + body: Final = sent_body(responses_route) + assert "service_tier" not in body + assert body.get("metadata") == wire_metadata + assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + @pytest.mark.asyncio async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params( sail_env: None, responses_route: respx.Route, spend_capture: SpendCapture @@ -112,7 +180,7 @@ async def test_sail_responses_drop_an_unknown_tier_and_window_under_drop_params( @pytest.mark.parametrize( ("service_tier", "wire_metadata", "column_suffix"), [ - pytest.param("flex", {"trace_id": "t-1", "completion_window": "flex"}, "_flex", id="flex"), + pytest.param("balanced", {"trace_id": "t-1", "completion_window": "balanced"}, "_balanced", id="balanced"), pytest.param(None, {"trace_id": "t-1"}, "", id="no-tier"), ], ) From 9ffba202fc40456608b7ebc863930b1c1a31128e Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Thu, 15 Oct 2026 11:29:15 -0700 Subject: [PATCH 2/7] fix(sail): retain the billed service tier on streamed messages --- .../streaming_chunk_builder_utils.py | 1 + .../test_sail_messages_transformation.py | 46 ++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 2ceb0042f1f..ab07b7e3f15 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1183,6 +1183,7 @@ class ChunkProcessor: for field, value in ( ("inference_geo", calculated_usage_per_chunk["inference_geo"]), ("speed", calculated_usage_per_chunk["speed"]), + ("service_tier", self._last_provider_pricing_field(chunks, "service_tier")), ) if value is not None } diff --git a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py index c6e74534791..5069db92484 100644 --- a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py +++ b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py @@ -1,3 +1,4 @@ +import json from typing import Final import httpx @@ -5,7 +6,17 @@ import pytest import respx import litellm -from tests.unit.llms.sail.helpers import MODEL, SAIL_API_BASE, SpendCapture, cost_at, messages_body, sent_body +from tests.unit.llms.sail.helpers import ( + CACHED_TOKENS, + COMPLETION_TOKENS, + MODEL, + PROMPT_TOKENS, + SAIL_API_BASE, + SpendCapture, + cost_at, + messages_body, + sent_body, +) MESSAGES: Final = [{"role": "user", "content": "hi"}] @@ -29,3 +40,36 @@ async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier( assert "service_tier" not in body assert "completion_window" not in (body.get("metadata") or {}) assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + + +@pytest.mark.asyncio +async def test_sail_streamed_messages_bill_the_reported_window( + sail_env: None, respx_mock: respx.MockRouter, spend_capture: SpendCapture +) -> None: + body: Final = messages_body() + usage: Final = { + "input_tokens": PROMPT_TOKENS - CACHED_TOKENS, + "cache_read_input_tokens": CACHED_TOKENS, + "output_tokens": COMPLETION_TOKENS, + "service_tier": "balanced", + } + events: Final = ( + {"type": "message_start", "message": {**body, "content": [], "usage": {**usage, "output_tokens": 0}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": usage}, + {"type": "message_stop"}, + ) + stream: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + respx_mock.post(f"{SAIL_API_BASE}/messages").mock( + return_value=httpx.Response(200, content=stream, headers={"content-type": "text/event-stream"}) + ) + + response: Final = await litellm.anthropic_messages( + model=MODEL, messages=MESSAGES, max_tokens=16, stream=True, litellm_call_id=spend_capture.call_id + ) + chunks: Final = tuple([chunk async for chunk in response]) + + assert chunks + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) From 80c33af6c47918e84255d9788f0b26fdb571162f Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Thu, 15 Oct 2026 15:35:56 -0700 Subject: [PATCH 3/7] fix(sail): forward caller windows on chat and messages --- litellm/cost_calculator.py | 9 ++- litellm/llms/anthropic/chat/transformation.py | 2 +- .../pass_through/messages/handler.py | 14 ++-- .../anthropic_messages/transformation.py | 8 ++- litellm/llms/sail/chat/transformation.py | 10 +-- litellm/llms/sail/common_utils.py | 48 +++++++------- litellm/llms/sail/messages/__init__.py | 0 litellm/llms/sail/messages/transformation.py | 38 +++++++++++ litellm/llms/sail/responses/transformation.py | 4 +- litellm/main.py | 6 ++ litellm/utils.py | 4 ++ .../chat/test_sail_chat_transformation.py | 49 ++++++++++++++ .../test_sail_messages_transformation.py | 64 +++++++++++++++++-- 13 files changed, 209 insertions(+), 47 deletions(-) create mode 100644 litellm/llms/sail/messages/__init__.py create mode 100644 litellm/llms/sail/messages/transformation.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 43ce71af7b8..b5aa9c17633 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1517,8 +1517,15 @@ def completion_cost( if explicit_tier is not None: service_tier = explicit_tier else: + requested_tier: Final[object] + if custom_llm_provider == "sail" and optional_params is not None: + from litellm.llms.sail.common_utils import billed_service_tier + + requested_tier = billed_service_tier(optional_params) + else: + requested_tier = optional_params.get("service_tier") if optional_params is not None else None service_tier = _resolve_billable_service_tier( # rebind-ok: resolved from request then response - requested=optional_params.get("service_tier") if optional_params is not None else None, + requested=requested_tier, served=_served_service_tier(completion_response, cost_per_token_usage_object), ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index f9cb1cbf091..b8bbeb12665 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -388,7 +388,7 @@ DROP_UNSUPPORTED_SERVICE_TIER_WARNING: Final = ( "Anthropic only accepts 'auto' and 'standard_only'." ) -MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle"} +MESSAGES_SERVICE_TIER_PROVIDERS: Final = SERVICE_TIER_PROVIDERS | {"bedrock_mantle", "sail"} DROP_SERVICE_TIER_PROVIDER_WARNING: Final = ( "Dropping `service_tier` for model=%s on provider=%s. Only the Anthropic API routes accept it." diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 00bf1774dd0..c7c93240ccd 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -509,9 +509,8 @@ def anthropic_messages_handler( presanitized=bool(kwargs.pop("_litellm_messages_presanitized", False)), ) - metadata = validate_anthropic_api_metadata(metadata) - local_vars: Final = locals() + anthropic_metadata: Final = validate_anthropic_api_metadata(metadata) is_async: Final = kwargs.pop("is_async", False) # Use provided client or create a new one litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") @@ -580,7 +579,7 @@ def anthropic_messages_handler( max_tokens=max_tokens, messages=messages, model=model, - metadata=metadata, + metadata=anthropic_metadata, stop_sequences=stop_sequences, stream=stream, system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower @@ -612,7 +611,7 @@ def anthropic_messages_handler( max_tokens=max_tokens, messages=messages, model=original_model, - metadata=metadata, + metadata=anthropic_metadata, stop_sequences=stop_sequences, stream=stream, system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower @@ -642,7 +641,7 @@ def anthropic_messages_handler( max_tokens=max_tokens, messages=messages, model=original_model, - metadata=metadata, + metadata=anthropic_metadata, stop_sequences=stop_sequences, stream=stream, system=cast( # cast-ok: anthropic accepts list system blocks; handler signature is narrower @@ -667,10 +666,11 @@ def anthropic_messages_handler( f"custom_llm_provider is required for Anthropic messages, passed in model={model}, custom_llm_provider={custom_llm_provider}" ) - local_vars.update(kwargs) + request_metadata: Final = anthropic_messages_provider_config.request_metadata(metadata) + request_params: Final = {**local_vars, **kwargs, "metadata": request_metadata} anthropic_messages_optional_request_params: Final = ( AnthropicMessagesRequestUtils.get_requested_anthropic_messages_optional_param( - params=local_vars, + params=request_params, model=model, drop_params=litellm_params.get("drop_params") is True, custom_llm_provider=custom_llm_provider, diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index bd6c53a0453..68c1315b225 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -1,9 +1,10 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -19,6 +20,11 @@ else: class BaseAnthropicMessagesConfig(ABC): + def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None: + if metadata is None: + return None + return AnthropicMetadata.model_validate(metadata).model_dump(exclude_none=True) + @abstractmethod def validate_anthropic_messages_environment( # use different name because return type is different from base config's validate_environment self, diff --git a/litellm/llms/sail/chat/transformation.py b/litellm/llms/sail/chat/transformation.py index b2d39f24a57..6eb88e820cd 100644 --- a/litellm/llms/sail/chat/transformation.py +++ b/litellm/llms/sail/chat/transformation.py @@ -3,17 +3,17 @@ from typing import Final from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.sail.common_utils import ( - chat_completion_window, chat_request_for_sail, extra_body_for_sail, json_body, + params_with_completion_window, ) from litellm.types.llms.openai import AllMessageValues _REJECTED_BY_SAIL: Final = frozenset( {"stop", "seed", "frequency_penalty", "presence_penalty", "logit_bias", "logprobs", "top_logprobs"} ) -_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user") +_ACCEPTED_BY_SAIL: Final = ("reasoning_effort", "user", "metadata") class SailChatConfig(OpenAIGPTConfig): @@ -31,13 +31,15 @@ class SailChatConfig(OpenAIGPTConfig): model: str, drop_params: bool, ) -> dict: # mutable-ok: return type fixed by the base interface - chat_completion_window(non_default_params.get("service_tier"), model=model, drop_params=drop_params) - return super().map_openai_params( + mapped: Final = super().map_openai_params( non_default_params=non_default_params, optional_params=optional_params, model=model, drop_params=drop_params, ) + return json_body( + params_with_completion_window(mapped, model=model, drop_params=drop_params, allow_drop_window=False) + ) def transform_request( self, diff --git a/litellm/llms/sail/common_utils.py b/litellm/llms/sail/common_utils.py index e6e4f6448ca..2a7e3c96ef7 100644 --- a/litellm/llms/sail/common_utils.py +++ b/litellm/llms/sail/common_utils.py @@ -105,17 +105,6 @@ def _tier_source(service_tier: object, window: str | None) -> str: return f"service_tier={service_tier!r} (completion_window {window!r})" -def chat_completion_window(service_tier: object, *, model: str, drop_params: bool) -> str | None: - """Chat Completions requests are always synchronous, so a tier that selects - ``flex`` is rejected. Chat billing reads the caller's ``service_tier``, not - the mapped params, so dropping it would bill flex for a request Sail runs - at asap, and ``drop_params`` does not drop it.""" - window: Final = completion_window_for_service_tier(service_tier, model=model, drop_params=drop_params) - return _window_sail_serves( - window, background=False, source=_tier_source(service_tier, window), model=model, drop_params=None - ) - - def _metadata_without_caller_window( metadata: object, *, field: str, model: str, drop_params: bool ) -> Mapping[str, object]: @@ -152,19 +141,18 @@ def extra_body_for_sail( def chat_request_for_sail(request: Mapping[str, object], *, model: str, drop_params: bool) -> Mapping[str, object]: - raw_tier: Final = request.get("service_tier") - window: Final = chat_completion_window(raw_tier, model=model, drop_params=drop_params) - caller_metadata: Final = _metadata_without_caller_window( - request.get("metadata"), field="metadata", model=model, drop_params=drop_params + normalized: Final = params_with_completion_window( + request, model=model, drop_params=drop_params, allow_drop_window=False ) - metadata: Final = MappingProxyType({**caller_metadata, "completion_window": window}) if window else caller_metadata - extra_body: Final = request.get("extra_body") + extra_body: Final = normalized.get("extra_body") return MappingProxyType( { - **without_keys(request, frozenset({"service_tier", "metadata", "extra_body"})), - **(_entry("metadata", metadata) if metadata else _EMPTY), + **without_keys(normalized, frozenset({"service_tier", "extra_body"})), **( - _entry("extra_body", extra_body_for_sail(extra_body, metadata, model=model, drop_params=drop_params)) + _entry( + "extra_body", + extra_body_for_sail(extra_body, normalized.get("metadata"), model=model, drop_params=drop_params), + ) if isinstance(extra_body, Mapping) else _EMPTY ), @@ -184,11 +172,9 @@ def _caller_completion_window(window: object, *, model: str, drop_params: bool) ) -def responses_params_with_completion_window( - params: Mapping[str, object], *, model: str, drop_params: bool +def params_with_completion_window( + params: Mapping[str, object], *, model: str, drop_params: bool, allow_drop_window: bool = True ) -> Mapping[str, object]: - """Responses billing reads these mapped params, so ``service_tier`` is kept - as the tier whose price columns match the window and stripped from the body later.""" raw_tier: Final = params.get("service_tier") raw_metadata: Final = params.get("metadata") metadata: Final[Mapping[str, object]] = raw_metadata if isinstance(raw_metadata, Mapping) else _EMPTY @@ -210,12 +196,12 @@ def responses_params_with_completion_window( ) window: Final = _window_sail_serves( caller_window or tier_window, - background=params.get("background") is True, + background=allow_drop_window and params.get("background") is True, source=( f"metadata.completion_window={caller_window!r}" if caller_window else _tier_source(raw_tier, tier_window) ), model=model, - drop_params=drop_params, + drop_params=drop_params if allow_drop_window else None, ) other_metadata: Final = without_keys(metadata, frozenset({"completion_window"})) wire_metadata: Final = ( @@ -229,3 +215,13 @@ def responses_params_with_completion_window( **(_entry("service_tier", billed_tier) if billed_tier else _EMPTY), } ) + + +def billed_service_tier(params: Mapping[str, object]) -> str | None: + metadata: Final = params.get("metadata") + if isinstance(metadata, Mapping): + window: Final = metadata.get("completion_window") + if isinstance(window, str) and window.lower() in _BILLED_TIER_FOR_WINDOW: + return _BILLED_TIER_FOR_WINDOW[window.lower()] + service_tier: Final = params.get("service_tier") + return service_tier if isinstance(service_tier, str) else None diff --git a/litellm/llms/sail/messages/__init__.py b/litellm/llms/sail/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/sail/messages/transformation.py b/litellm/llms/sail/messages/transformation.py new file mode 100644 index 00000000000..d5eee2fd61c --- /dev/null +++ b/litellm/llms/sail/messages/transformation.py @@ -0,0 +1,38 @@ +from collections.abc import Mapping +from typing import Final + +from litellm.llms.openai_like.messages.transformation import JSONProviderAnthropicMessagesConfig +from litellm.llms.sail.common_utils import json_body, params_with_completion_window, without_keys +from litellm.types.router import GenericLiteLLMParams + + +class SailAnthropicMessagesConfig(JSONProviderAnthropicMessagesConfig): + def get_supported_anthropic_messages_params(self, model: str) -> list: # mutable-ok: matches the provider interface + return [*super().get_supported_anthropic_messages_params(model), "metadata"] + + def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None: + if metadata is None: + return None + return {key: value for key, value in metadata.items() if isinstance(value, str)} or None + + def transform_anthropic_messages_request( + self, + model: str, + messages: list[dict], # mutable-ok: matches the native Messages provider interface + anthropic_messages_optional_request_params: dict, # mutable-ok: matches the provider interface + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: matches the native Messages provider interface + ) -> dict: # mutable-ok: HTTP bodies and the provider interface require plain dicts + normalized: Final = params_with_completion_window( + anthropic_messages_optional_request_params, + model=model, + drop_params=bool(litellm_params.drop_params), + allow_drop_window=False, + ) + return super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=json_body(without_keys(normalized, frozenset({"service_tier"}))), + litellm_params=litellm_params, + headers=headers, + ) diff --git a/litellm/llms/sail/responses/transformation.py b/litellm/llms/sail/responses/transformation.py index c3c39a125f5..9dd86569084 100644 --- a/litellm/llms/sail/responses/transformation.py +++ b/litellm/llms/sail/responses/transformation.py @@ -5,7 +5,7 @@ from litellm.llms.openai_like.dynamic_config import create_responses_config_clas from litellm.llms.sail.common_utils import ( extra_body_for_sail, json_body, - responses_params_with_completion_window, + params_with_completion_window, sail_provider_config, without_keys, ) @@ -20,7 +20,7 @@ class SailResponsesAPIConfig(create_responses_config_class(sail_provider_config( model: str, drop_params: bool, ) -> dict: # mutable-ok: return type fixed by the base interface - params: Final = responses_params_with_completion_window( + params: Final = params_with_completion_window( super().map_openai_params( response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params ), diff --git a/litellm/main.py b/litellm/main.py index d8e62ebcdad..313e769e4d9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -161,6 +161,7 @@ from litellm.utils import ( TextCompletionStreamWrapper, TranscriptionResponse, Usage, + add_openai_metadata, add_provider_specific_params_to_optional_params, async_mock_completion_streaming_obj, convert_to_model_response_object, @@ -5634,6 +5635,10 @@ def completion( if bridges_to_responses_api else kwargs.get("allowed_openai_params") ) + requester_metadata: Final = metadata.get("requester_metadata") if isinstance(metadata, dict) else None + caller_metadata: Final = ( + add_openai_metadata(requester_metadata) if isinstance(requester_metadata, dict) else None + ) optional_param_args: Final = { "functions": functions, "function_call": function_call, @@ -5680,6 +5685,7 @@ def completion( "prompt_cache_key": prompt_cache_key, "allowed_openai_params": allowed_openai_params, "base_model": base_model, + **({"metadata": caller_metadata} if custom_llm_provider == "sail" else {}), } optional_params = get_optional_params(**optional_param_args, **non_default_params) processed_non_default_params: Final = pre_process_non_default_params( diff --git a/litellm/utils.py b/litellm/utils.py index c0c80f09e4a..b906d2ce757 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9197,6 +9197,10 @@ class ProviderConfigManager: from litellm.llms.openai_like.json_loader import JSONProviderRegistry json_provider: Final = JSONProviderRegistry.get(provider.value) + if provider == LlmProviders.SAIL and json_provider is not None: + from litellm.llms.sail.messages.transformation import SailAnthropicMessagesConfig + + return SailAnthropicMessagesConfig(json_provider) if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints: from litellm.llms.openai_like.messages.transformation import ( JSONProviderAnthropicMessagesConfig, diff --git a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py index e61f6382040..b7138a60873 100644 --- a/tests/unit/llms/sail/chat/test_sail_chat_transformation.py +++ b/tests/unit/llms/sail/chat/test_sail_chat_transformation.py @@ -16,6 +16,55 @@ from tests.unit.llms.sail.helpers import ( ) MESSAGES: Final = [{"role": "user", "content": "hi"}] + + +@pytest.mark.asyncio +async def test_sail_chat_forwards_the_callers_window_without_proxy_metadata( + sail_env: None, chat_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + metadata={ + "requester_metadata": {"completion_window": "balanced", "trace_id": "caller-trace"}, + "user_api_key_hash": "proxy-private", + }, + litellm_call_id=spend_capture.call_id, + ) + + assert sent_body(chat_route)["metadata"] == {"completion_window": "balanced", "trace_id": "caller-trace"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_caller_metadata_flex_window( + sail_env: None, chat_route: respx.Route, drop_params: bool +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="background=true"): + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + metadata={"requester_metadata": {"completion_window": "flex"}}, + drop_params=drop_params, + ) + + assert not chat_route.called + + +@pytest.mark.asyncio +async def test_sail_chat_rejects_a_conflicting_caller_window_and_tier(sail_env: None, chat_route: respx.Route) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="different completion windows"): + await litellm.acompletion( + model=MODEL, + messages=MESSAGES, + service_tier="priority", + metadata={"requester_metadata": {"completion_window": "balanced"}}, + ) + + assert not chat_route.called + + TIER_CASES: Final = [ pytest.param(None, None, "", id="no-tier"), pytest.param("auto", None, "", id="auto"), diff --git a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py index 5069db92484..34c882f5677 100644 --- a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py +++ b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py @@ -21,15 +21,40 @@ from tests.unit.llms.sail.helpers import ( MESSAGES: Final = [{"role": "user", "content": "hi"}] +@pytest.mark.asyncio +async def test_sail_messages_forward_the_callers_window_and_bill_its_price( + sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture +) -> None: + await litellm.anthropic_messages( + model=MODEL, + messages=MESSAGES, + max_tokens=16, + metadata={"completion_window": "balanced", "trace_id": "caller-trace"}, + litellm_metadata={"user_api_key_hash": "proxy-private"}, + litellm_call_id=spend_capture.call_id, + ) + + assert sent_body(messages_route)["metadata"] == {"completion_window": "balanced", "trace_id": "caller-trace"} + assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) + + @pytest.fixture def messages_route(respx_mock: respx.MockRouter) -> respx.Route: return respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=httpx.Response(200, json=messages_body())) -@pytest.mark.parametrize("service_tier", [None, "auto", "priority", "flex", "balanced", "scale"]) +@pytest.mark.parametrize( + ("service_tier", "window", "suffix"), + [(None, None, ""), ("auto", None, ""), ("priority", "asap", ""), ("balanced", "balanced", "_balanced")], +) @pytest.mark.asyncio -async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier( - sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture, service_tier: str | None +async def test_sail_messages_send_and_bill_the_selected_window( + sail_env: None, + messages_route: respx.Route, + spend_capture: SpendCapture, + service_tier: str | None, + window: str | None, + suffix: str, ) -> None: await litellm.anthropic_messages( model=MODEL, messages=MESSAGES, max_tokens=16, service_tier=service_tier, litellm_call_id=spend_capture.call_id @@ -38,8 +63,37 @@ async def test_sail_messages_send_no_window_and_bill_asap_whatever_the_tier( body: Final = sent_body(messages_route) assert body["messages"] == MESSAGES assert "service_tier" not in body - assert "completion_window" not in (body.get("metadata") or {}) - assert await spend_capture.settled_cost() == pytest.approx(cost_at("")) + assert body.get("metadata") == ({"completion_window": window} if window else None) + assert await spend_capture.settled_cost() == pytest.approx(cost_at(suffix)) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("as_metadata", [False, True]) +@pytest.mark.asyncio +async def test_sail_messages_reject_flex_before_sending( + sail_env: None, messages_route: respx.Route, drop_params: bool, as_metadata: bool +) -> None: + params: Final = {"metadata": {"completion_window": "flex"}} if as_metadata else {"service_tier": "flex"} + with pytest.raises(litellm.UnsupportedParamsError, match="background=true"): + await litellm.anthropic_messages( + model=MODEL, messages=MESSAGES, max_tokens=16, drop_params=drop_params, **params + ) + + assert not messages_route.called + + +@pytest.mark.asyncio +async def test_sail_messages_reject_conflicting_window_and_tier(sail_env: None, messages_route: respx.Route) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="different completion windows"): + await litellm.anthropic_messages( + model=MODEL, + messages=MESSAGES, + max_tokens=16, + service_tier="priority", + metadata={"completion_window": "balanced"}, + ) + + assert not messages_route.called @pytest.mark.asyncio From b9063fdcacf47384b4a4118206d4dd43529880dc Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Thu, 15 Oct 2026 15:58:07 -0700 Subject: [PATCH 4/7] fix(sail): type completion window metadata and messages --- litellm/cost_calculator.py | 14 ++++++++------ .../pass_through/messages/transformation.py | 4 ++-- .../llms/openai_like/messages/transformation.py | 2 +- litellm/llms/sail/common_utils.py | 5 +++-- litellm/llms/sail/messages/transformation.py | 12 +++++++----- litellm/main.py | 8 +++++--- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5aa9c17633..ad497f1f00f 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1517,13 +1517,15 @@ def completion_cost( if explicit_tier is not None: service_tier = explicit_tier else: - requested_tier: Final[object] - if custom_llm_provider == "sail" and optional_params is not None: - from litellm.llms.sail.common_utils import billed_service_tier + from litellm.llms.sail.common_utils import billed_service_tier - requested_tier = billed_service_tier(optional_params) - else: - requested_tier = optional_params.get("service_tier") if optional_params is not None else None + requested_tier: Final[object] = ( + billed_service_tier(optional_params) + if custom_llm_provider == "sail" and optional_params is not None + else optional_params.get("service_tier") + if optional_params is not None + else None + ) service_tier = _resolve_billable_service_tier( # rebind-ok: resolved from request then response requested=requested_tier, served=_served_service_tier(completion_response, cost_per_token_usage_object), diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index cb96708f200..049c0a70dbe 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -80,7 +80,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _resolved_provider(self) -> str: return self.custom_llm_provider or "anthropic" - def get_supported_anthropic_messages_params(self, model: str) -> list: + def get_supported_anthropic_messages_params(self, model: str) -> list[str]: return [ "messages", "model", @@ -577,7 +577,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): anthropic_messages_optional_request_params: dict, litellm_params: GenericLiteLLMParams, headers: dict, - ) -> dict: + ) -> dict[str, object]: """ No transformation is needed for Anthropic messages diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index 2e9a300e2fd..4d610d540e9 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -73,7 +73,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): anthropic_messages_optional_request_params: dict, # mutable-ok: matches dict-typed base signature litellm_params: GenericLiteLLMParams, headers: dict, # mutable-ok: matches dict-typed base signature - ) -> dict: # mutable-ok: matches dict-typed base signature + ) -> dict[str, object]: # mutable-ok: matches dict-typed base signature """ Anthropic ignores prompt-caching hints it cannot honor, but strict non-Anthropic implementations of the Messages API 400 the whole request diff --git a/litellm/llms/sail/common_utils.py b/litellm/llms/sail/common_utils.py index 2a7e3c96ef7..f2f3e948376 100644 --- a/litellm/llms/sail/common_utils.py +++ b/litellm/llms/sail/common_utils.py @@ -3,6 +3,7 @@ from types import MappingProxyType from typing import Final, Literal, TypeAlias import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.openai_like.json_loader import JSONProviderRegistry, SimpleProviderConfig from litellm.types.utils import LlmProviders @@ -218,8 +219,8 @@ def params_with_completion_window( def billed_service_tier(params: Mapping[str, object]) -> str | None: - metadata: Final = params.get("metadata") - if isinstance(metadata, Mapping): + metadata: Final = as_str_mapping(params.get("metadata")) + if metadata is not None: window: Final = metadata.get("completion_window") if isinstance(window, str) and window.lower() in _BILLED_TIER_FOR_WINDOW: return _BILLED_TIER_FOR_WINDOW[window.lower()] diff --git a/litellm/llms/sail/messages/transformation.py b/litellm/llms/sail/messages/transformation.py index d5eee2fd61c..3e0dd9a4b24 100644 --- a/litellm/llms/sail/messages/transformation.py +++ b/litellm/llms/sail/messages/transformation.py @@ -7,7 +7,9 @@ from litellm.types.router import GenericLiteLLMParams class SailAnthropicMessagesConfig(JSONProviderAnthropicMessagesConfig): - def get_supported_anthropic_messages_params(self, model: str) -> list: # mutable-ok: matches the provider interface + def get_supported_anthropic_messages_params( + self, model: str + ) -> list[str]: # mutable-ok: matches the provider interface return [*super().get_supported_anthropic_messages_params(model), "metadata"] def request_metadata(self, metadata: Mapping[str, object] | None) -> Mapping[str, object] | None: @@ -18,11 +20,11 @@ class SailAnthropicMessagesConfig(JSONProviderAnthropicMessagesConfig): def transform_anthropic_messages_request( self, model: str, - messages: list[dict], # mutable-ok: matches the native Messages provider interface - anthropic_messages_optional_request_params: dict, # mutable-ok: matches the provider interface + messages: list[dict[str, object]], # mutable-ok: matches the native Messages provider interface + anthropic_messages_optional_request_params: dict[str, object], # mutable-ok: matches the provider interface litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: matches the native Messages provider interface - ) -> dict: # mutable-ok: HTTP bodies and the provider interface require plain dicts + headers: dict[str, str], # mutable-ok: matches the native Messages provider interface + ) -> dict[str, object]: # mutable-ok: HTTP bodies and the provider interface require plain dicts normalized: Final = params_with_completion_window( anthropic_messages_optional_request_params, model=model, diff --git a/litellm/main.py b/litellm/main.py index 313e769e4d9..49c8b7f8bd0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -77,6 +77,7 @@ from litellm.litellm_core_utils.chat_completion_agentic_loop import ( maybe_run_chat_completion_agentic_loop, ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( AWS_CREDENTIAL_KWARGS_KEYS, @@ -5635,10 +5636,11 @@ def completion( if bridges_to_responses_api else kwargs.get("allowed_openai_params") ) - requester_metadata: Final = metadata.get("requester_metadata") if isinstance(metadata, dict) else None - caller_metadata: Final = ( - add_openai_metadata(requester_metadata) if isinstance(requester_metadata, dict) else None + metadata_mapping: Final = as_str_mapping(metadata) + requester_metadata: Final = ( + as_str_mapping(metadata_mapping.get("requester_metadata")) if metadata_mapping is not None else None ) + caller_metadata: Final = add_openai_metadata(requester_metadata) optional_param_args: Final = { "functions": functions, "function_call": function_call, From 72d7d6f7b8db0074be57d0a8d7b3b4b10999a321 Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Thu, 15 Oct 2026 16:29:01 -0700 Subject: [PATCH 5/7] ci: retry intermittent MCP subscription checks From e4963076859bce98e503ee2c03306b5c1a8217b4 Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Thu, 15 Oct 2026 17:12:07 -0700 Subject: [PATCH 6/7] test(sail): cover Messages window metadata discovery --- .../sail/messages/test_sail_messages_transformation.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py index 34c882f5677..d28ce22a594 100644 --- a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py +++ b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py @@ -6,6 +6,7 @@ import pytest import respx import litellm +from litellm.utils import ProviderConfigManager from tests.unit.llms.sail.helpers import ( CACHED_TOKENS, COMPLETION_TOKENS, @@ -21,6 +22,15 @@ from tests.unit.llms.sail.helpers import ( MESSAGES: Final = [{"role": "user", "content": "hi"}] +def test_sail_messages_advertise_window_metadata_support() -> None: + config: Final = ProviderConfigManager.get_provider_anthropic_messages_config( + model=MODEL, provider=litellm.LlmProviders.SAIL + ) + + assert config is not None + assert "metadata" in config.get_supported_anthropic_messages_params(model=MODEL) + + @pytest.mark.asyncio async def test_sail_messages_forward_the_callers_window_and_bill_its_price( sail_env: None, messages_route: respx.Route, spend_capture: SpendCapture From 8c81a0a85dc98f5e948e2734cf0971d06462e4e0 Mon Sep 17 00:00:00 2001 From: Kavin Anand Date: Thu, 15 Oct 2026 18:02:32 -0700 Subject: [PATCH 7/7] fix(sail): bill Messages after parameter drops --- litellm/llms/custom_httpx/llm_http_handler.py | 27 +++-- .../test_sail_messages_transformation.py | 101 +++++++++++++++--- 2 files changed, 97 insertions(+), 31 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 610321fe1f6..2cf8779f5c6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2269,20 +2269,6 @@ class BaseLLMHTTPHandler: if explicit_vertex_location else MappingProxyType({}) ) - logging_obj.update_from_kwargs( - kwargs=kwargs, - model=model, - optional_params=dict(anthropic_messages_optional_request_params), - litellm_params={ - "preset_cache_key": None, - "stream_response": {}, - "model_info": kwargs.get("model_info"), - **vertex_location_params, - **anthropic_messages_optional_request_params, - }, - custom_llm_provider=custom_llm_provider, - ) - additional_drop_params: Final[list[str]] = litellm_params.get("additional_drop_params") or [] if additional_drop_params: from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value @@ -2300,6 +2286,19 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, headers=headers, ) + logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model, + optional_params=dict(anthropic_messages_optional_request_params), + litellm_params={ + "preset_cache_key": None, + "stream_response": {}, + "model_info": kwargs.get("model_info"), + **vertex_location_params, + **anthropic_messages_optional_request_params, + }, + custom_llm_provider=custom_llm_provider, + ) logging_obj.stream = stream logging_obj.model_call_details.update(request_body) diff --git a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py index d28ce22a594..9fac963f2d7 100644 --- a/tests/unit/llms/sail/messages/test_sail_messages_transformation.py +++ b/tests/unit/llms/sail/messages/test_sail_messages_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Mapping from typing import Final import httpx @@ -110,24 +111,10 @@ async def test_sail_messages_reject_conflicting_window_and_tier(sail_env: None, async def test_sail_streamed_messages_bill_the_reported_window( sail_env: None, respx_mock: respx.MockRouter, spend_capture: SpendCapture ) -> None: - body: Final = messages_body() - usage: Final = { - "input_tokens": PROMPT_TOKENS - CACHED_TOKENS, - "cache_read_input_tokens": CACHED_TOKENS, - "output_tokens": COMPLETION_TOKENS, - "service_tier": "balanced", - } - events: Final = ( - {"type": "message_start", "message": {**body, "content": [], "usage": {**usage, "output_tokens": 0}}}, - {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, - {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, - {"type": "content_block_stop", "index": 0}, - {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": usage}, - {"type": "message_stop"}, - ) - stream: Final = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) respx_mock.post(f"{SAIL_API_BASE}/messages").mock( - return_value=httpx.Response(200, content=stream, headers={"content-type": "text/event-stream"}) + return_value=httpx.Response( + 200, content=_messages_stream("balanced"), headers={"content-type": "text/event-stream"} + ) ) response: Final = await litellm.anthropic_messages( @@ -137,3 +124,83 @@ async def test_sail_streamed_messages_bill_the_reported_window( assert chunks assert await spend_capture.settled_cost() == pytest.approx(cost_at("_balanced")) + + +def _messages_stream(service_tier: str | None = None) -> str: + body: Final = messages_body() + usage: Final = { + "input_tokens": PROMPT_TOKENS - CACHED_TOKENS, + "cache_read_input_tokens": CACHED_TOKENS, + "output_tokens": COMPLETION_TOKENS, + **({"service_tier": service_tier} if service_tier else {}), + } + events: Final = ( + {"type": "message_start", "message": {**body, "content": [], "usage": {**usage, "output_tokens": 0}}}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": usage}, + {"type": "message_stop"}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("params", "drop_path", "wire_metadata", "suffix"), + [ + ({"service_tier": "flex"}, "service_tier", None, ""), + ({"service_tier": "balanced"}, "service_tier", None, ""), + ({"metadata": {"completion_window": "balanced"}}, "metadata", None, ""), + ( + {"metadata": {"completion_window": "balanced", "trace_id": "caller"}}, + "metadata.completion_window", + {"trace_id": "caller"}, + "", + ), + ( + {"service_tier": "flex", "metadata": {"completion_window": "balanced"}}, + "service_tier", + {"completion_window": "balanced"}, + "_balanced", + ), + ( + {"service_tier": "balanced", "metadata": {"completion_window": "flex", "trace_id": "caller"}}, + "metadata.completion_window", + {"completion_window": "balanced", "trace_id": "caller"}, + "_balanced", + ), + ], +) +@pytest.mark.asyncio +async def test_sail_messages_bill_the_window_sent_after_parameter_drops( + sail_env: None, + respx_mock: respx.MockRouter, + spend_capture: SpendCapture, + params: Mapping[str, object], + drop_path: str, + wire_metadata: Mapping[str, str] | None, + suffix: str, + stream: bool, +) -> None: + upstream: Final = ( + httpx.Response(200, content=_messages_stream(), headers={"content-type": "text/event-stream"}) + if stream + else httpx.Response(200, json=messages_body()) + ) + route: Final = respx_mock.post(f"{SAIL_API_BASE}/messages").mock(return_value=upstream) + response: Final = await litellm.anthropic_messages( + model=MODEL, + messages=MESSAGES, + max_tokens=16, + stream=stream, + additional_drop_params=[drop_path], + litellm_call_id=spend_capture.call_id, + **params, + ) + if stream: + async for _ in response: + pass + + assert sent_body(route).get("metadata") == wire_metadata + assert await spend_capture.settled_cost() == pytest.approx(cost_at(suffix))