From 28c80048ed47d53befd819f22b5ac1bb112ec3fd Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 18:51:58 -0800 Subject: [PATCH 01/34] fix(tool_choice): accept OpenAI allowed_tools on chat completions and flatten it for the Responses bridge (#49043) * fix(tool_choice): accept OpenAI allowed_tools on chat completions and flatten it for the Responses bridge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: type the allowed_tools tool_choice fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: raw regex for the allowed_tools provider gate match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tool_choice): move the allowed_tools provider gate out of completion() so pyright can still analyze it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tool_choice): keep the allowed_tools helpers typed so the basedpyright gate stays level Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tool_choice): type the allowed_tools validation boundary with Any instead of casts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tool_choice): widen tool_choice to object at the allowed_tools helpers with a cast-ok reason Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(tool_choice): drop allowed_tools test docstrings, type the new tests, and revert unrelated reformatting Also reword the allowed_tools 400 so it no longer blames the flat Responses layout for every malformed choice --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../transformation.py | 22 +++- .../allowed_tools_tool_choice.py | 111 ++++++++++++++++++ litellm/llms/azure/chat/gpt_transformation.py | 16 ++- litellm/main.py | 11 ++ litellm/utils.py | 25 +++- ...responses_transformation_transformation.py | 43 +++++++ .../test_allowed_tools_tool_choice.py | 100 ++++++++++++++++ .../test_azure_chat_gpt_transformation.py | 36 ++++++ tests/unit/test_main.py | 57 +++++++++ tests/unit/test_utils.py | 34 ++++++ 10 files changed, 448 insertions(+), 7 deletions(-) create mode 100644 litellm/litellm_core_utils/allowed_tools_tool_choice.py create mode 100644 tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c9d9cc25049..0a500173c5b 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -16,6 +16,7 @@ from openai.types.responses.response_input_param import ( ResponseCustomToolCallOutputParam, ResponseCustomToolCallParam, ) +from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam from openai.types.responses.tool_param import FunctionToolParam @@ -34,6 +35,10 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( requested_phase, ) from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + parse_chat_allowed_tools_tool_choice, + responses_allowed_tools_tool_choice, +) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -783,13 +788,28 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): def _normalize_tool_choice_for_responses_api( self, tool_choice: _ToolChoiceT - ) -> _ToolChoiceT | ToolChoiceFunctionParam | ToolChoiceCustomParam | Literal["auto", "none", "required"]: + ) -> ( + _ToolChoiceT + | ToolChoiceFunctionParam + | ToolChoiceCustomParam + | ToolChoiceAllowedParam + | Literal["auto", "none", "required"] + ): """Chat tool_choice nests the name under function/custom; Responses API expects top-level name.""" if not isinstance(tool_choice, dict): return tool_choice choice_type: Final = tool_choice.get("type") if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"): return choice_type + if choice_type == "allowed_tools": + parsed: Final = parse_chat_allowed_tools_tool_choice( + cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ) + ) + flat: Final = responses_allowed_tools_tool_choice(parsed) if parsed is not None else None + if flat is not None: + return flat if choice_type not in ("function", "custom"): return tool_choice if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"): diff --git a/litellm/litellm_core_utils/allowed_tools_tool_choice.py b/litellm/litellm_core_utils/allowed_tools_tool_choice.py new file mode 100644 index 00000000000..5c8f5d83ece --- /dev/null +++ b/litellm/litellm_core_utils/allowed_tools_tool_choice.py @@ -0,0 +1,111 @@ +from collections.abc import Mapping +from typing import Final, Literal + +from openai.types.responses.tool_choice_allowed_param import ToolChoiceAllowedParam +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError + +from litellm.exceptions import BadRequestError + +ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: Final = frozenset({"openai", "azure", "litellm_proxy"}) + +CHAT_ALLOWED_TOOLS_LAYOUT: Final = ( + '{"type": "allowed_tools", "allowed_tools": {"mode": "auto" | "required", "tools": []}}' +) + + +class ChatAllowedTools(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + mode: Literal["auto", "required"] + tools: tuple[Mapping[str, object], ...] = Field(min_length=1) + + +class ChatAllowedToolsToolChoice(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + type: Literal["allowed_tools"] + allowed_tools: ChatAllowedTools + + +_CHAT_ALLOWED_TOOLS_TOOL_CHOICE: Final = TypeAdapter(ChatAllowedToolsToolChoice) + + +_SELECTOR: Final = TypeAdapter(dict[str, object]) + + +class _ToolChoiceType(BaseModel): + model_config = ConfigDict(frozen=True) + + type: str + + +_TOOL_CHOICE_TYPE: Final = TypeAdapter(_ToolChoiceType) + + +def is_allowed_tools_tool_choice(tool_choice: object) -> bool: + try: + return _TOOL_CHOICE_TYPE.validate_python(tool_choice).type == "allowed_tools" + except ValidationError: + return False + + +def raise_if_allowed_tools_tool_choice_unsupported( + tool_choice: object, + custom_llm_provider: str, + model: str, +) -> None: + """Providers outside the set would silently send the request with no tool restriction at all, so the call stops + here with a 400. ``drop_params`` is deliberately not consulted: dropping a restriction widens what the model may call.""" + if not is_allowed_tools_tool_choice(tool_choice) or custom_llm_provider in ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS: + return + raise BadRequestError( + message=( + "tool_choice type 'allowed_tools' is only supported for " + f"{', '.join(sorted(ALLOWED_TOOLS_TOOL_CHOICE_PROVIDERS))}, got custom_llm_provider={custom_llm_provider}. " + "drop_params does not drop it, since that would let the model call tools the request excluded. " + "Send only the allowed tools in `tools` with tool_choice='auto' or 'required' instead." + ), + model=model, + llm_provider=custom_llm_provider, + ) + + +def parse_chat_allowed_tools_tool_choice(tool_choice: object) -> ChatAllowedToolsToolChoice | None: + try: + return _CHAT_ALLOWED_TOOLS_TOOL_CHOICE.validate_python(tool_choice) + except ValidationError: + return None + + +def _has_name(selector: Mapping[str, object]) -> bool: + name: Final = selector.get("name") + return isinstance(name, str) and name != "" + + +def _responses_allowed_tool(tool: Mapping[str, object]) -> Mapping[str, object] | None: + tool_type: Final = tool.get("type") + if tool_type not in ("function", "custom"): + return tool + if tool_type not in tool: + return tool if _has_name(tool) else None + try: + nested: Final = _SELECTOR.validate_python(tool[tool_type]) + except ValidationError: + return None + outer: Final = {key: value for key, value in tool.items() if key != tool_type} + if not _has_name(nested) or outer.keys() & nested.keys(): + return None + return {**outer, **nested} + + +def responses_allowed_tools_tool_choice(choice: ChatAllowedToolsToolChoice) -> ToolChoiceAllowedParam | None: + """The flat Responses API layout of a chat ``allowed_tools`` choice, or ``None`` when an entry is malformed or + carries conflicting keys, so the caller sends the original rather than a rewrite that could mean something looser.""" + flattened: Final = tuple(_responses_allowed_tool(tool) for tool in choice.allowed_tools.tools) + if any(tool is None for tool in flattened): + return None + return ToolChoiceAllowedParam( + type="allowed_tools", + mode=choice.allowed_tools.mode, + tools=[dict(tool) for tool in flattened if tool is not None], + ) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index fdb012756fc..0fe7caa1e16 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,11 +1,17 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import ( + TYPE_CHECKING, + Any, + Final, + cast, # noqa: TID251 # non_default_params reaches the mapper as an untyped dict +) from httpx._models import Headers, Response import litellm from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support +from litellm.litellm_core_utils.allowed_tools_tool_choice import is_allowed_tools_tool_choice from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( chat_completion_prompt_cache_breakpoints_for_wire, @@ -245,12 +251,12 @@ class AzureOpenAIConfig(BaseConfig): or (api_version_year == "2023" and api_version_month < "12") or (api_version_year == "2023" and api_version_month == "12" and api_version_day < "01") ): - if litellm.drop_params is True or (drop_params is not None and drop_params is True): - pass - else: + if is_allowed_tools_tool_choice( + cast(object, value) # cast-ok: widening an untyped dict value to object is always sound + ) or not (litellm.drop_params is True or (drop_params is not None and drop_params is True)): raise UnsupportedParamsError( status_code=400, - message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""", + message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. drop_params never drops an allowed_tools tool_choice, since that would let the model call tools the request excluded. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""", ) elif value == "required" and ( api_version_year == "2024" and api_version_month <= "05" diff --git a/litellm/main.py b/litellm/main.py index 78767b03eb7..38dd9f90be9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -70,6 +70,9 @@ from litellm.constants import ( ) from litellm.exceptions import LiteLLMUnknownProvider from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + raise_if_allowed_tools_tool_choice_unsupported, +) from litellm.litellm_core_utils.asyncify import asyncify, run_async_function from litellm.litellm_core_utils.audio_utils.utils import ( calculate_request_duration, @@ -5956,6 +5959,14 @@ def completion( if not is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): tools = _drop_anthropic_only_tool_keys(tools=tools) + raise_if_allowed_tools_tool_choice_unsupported( + tool_choice=cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ), + custom_llm_provider=custom_llm_provider, + model=model, + ) + if provider_specific_header is not None: headers.update( ProviderSpecificHeaderUtils.get_provider_specific_headers( diff --git a/litellm/utils.py b/litellm/utils.py index 55423a42260..5a0fc18dfdc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -83,6 +83,10 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + CHAT_ALLOWED_TOOLS_LAYOUT, + parse_chat_allowed_tools_tool_choice, +) from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, normalize_drop_params, @@ -9470,9 +9474,28 @@ def validate_chat_completion_tool_choice( tool_choice_type = tool_choice.get("type") if tool_choice_type in ("auto", "none", "required") and "function" not in tool_choice: return tool_choice_type + if ( + tool_choice_type == "allowed_tools" + and ( + parse_chat_allowed_tools_tool_choice( + cast( # cast-ok: widening an untyped dict to object is always sound, the helper validates it with pydantic + object, tool_choice + ) + ) + is None + ) + ): + raise BadRequestError( + message=( + f"Invalid tool choice, tool_choice={tool_choice}. On chat completions an allowed_tools " + f"tool_choice must be {CHAT_ALLOWED_TOOLS_LAYOUT}" + ), + model=model, + llm_provider="", + ) # Standard OpenAI format: {"type": "function", "function": {...}} - if tool_choice.get("type") is None or tool_choice.get("function") is None: + if tool_choice_type != "allowed_tools" and (tool_choice_type is None or tool_choice.get("function") is None): raise BadRequestError( message=f"Invalid tool choice, tool_choice={tool_choice}. Please ensure tool_choice follows the OpenAI spec", model=model, diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 558350a790a..9e40e128952 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3064,6 +3064,38 @@ def test_map_optional_params_tool_choice_chat_nested_to_responses_api(): {"type": "custom", "name": "ApplyPatch"}, ), ({"type": "custom"}, {"type": "custom"}), + ( + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "custom": {"name": "apply_patch"}}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "name": "already_flat"}, + ], + }, + }, + { + "type": "allowed_tools", + "mode": "required", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "apply_patch"}, + {"type": "mcp", "server_label": "docs"}, + {"type": "function", "name": "already_flat"}, + ], + }, + ), + ( + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}}, + ), + ( + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + ), ], ) def test_normalize_tool_choice_for_responses_api(tool_choice, expected): @@ -4105,6 +4137,16 @@ def test_output_item_done_with_stream_map_keeps_empty_delta(): ({"type": "required"}, "required"), ("auto", "auto"), ({"type": "function", "function": {"name": "get_weather"}}, {"type": "function", "name": "get_weather"}), + ( + { + "type": "allowed_tools", + "allowed_tools": { + "mode": "required", + "tools": [{"type": "function", "function": {"name": "get_weather"}}], + }, + }, + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_weather"}]}, + ), ], ) async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire( @@ -4177,6 +4219,7 @@ async def test_acompletion_bridge_normalizes_tool_choice_on_the_wire( post_kwargs = mock_post.call_args.kwargs request_body = post_kwargs["json"] if "json" in post_kwargs else json.loads(post_kwargs["data"]) assert request_body["tool_choice"] == expected_wire_tool_choice + assert [tool["name"] for tool in request_body["tools"]] == ["get_weather"] def _make_incomplete_responses_api_response( diff --git a/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py new file mode 100644 index 00000000000..6ebbdcf73bf --- /dev/null +++ b/tests/unit/litellm_core_utils/test_allowed_tools_tool_choice.py @@ -0,0 +1,100 @@ +import copy +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.allowed_tools_tool_choice import ( + ChatAllowedToolsToolChoice, + is_allowed_tools_tool_choice, + parse_chat_allowed_tools_tool_choice, + responses_allowed_tools_tool_choice, +) + +_CHAT_LAYOUT: Final[dict[str, object]] = { + "type": "allowed_tools", + "allowed_tools": { + "mode": "auto", + "tools": [ + {"type": "function", "function": {"name": "get_time"}}, + {"type": "custom", "custom": {"name": "apply_patch", "x-vendor": 1}, "x-trace": "a"}, + {"type": "mcp", "server_label": "docs", "server_url": "https://docs.example"}, + {"type": "function", "name": "already_flat"}, + ], + }, +} + + +def _parsed(tool_choice: Mapping[str, object]) -> ChatAllowedToolsToolChoice: + parsed: Final = parse_chat_allowed_tools_tool_choice(tool_choice) + assert parsed is not None + return parsed + + +def test_responses_layout_lifts_mode_and_tools_and_flattens_names_in_order(): + assert responses_allowed_tools_tool_choice(_parsed(_CHAT_LAYOUT)) == { + "type": "allowed_tools", + "mode": "auto", + "tools": [ + {"type": "function", "name": "get_time"}, + {"type": "custom", "name": "apply_patch", "x-vendor": 1, "x-trace": "a"}, + {"type": "mcp", "server_label": "docs", "server_url": "https://docs.example"}, + {"type": "function", "name": "already_flat"}, + ], + } + + +def test_responses_layout_leaves_the_caller_dict_untouched(): + original: Final = copy.deepcopy(_CHAT_LAYOUT) + flat: Final = responses_allowed_tools_tool_choice(_parsed(_CHAT_LAYOUT)) + assert flat is not None + next(iter(flat["tools"]))["name"] = "mutated" + assert _CHAT_LAYOUT == original + + +@pytest.mark.parametrize( + "tool", + [ + {"type": "function"}, + {"type": "function", "name": ""}, + {"type": "function", "function": {}}, + {"type": "function", "function": "get_time"}, + {"type": "custom", "custom": {"name": ""}}, + {"type": "function", "name": "get_time", "function": {"name": "get_weather"}}, + {"type": "function", "function": {"name": "get_time", "type": "custom"}}, + ], +) +def test_responses_layout_refuses_to_rewrite_a_malformed_or_conflicting_entry(tool: dict[str, object]): + tool_choice: Final[dict[str, object]] = { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [tool]}, + } + assert responses_allowed_tools_tool_choice(_parsed(tool_choice)) is None + + +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "auto", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": []}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "any", "tools": [{"type": "function"}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}, "extra": 1}, + {"type": "function", "function": {"name": "get_time"}}, + ], +) +def test_parse_rejects_anything_but_the_chat_layout(tool_choice: dict[str, object]): + assert parse_chat_allowed_tools_tool_choice(tool_choice) is None + + +@pytest.mark.parametrize( + ("tool_choice", "expected"), + [ + (_CHAT_LAYOUT, True), + ({"type": "allowed_tools"}, True), + ({"type": "function"}, False), + ("auto", False), + (None, False), + ], +) +def test_is_allowed_tools_tool_choice(tool_choice: Mapping[str, object] | str | None, expected: bool): + assert is_allowed_tools_tool_choice(tool_choice) is expected diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 1f1ff6a5748..dd29ee83bc6 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -75,6 +75,42 @@ def test_map_openai_params_with_preview_api_version(): ) +def _map_tool_choice_on_pre_tool_choice_api_version(tool_choice: dict[str, object]) -> dict[str, object]: + return AzureOpenAIConfig().map_openai_params( + {"tool_choice": tool_choice, "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + {}, + "gpt-4o", + drop_params=True, + api_version="2023-07-01-preview", + ) + + +def test_map_openai_params_drop_params_still_drops_a_named_tool_choice_on_old_api_version(): + mapped: Final = _map_tool_choice_on_pre_tool_choice_api_version( + {"type": "function", "function": {"name": "get_time"}} + ) + assert "tool_choice" not in mapped + + +def test_map_openai_params_drop_params_never_drops_allowed_tools_on_old_api_version(): + with pytest.raises(litellm.UnsupportedParamsError, match="allowed_tools") as exc: + _map_tool_choice_on_pre_tool_choice_api_version( + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": [{"type": "function"}]}} + ) + assert exc.value.status_code == 400 + + +def test_map_openai_params_forwards_allowed_tools_on_current_api_version(): + tool_choice: Final[dict[str, object]] = { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, + } + mapped: Final = AzureOpenAIConfig().map_openai_params( + {"tool_choice": tool_choice}, {}, "gpt-4o", drop_params=True, api_version="2025-04-01-preview" + ) + assert mapped["tool_choice"] == tool_choice + + def test_transform_request_hoists_tool_message_image(): """Azure builds its request via convert_to_azure_openai_messages without the OpenAIGPTConfig._transform_messages pipeline, so transform_request must hoist diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 85864b2024e..95c5958c748 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -553,6 +553,63 @@ def test_completion_strips_eager_input_streaming_before_openai(respx_mock: respx assert sent_tool["function"]["name"] == "write_file" +_ALLOWED_TOOLS_TOOL_CHOICE: Final = { + "type": "allowed_tools", + "allowed_tools": {"mode": "required", "tools": [{"type": "function", "function": {"name": "get_time"}}]}, +} +_TWO_TOOLS: Final = [ + {"type": "function", "function": {"name": "get_time", "parameters": {"type": "object", "properties": {}}}}, + {"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}}, +] + + +@pytest.mark.parametrize("model", ["openai/gpt-5.5", "azure/my-gpt-deployment"]) +def test_completion_forwards_allowed_tools_tool_choice_to_openai_compatible_chat( + respx_mock: respx.MockRouter, openai_api_response: dict[str, object], model: str +): + mock_route: Final = respx_mock.post(url__regex=r"http://localhost:12346/.*chat/completions.*").mock( + return_value=httpx.Response(status_code=200, json=openai_api_response) + ) + + litellm.completion( + model=model, + messages=[{"role": "user", "content": "What time is it?"}], + tools=_TWO_TOOLS, + tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, + api_base="http://localhost:12346/v1", + api_key="fake_api_key", + api_version="2025-04-01-preview", + ) + + assert mock_route.called + sent: Final = json.loads(respx_mock.calls[0].request.content) + assert sent["tool_choice"] == _ALLOWED_TOOLS_TOOL_CHOICE + assert [tool["function"]["name"] for tool in sent["tools"]] == ["get_time", "get_weather"] + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("model", ["anthropic/claude-sonnet-4-5", "gemini/gemini-2.5-flash", "bedrock/converse/x"]) +def test_completion_allowed_tools_tool_choice_is_a_400_where_the_provider_cannot_enforce_it( + respx_mock: respx.MockRouter, model: str, drop_params: bool +): + with pytest.raises( + litellm.BadRequestError, match=r"allowed_tools.*only supported for azure, litellm_proxy, openai" + ) as exc: + litellm.completion( + model=model, + messages=[{"role": "user", "content": "What time is it?"}], + tools=_TWO_TOOLS, + tool_choice=_ALLOWED_TOOLS_TOOL_CHOICE, + drop_params=drop_params, + api_key="fake_api_key", + aws_access_key_id="fake", + aws_secret_access_key="fake", + aws_region_name="us-east-1", + ) + assert exc.value.status_code == 400 + assert not respx_mock.calls + + _CLAUDE_TOOL_SEARCH_CALL: Final = { "id": "srvtoolu_01search", "type": "function", diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index a9b56568afb..19b3f317927 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -8651,6 +8651,40 @@ def test_validate_tool_choice_invalid_dict_is_a_400(tool_choice): assert exc_info.value.model == MODEL +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize("mode", ["auto", "required"]) +def test_validate_tool_choice_allowed_tools_chat_layout_passes_through(mode: str): + tool_choice: Final = { + "type": "allowed_tools", + "allowed_tools": { + "mode": mode, + "tools": [{"type": "function", "function": {"name": "get_time"}}, {"type": "mcp", "server_label": "docs"}], + }, + } + assert validate_chat_completion_tool_choice(tool_choice, model=MODEL) is tool_choice + + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "allowed_tools", "mode": "required", "tools": [{"type": "function", "name": "get_time"}]}, + {"type": "allowed_tools"}, + {"type": "allowed_tools", "allowed_tools": {"mode": "required"}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "required", "tools": []}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "none", "tools": [{"type": "function"}]}}, + {"type": "allowed_tools", "allowed_tools": {"tools": [{"type": "function", "function": {"name": "a"}}]}}, + {"type": "allowed_tools", "allowed_tools": {"mode": "auto", "tools": "get_time"}}, + ], +) +def test_validate_tool_choice_allowed_tools_malformed_is_a_400(tool_choice: dict[str, object]): + with pytest.raises(litellm.BadRequestError, match="allowed_tools") as exc_info: + validate_chat_completion_tool_choice(tool_choice, model=MODEL) + assert exc_info.value.status_code == 400 + assert exc_info.value.model == MODEL + assert '"mode": "auto" | "required"' in str(exc_info.value) + + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") @pytest.mark.parametrize("tool_choice", [123, []]) def test_validate_tool_choice_invalid_type_is_a_400(tool_choice): From 704e73d6fe27ed299b733d1c39e1ae7757c8fc5e Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:58:39 -0800 Subject: [PATCH 02/34] fix(cost-map): give the bare gpt-4o, gpt-4o-mini, gpt-4.1 and gpt-4.1-mini aliases their 2027-01-14 shutdown date (#49707) * fix(cost-map): give the bare gpt-4o, gpt-4o-mini, gpt-4.1 and gpt-4.1-mini aliases their 2027-01-14 shutdown date * fix(proxy): report a deployment's own provider retirement date before the bare model's An Azure deployment declared as model gpt-4o with custom_llm_provider azure, or as azure/ with base_model gpt-4o, now resolves azure/gpt-4o's date instead of the bare OpenAI row's. The GPT-4 series alias test now reads both cost maps with typed rows * fix(proxy): read Azure's retirement schedule for azure_ai deployments An azure_ai deployment with no azure_ai row of its own now reports the azure/ date, and a cost-map row wins over the provider-less copy the router registers under the deployment's key, which carries the bare model's date. * fix(proxy): let custom and registered base_model rows keep their deprecation date Only the copy the router writes at startup, which has no provider and inherits the bare row's date, gives way to a later candidate * fix(proxy): report a pinned snapshot's own earlier deprecation date over its base_model's * fix(proxy): recognize the router's copied cost-map rows by the loaded catalog * fix(proxy): read Azure's schedule for a provider-prefixed azure_ai base_model --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 4 + .../proxy/common_utils/model_deprecation.py | 121 ++++++++---- model_prices_and_context_window.json | 4 + .../common_utils/test_model_deprecation.py | 172 ++++++++++++++---- .../test_model_prices_and_context_window.py | 28 +++ 5 files changed, 263 insertions(+), 66 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cf8544316e9..032b7d17423 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34809,6 +34809,7 @@ "gpt-4.1": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, + "deprecation_date": "2027-01-14", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_priority": 3.5e-06, @@ -34894,6 +34895,7 @@ "gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, + "deprecation_date": "2027-01-14", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "input_cost_per_token_priority": 7e-07, @@ -35053,6 +35055,7 @@ "gpt-4o": { "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, + "deprecation_date": "2027-01-14", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 4.25e-06, @@ -35397,6 +35400,7 @@ "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, + "deprecation_date": "2027-01-14", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_priority": 2.5e-07, diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 21ad8cf1fde..5c764d1a33e 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -1,14 +1,15 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from datetime import date, datetime, timezone from itertools import groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, TypeAlias import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.types.proxy.model_deprecation import ( DEFAULT_DEPRECATION_WARN_DAYS, DeprecationStatus, @@ -20,6 +21,9 @@ if TYPE_CHECKING: from litellm.router import Router _NO_MODEL_METADATA: Final[Mapping[str, object]] = MappingProxyType({}) +_SHARED_SCHEDULE_PROVIDER: Final[Mapping[str, str]] = MappingProxyType({"azure_ai": "azure"}) + +LoadedCatalog: TypeAlias = Callable[[], Mapping[str, Mapping[str, object]]] @dataclass(frozen=True, slots=True) @@ -27,6 +31,7 @@ class _ResolvedDeprecation: deprecation_date: date litellm_model: str | None litellm_provider: str | None + copied_from_another_row: bool = False def _parse_deprecation_date(raw_value: object) -> date | None: @@ -47,8 +52,8 @@ def _successor_model(raw_value: object) -> str | None: return raw_value.strip() or None if isinstance(raw_value, str) else None -def _cost_map_lookup(model_key: object) -> _ResolvedDeprecation | None: - if not isinstance(model_key, str) or not model_key: +def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _ResolvedDeprecation | None: + if model_key is None: return None entry: Final = litellm.model_cost.get(model_key) if not isinstance(entry, Mapping): @@ -61,6 +66,51 @@ def _cost_map_lookup(model_key: object) -> _ResolvedDeprecation | None: deprecation_date=parsed, litellm_model=model_key, litellm_provider=provider if isinstance(provider, str) else None, + copied_from_another_row=isinstance(entry, AliasedCostEntry) + and "deprecation_date" not in entry.registered_fields + and model_key not in catalog, + ) + + +def _first_own_row(model_keys: Sequence[str | None], catalog: Mapping[str, object]) -> _ResolvedDeprecation | None: + found: Final = tuple( + resolved for resolved in (_cost_map_lookup(key, catalog) for key in model_keys) if resolved is not None + ) + return next((resolved for resolved in found if not resolved.copied_from_another_row), next(iter(found), None)) + + +def _deployment_provider(litellm_params: Mapping[str, object]) -> str | None: + declared: Final = litellm_params.get("custom_llm_provider") + if isinstance(declared, str) and declared: + return declared + model: Final = litellm_params.get("model") + prefix: Final = model.split("/", 1)[0] if isinstance(model, str) and "/" in model else None + return prefix if prefix in litellm.provider_list else None + + +def _provider_qualified(provider: str | None, model_key: str | None) -> str | None: + if provider is None or model_key is None or model_key.startswith(f"{provider}/"): + return None + return f"{provider}/{model_key}" + + +def _as_model_key(raw_value: object) -> str | None: + return raw_value if isinstance(raw_value, str) and raw_value else None + + +def _without_provider(provider: str | None, model_name: str | None) -> str | None: + if provider is None or model_name is None: + return model_name + return model_name.removeprefix(f"{provider}/") + + +def _schedule_keys(provider: str | None, model_name: str | None) -> tuple[str | None, ...]: + shared_provider: Final = _SHARED_SCHEDULE_PROVIDER.get(provider) if provider is not None else None + unprefixed: Final = _without_provider(provider, model_name) + return ( + _provider_qualified(provider, unprefixed), + _provider_qualified(shared_provider, unprefixed), + model_name, ) @@ -70,34 +120,37 @@ def _mapping_field(deployment: Mapping[str, object], key: str) -> Mapping[str, o def _resolve_deployment_deprecation( - deployment: Mapping[str, object], + deployment: Mapping[str, object], catalog: Mapping[str, object] ) -> _ResolvedDeprecation | None: """Resolve a deployment's deprecation date, preferring its explicit override""" model_info: Final = _mapping_field(deployment, "model_info") - raw_model: Final = _mapping_field(deployment, "litellm_params").get("model") + litellm_params: Final = _mapping_field(deployment, "litellm_params") + model_key: Final = _as_model_key(litellm_params.get("model")) override: Final = _parse_deprecation_date(model_info.get("deprecation_date")) if override is not None: - provider: Final = model_info.get("litellm_provider") + override_provider: Final = model_info.get("litellm_provider") return _ResolvedDeprecation( deprecation_date=override, - litellm_model=raw_model if isinstance(raw_model, str) else None, - litellm_provider=provider if isinstance(provider, str) else None, + litellm_model=model_key, + litellm_provider=override_provider if isinstance(override_provider, str) else None, ) - unprefixed: Final = raw_model.split("/", 1)[1] if isinstance(raw_model, str) and "/" in raw_model else None - return next( - ( - resolved - for resolved in ( - _cost_map_lookup(model_info.get("base_model")), - _cost_map_lookup(raw_model), - _cost_map_lookup(unprefixed), - ) - if resolved is not None - ), - None, + provider: Final = _deployment_provider(litellm_params) + base_model: Final = _as_model_key(model_info.get("base_model")) + model_name: Final = model_key.split("/", 1)[1] if model_key is not None and "/" in model_key else model_key + picks: Final = tuple( + pick + for pick in ( + _first_own_row(_schedule_keys(provider, base_model), catalog), + _first_own_row( + (_provider_qualified(provider, model_key), model_key, *_schedule_keys(provider, model_name)), catalog + ), + ) + if pick is not None ) + own_rows: Final = tuple(pick for pick in picks if not pick.copied_from_another_row) + return min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None) def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus: @@ -124,13 +177,17 @@ def _checked_successor(model_name: str, model_info: Mapping[str, object], llm_ro def _build_info( - deployment: Mapping[str, object], llm_router: Router, today: date, warn_within_days: int + deployment: Mapping[str, object], + llm_router: Router, + today: date, + warn_within_days: int, + catalog: Mapping[str, object], ) -> ModelDeprecationInfo | None: model_name: Final = deployment.get("model_name") if not isinstance(model_name, str) or not model_name: return None - resolved: Final = _resolve_deployment_deprecation(deployment) + resolved: Final = _resolve_deployment_deprecation(deployment, catalog) if resolved is None: return None @@ -182,15 +239,12 @@ def _bucket(models: Sequence[ModelDeprecationInfo], status: DeprecationStatus) - ) -def _collect_from_router(llm_router: Router, today: date, warn_within_days: int) -> tuple[ModelDeprecationInfo, ...]: +def _collect_from_router( + llm_router: Router, today: date, warn_within_days: int, catalog: Mapping[str, object] +) -> tuple[ModelDeprecationInfo, ...]: deployments: Final = llm_router.registry().deployments() or () - deduped: Final = _dedupe( - tuple( - info - for info in (_build_info(deployment, llm_router, today, warn_within_days) for deployment in deployments) - if info is not None - ) - ) + infos: Final = (_build_info(deployment, llm_router, today, warn_within_days, catalog) for deployment in deployments) + deduped: Final = _dedupe(tuple(info for info in infos if info is not None)) verbose_logger.debug( "model_deprecation: %d/%d deployments carry a deprecation date", @@ -204,12 +258,15 @@ def collect_model_deprecations( llm_router: Router | None, warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS, today: date | None = None, + loaded_catalog: LoadedCatalog = GetModelCostMap.loaded_model_cost_map, ) -> ModelDeprecationResponse: """Bucket every deployment carrying a deprecation date by how urgent it is""" snapshot_time: Final = datetime.now(timezone.utc) effective_today: Final = today or snapshot_time.date() deduped: Final = ( - _collect_from_router(llm_router, effective_today, warn_within_days) if llm_router is not None else () + _collect_from_router(llm_router, effective_today, warn_within_days, loaded_catalog()) + if llm_router is not None + else () ) return ModelDeprecationResponse( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cf8544316e9..032b7d17423 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34809,6 +34809,7 @@ "gpt-4.1": { "cache_read_input_token_cost": 5e-07, "cache_read_input_token_cost_priority": 8.75e-07, + "deprecation_date": "2027-01-14", "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_priority": 3.5e-06, @@ -34894,6 +34895,7 @@ "gpt-4.1-mini": { "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_priority": 1.75e-07, + "deprecation_date": "2027-01-14", "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, "input_cost_per_token_priority": 7e-07, @@ -35053,6 +35055,7 @@ "gpt-4o": { "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_priority": 2.125e-06, + "deprecation_date": "2027-01-14", "input_cost_per_token": 2.5e-06, "input_cost_per_token_batches": 1.25e-06, "input_cost_per_token_priority": 4.25e-06, @@ -35397,6 +35400,7 @@ "gpt-4o-mini": { "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_priority": 1.25e-07, + "deprecation_date": "2027-01-14", "input_cost_per_token": 1.5e-07, "input_cost_per_token_batches": 7.5e-08, "input_cost_per_token_priority": 2.5e-07, diff --git a/tests/unit/proxy/common_utils/test_model_deprecation.py b/tests/unit/proxy/common_utils/test_model_deprecation.py index 4f9f947fadb..7e96ad591b7 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation.py @@ -5,14 +5,17 @@ Slack integration — so they can run without the full proxy stack. """ import logging +from collections.abc import Mapping from datetime import date, datetime, timezone +from typing import Final from unittest.mock import MagicMock - +import pytest import litellm from litellm import Router from litellm._logging import verbose_logger +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.proxy.common_utils.model_deprecation import ( _classify, _parse_deprecation_date, @@ -37,12 +40,7 @@ class TestParseDeprecationDate: def test_should_return_none_for_documentation_sentinel(self): # The JSON map ships a sentinel string under the "sample_spec" key. - assert ( - _parse_deprecation_date( - "date when the model becomes deprecated in the format YYYY-MM-DD" - ) - is None - ) + assert _parse_deprecation_date("date when the model becomes deprecated in the format YYYY-MM-DD") is None def test_should_return_none_for_none(self): assert _parse_deprecation_date(None) is None @@ -51,9 +49,7 @@ class TestParseDeprecationDate: assert _parse_deprecation_date(12345) is None def test_should_narrow_datetime_to_date(self): - assert _parse_deprecation_date( - datetime(2026, 12, 31, 23, 59, tzinfo=timezone.utc) - ) == date(2026, 12, 31) + assert _parse_deprecation_date(datetime(2026, 12, 31, 23, 59, tzinfo=timezone.utc)) == date(2026, 12, 31) class TestClassify: @@ -134,9 +130,7 @@ class TestCollectModelDeprecations: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=today - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=today) assert [m.model_name for m in snapshot.deprecated] == ["deprecated-alias"] assert [m.model_name for m in snapshot.imminent] == ["imminent-alias"] @@ -166,9 +160,7 @@ class TestCollectModelDeprecations: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=today - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=today) assert len(snapshot.imminent) == 1 assert snapshot.imminent[0].deprecation_date == date(2026, 6, 10) @@ -195,9 +187,7 @@ class TestCollectModelDeprecations: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=today - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=today) assert len(snapshot.imminent) == 1 @@ -217,9 +207,7 @@ class TestCollectModelDeprecations: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=date(2026, 6, 1) - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1)) assert [m.litellm_model for m in snapshot.imminent] == ["gpt-4o"] @@ -244,9 +232,7 @@ class TestCollectModelDeprecations: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=date(2026, 6, 1) - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1)) assert len(snapshot.imminent) == 1 assert len(snapshot.upcoming) == 1 @@ -268,14 +254,136 @@ class TestCollectModelDeprecations: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=today - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=today) assert len(snapshot.imminent) == 1 assert snapshot.imminent[0].litellm_model == "base-thing" +class TestProviderScopedResolution: + CATALOG = { + "gpt-4o": {"deprecation_date": "2027-01-14", "litellm_provider": "openai"}, + "azure/gpt-4o": {"deprecation_date": "2027-04-14", "litellm_provider": "azure"}, + "github_copilot/gpt-4o": {"litellm_provider": "github_copilot"}, + "azure_ai/gpt-4.1": AliasedCostEntry( + {"deprecation_date": "2027-05-01", "litellm_provider": "azure_ai", "mode": "chat"}, () + ), + "custom-base": {"deprecation_date": "2026-12-01"}, + "gpt-4.1": {"deprecation_date": "2027-01-14", "litellm_provider": "openai"}, + "azure/gpt-4.1": {"deprecation_date": "2027-04-14", "litellm_provider": "azure"}, + "gpt-4o-2024-05-13": {"deprecation_date": "2026-10-23", "litellm_provider": "openai"}, + "azure/gpt-4o-2024-05-13": {"deprecation_date": "2026-12-09", "litellm_provider": "azure"}, + "azure/aliased-base": AliasedCostEntry( + {"deprecation_date": "2026-11-20"}, ("azure/aliased-base", "azure/aliased-base-v1") + ), + "aliased-base": {"deprecation_date": "2027-02-01"}, + } + COST_MAP = { + **CATALOG, + "azure_ai/gpt-4o": AliasedCostEntry( + {"deprecation_date": "2027-01-14", "litellm_provider": "azure", "mode": "chat"}, () + ), + "registered-base": AliasedCostEntry({"deprecation_date": "2026-12-15"}, (), frozenset({"deprecation_date"})), + } + + @pytest.mark.parametrize( + ("litellm_params", "model_info", "expected"), + [ + ({"model": "gpt-4o"}, {}, ("gpt-4o", date(2027, 1, 14), "openai")), + ({"model": "openai/gpt-4o"}, {}, ("gpt-4o", date(2027, 1, 14), "openai")), + ( + {"model": "gpt-4o", "custom_llm_provider": "azure"}, + {}, + ("azure/gpt-4o", date(2027, 4, 14), "azure"), + ), + ({"model": "azure/gpt-4o"}, {}, ("azure/gpt-4o", date(2027, 4, 14), "azure")), + ( + {"model": "azure/prod-chat"}, + {"base_model": "gpt-4o"}, + ("azure/gpt-4o", date(2027, 4, 14), "azure"), + ), + ( + {"model": "gpt-4o", "custom_llm_provider": "github_copilot"}, + {}, + ("gpt-4o", date(2027, 1, 14), "openai"), + ), + ({"model": "azure_ai/gpt-4o"}, {}, ("azure/gpt-4o", date(2027, 4, 14), "azure")), + ( + {"model": "azure_ai/prod-chat"}, + {"base_model": "gpt-4o"}, + ("azure/gpt-4o", date(2027, 4, 14), "azure"), + ), + ( + {"model": "azure_ai/gpt-4o"}, + {"base_model": "azure_ai/gpt-4o"}, + ("azure/gpt-4o", date(2027, 4, 14), "azure"), + ), + ( + {"model": "azure_ai/prod-chat"}, + {"base_model": "azure_ai/gpt-4o"}, + ("azure/gpt-4o", date(2027, 4, 14), "azure"), + ), + ({"model": "azure_ai/gpt-4.1"}, {}, ("azure_ai/gpt-4.1", date(2027, 5, 1), "azure_ai")), + ({"model": "azure/gpt-4o"}, {"base_model": "custom-base"}, ("custom-base", date(2026, 12, 1), None)), + ( + {"model": "azure/gpt-4o"}, + {"base_model": "registered-base"}, + ("registered-base", date(2026, 12, 15), None), + ), + ( + {"model": "azure/prod-chat"}, + {"base_model": "aliased-base"}, + ("azure/aliased-base", date(2026, 11, 20), None), + ), + ( + {"model": "openai/gpt-4o-2024-05-13"}, + {"base_model": "gpt-4o"}, + ("gpt-4o-2024-05-13", date(2026, 10, 23), "openai"), + ), + ( + {"model": "azure/gpt-4o-2024-05-13"}, + {"base_model": "gpt-4o"}, + ("azure/gpt-4o-2024-05-13", date(2026, 12, 9), "azure"), + ), + ], + ids=[ + "bare", + "openai-prefix", + "declared-azure", + "azure-prefix", + "azure-base-model", + "undated-own-row", + "azure-ai-reads-azure", + "azure-ai-base-model-reads-azure", + "azure-ai-copied-base-model-yields-to-azure", + "azure-ai-prefixed-base-model-on-custom-deployment-reads-azure", + "azure-ai-own-row-wins", + "custom-base-model-row-wins", + "registered-base-model-row-wins", + "aliased-base-model-row-wins", + "openai-snapshot-shuts-down-before-its-base-model", + "azure-snapshot-shuts-down-before-its-base-model", + ], + ) + def test_should_prefer_the_deployments_own_provider_date( + self, + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, str], + model_info: Mapping[str, str], + expected: tuple[str, date, str | None], + ) -> None: + monkeypatch.setattr(litellm, "model_cost", self.COST_MAP) + router: Final = _make_router( + [{"model_name": "alias", "litellm_params": litellm_params, "model_info": {"id": "1", **model_info}}] + ) + + snapshot: Final = collect_model_deprecations( + llm_router=router, warn_within_days=30, today=date(2026, 6, 1), loaded_catalog=lambda: self.CATALOG + ) + + assert [(m.litellm_model, m.deprecation_date, m.litellm_provider) for m in snapshot.upcoming] == [expected] + + class TestFormatDeprecationAlertMessage: def test_should_return_none_when_nothing_to_alert(self): snapshot = collect_model_deprecations(llm_router=None) @@ -321,9 +429,7 @@ class TestFormatDeprecationAlertMessage: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=today - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=today) message = format_deprecation_alert_message(snapshot) assert message is not None @@ -350,9 +456,7 @@ class TestFormatDeprecationAlertMessage: ] ) - snapshot = collect_model_deprecations( - llm_router=router, warn_within_days=30, today=today - ) + snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=today) message = format_deprecation_alert_message(snapshot) assert message is not None diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 4e8ede5c446..759eee51ba4 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -388,6 +388,34 @@ def test_vertex_gemini_flash_rows_carry_the_documented_retirement_date(prices: d assert prices[f"{prefix}{model}"]["deprecation_date"] == VERTEX_GEMINI_FLASH_RETIREMENT_DATES[model] +# https://developers.openai.com/api/docs/deprecations, the 2026-10-14 GPT-4 series notice +OPENAI_GPT_4_SERIES_SNAPSHOTS: Final = MappingProxyType( + { + "gpt-4o": ("gpt-4o-2024-08-06", "gpt-4o-2024-11-20"), + "gpt-4o-mini": ("gpt-4o-mini-2024-07-18",), + "gpt-4.1": ("gpt-4.1-2025-04-14",), + "gpt-4.1-mini": ("gpt-4.1-mini-2025-04-14",), + } +) + + +class _ShutdownRow(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + deprecation_date: str | None = None + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +@pytest.mark.parametrize("alias", sorted(OPENAI_GPT_4_SERIES_SNAPSHOTS)) +def test_openai_gpt_4_series_aliases_retire_with_their_snapshots(path: Path, alias: str) -> None: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + snapshot_dates: Final = frozenset( + _ShutdownRow.model_validate(prices[snapshot]).deprecation_date + for snapshot in OPENAI_GPT_4_SERIES_SNAPSHOTS[alias] + ) + assert snapshot_dates == {_ShutdownRow.model_validate(prices[alias]).deprecation_date} + + def _sampling_flags_across_provider_twins(prices: Mapping[str, Mapping[str, object]], key: str) -> frozenset[object]: bare: Final = key.split("/")[-1] twins: Final = tuple(twin for twin in (bare, f"vertex_ai/{bare}", f"gemini/{bare}") if twin in prices) From be50cbe9d8d5afa109ac17e6554245bf4b4511c4 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 19:40:46 -0800 Subject: [PATCH 03/34] fix(responses)!: authorize and decode comparison response IDs (#49402) * test(responses): reproduce comparison response ID ownership and wire failures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): guard stale retries and malformed comparison envelopes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): authorize and unwrap comparison response ids Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): require comparison IDs to follow response security policy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): deny raw comparison response ids by default Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): reject raw comparison response ids by default Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): enforce managed ownership with unmanaged ID opt-in Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/openai/responses/transformation.py | 37 +- litellm/proxy/hooks/responses_id_security.py | 72 +++- litellm/types/litellm_params.py | 4 + litellm/types/llms/openai.py | 1 + litellm/types/utils.py | 1 + .../test_responses_id_ownership.py | 382 +++++++++++++++++- .../providers/test_responses_core_wire.py | 189 ++++++++- .../test_pass_through_endpoints.py | 1 + tests/unit/test_responses_id_security.py | 266 +++++++++++- tests/unit/types/test_litellm_params.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 11 files changed, 927 insertions(+), 29 deletions(-) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index ef3b468dc1a..9f4a4fd6083 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -15,7 +15,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( deployment_prompt_cache_breakpoint_support, targets_openai_api, ) -from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers +from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, process_response_headers from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, @@ -45,6 +45,7 @@ from litellm.responses.litellm_completion_transformation.tool_search import ( tools_with_anthropic_tool_search_mapped, tools_without_anthropic_tool_search, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import * @@ -619,11 +620,45 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): "model": model, "input": replay_safe_input, **response_api_optional_request_params, + **self._prompt_cache_options_with_original_comparison_id(request_params), }, ) return final_request_params + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: GenericLiteLLMParams, + ) -> Mapping[str, object]: + return {**extra_body, **self._prompt_cache_options_with_original_comparison_id(extra_body)} + + def _prompt_cache_options_with_original_comparison_id(self, params: Mapping[str, object]) -> Mapping[str, object]: + options: Final = as_str_mapping(params.get("prompt_cache_options")) + comparison_response_id: Final = None if options is None else options.get("comparison_response_id") + if ( + self.custom_llm_provider != LlmProviders.OPENAI + or options is None + or not isinstance(comparison_response_id, str) + or not comparison_response_id.startswith("resp_") + ): + return {} + decoded_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(comparison_response_id) + decoded_provider: Final = decoded_id.get("custom_llm_provider") + original_response_id: Final = decoded_id.get("response_id", comparison_response_id) + rebuilt_envelope: Final = ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( + {"id": original_response_id}, decoded_provider, {"model_info": {"id": decoded_id.get("model_id")}} + ) + if ( + decoded_provider != LlmProviders.OPENAI + or original_response_id == comparison_response_id + or rebuilt_envelope.get("id") != comparison_response_id + ): + return {} + return {"prompt_cache_options": {**options, "comparison_response_id": original_response_id}} + def _prepared_input_and_tools( self, model: str, diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py index 61ab1eb4771..0269a784445 100644 --- a/litellm/proxy/hooks/responses_id_security.py +++ b/litellm/proxy/hooks/responses_id_security.py @@ -14,6 +14,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import LitellmUserRoles from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -26,7 +27,13 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ) from litellm.types.responses.main import DeleteResponseResult -from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypesLiteral, LLMResponseTypes, SpecialEnums +from litellm.types.utils import ( + ADDRESSED_COMPARISON_RESPONSE_ID_FIELD, + ADDRESSED_RESPONSE_ID_FIELD, + CallTypesLiteral, + LLMResponseTypes, + SpecialEnums, +) if TYPE_CHECKING: from litellm.caching.caching import DualCache @@ -70,6 +77,7 @@ def _proxy_signing_key() -> str | None: _RESPONSE_PAYLOAD_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_DATA_ADAPTER: Final = TypeAdapter(dict[str, object]) _ResponseObjectT = TypeVar("_ResponseObjectT", bound=BaseLiteLLMOpenAIResponseObject) _AddressedResultT = TypeVar("_AddressedResultT", bound=ResponsesAPIResponse | DeleteResponseResult) @@ -169,16 +177,64 @@ class ResponsesIDSecurity(CustomLogger): addressed_id_field: Final = _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE.get(call_type) if addressed_id_field is None: return None - retained_id: Final = data.get(ADDRESSED_RESPONSE_ID_FIELD) + request_data: Final = _REQUEST_DATA_ADAPTER.validate_python(data) + authorized_data: Final = ( + {**request_data, **self._authorized_comparison_response_id_fields(request_data, user_api_key_dict)} + if call_type == "aresponses" + else request_data + ) + retained_id: Final = authorized_data.get(ADDRESSED_RESPONSE_ID_FIELD) addressed_id: Final = ( - retained_id if isinstance(retained_id, str) and retained_id else data.get(addressed_id_field) + retained_id if isinstance(retained_id, str) and retained_id else authorized_data.get(addressed_id_field) ) if not isinstance(addressed_id, str) or not addressed_id: - return data - authorized_id: Final = self.authorize_response_id(addressed_id, user_api_key_dict) - data[addressed_id_field] = authorized_id - data[ADDRESSED_RESPONSE_ID_FIELD] = addressed_id - return data + return authorized_data + return { + **authorized_data, + addressed_id_field: self.authorize_response_id(addressed_id, user_api_key_dict), + ADDRESSED_RESPONSE_ID_FIELD: addressed_id, + } + + def _authorized_comparison_response_id_fields( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + ) -> Mapping[str, object]: + extra_body: Final = as_str_mapping(data.get("extra_body")) + overriding_extra_body: Final = ( + extra_body if extra_body is not None and "prompt_cache_options" in extra_body else None + ) + options: Final = as_str_mapping( + data.get("prompt_cache_options") + if overriding_extra_body is None + else overriding_extra_body.get("prompt_cache_options") + ) + requested_id: Final = None if options is None else options.get("comparison_response_id") + if options is None or not isinstance(requested_id, str): + return {} + addressed_id: Final = self._comparison_addressed_id( + requested_id, data.get(ADDRESSED_COMPARISON_RESPONSE_ID_FIELD) + ) + authorized_options: Final = { + **options, + "comparison_response_id": self.authorize_response_id(addressed_id, user_api_key_dict), + } + authorized_field: Final[Mapping[str, object]] = ( + {"prompt_cache_options": authorized_options} + if overriding_extra_body is None + else {"extra_body": {**overriding_extra_body, "prompt_cache_options": authorized_options}} + ) + return {**authorized_field, ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: addressed_id} + + def _comparison_addressed_id(self, requested_id: str, retained_id: object) -> str: + if ( + not isinstance(retained_id, str) + or self._is_encrypted_response_id(requested_id) + or not self._is_encrypted_response_id(retained_id) + ): + return requested_id + original_response_id, _, _ = self._decrypt_response_id(retained_id) + return retained_id if original_response_id == requested_id else requested_id def authorize_response_id( self, diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 1f90552e869..9f63d27f5f3 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -55,6 +55,7 @@ RoutingStrategyName: TypeAlias = Literal[ TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars" ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id" +ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: Final = "_litellm_addressed_comparison_response_id" STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected" WIRE_NAME: Final = "wire_name" @@ -407,6 +408,9 @@ class ProxyRequestState: secret_fields: "SecretFields | None" = None trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD)) addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD)) + addressed_comparison_response_id: str | None = field( + default=None, metadata=wire(ADDRESSED_COMPARISON_RESPONSE_ID_FIELD) + ) strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage")) stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD)) client_side_timeout: bool | None = None diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1d1628035ec..c803df04911 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -705,6 +705,7 @@ class AzureResponsesPromptCacheBreakpoint(TypedDict): class PromptCacheOptions(TypedDict, total=False): mode: ReadOnly[Literal["implicit", "explicit"]] ttl: ReadOnly[Literal["30m", "12h"]] + comparison_response_id: ReadOnly[str] class ChatCompletionThinkingBlock(TypedDict, total=False): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index eeb0956a06b..080ea9848b6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4356,6 +4356,7 @@ bedrock_batch_litellm_params: Final = BEDROCK_BATCH_KWARG_NAMES TRUSTED_CALLBACK_VARS_FIELD: Final = _litellm_params.TRUSTED_CALLBACK_VARS_FIELD ADDRESSED_RESPONSE_ID_FIELD: Final = _litellm_params.ADDRESSED_RESPONSE_ID_FIELD +ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: Final = _litellm_params.ADDRESSED_COMPARISON_RESPONSE_ID_FIELD anthropic_wif_litellm_params: Final = tuple(sorted(ANTHROPIC_WIF_KWARGS_KEYS)) openai_wif_litellm_params: Final = tuple(sorted(OPENAI_WIF_KWARGS_KEYS)) diff --git a/tests/integration/authorization/test_responses_id_ownership.py b/tests/integration/authorization/test_responses_id_ownership.py index 2ab26e75f50..c384ebb9434 100644 --- a/tests/integration/authorization/test_responses_id_ownership.py +++ b/tests/integration/authorization/test_responses_id_ownership.py @@ -3,7 +3,7 @@ from __future__ import annotations import json import os import uuid -from collections.abc import Callable +from collections.abc import Callable, Iterator, Mapping from pathlib import Path from typing import Final, Literal from urllib.parse import urlsplit @@ -16,7 +16,7 @@ from integration._support.process import owned_proxy from integration._support.responses_vendor import same_response from integration._support.wire import Reply, Request, Wire, wire_server from integration._support.worker_readiness import wait_for_worker_readiness -from openai.types.responses import ResponseInputParam +from openai.types.responses import ResponseCompletedEvent, ResponseInputParam from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state @@ -51,12 +51,6 @@ class _DeleteResponse(BaseModel): deleted: bool -def _deployments_in_group(gateway: Gateway, group: str) -> int: - entries: Final = gateway.get("/model/info")["data"] - assert isinstance(entries, list), entries - return sum(object_value(entry).get("model_name") == group for entry in entries) - - def _wait_for_workers(gateway: Gateway, group: str, deployments: int) -> None: wait_for_worker_readiness(gateway, group, deployments, _PROXY_WORKERS) @@ -138,6 +132,378 @@ def _every_id_route(client: openai.OpenAI, model: str, response_id: str) -> tupl ) +def _comparison_body(response_id: str, nested: bool) -> Mapping[str, object]: + options: Final = {"comparison_response_id": response_id} + if nested: + return { + "prompt_cache_options": {"mode": "explicit", "ttl": "12h"}, + "extra_body": {"prompt_cache_options": options}, + } + return {"prompt_cache_options": options} + + +@pytest.mark.parametrize( + ("prefix", "nested", "stream"), + (("/v1", False, False), ("", True, False), ("/openai/v1", False, True), ("/v1", True, True)), +) +def test_regression_comparison_response_id_owner_repeated_requests_reach_raw_provider_id( + gateway: Gateway, prefix: str, nested: bool, stream: bool +) -> None: + raw_id: Final = f"resp_comparison_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + assert (request.method, request.target) == ("POST", "/v1/responses") + body: Final = _JSON_OBJECT.validate_json(request.body) + if body["input"] == "create comparison response": + assert body == {"model": _MODEL, "input": "create comparison response"}, request.body + return Reply(body=_response(raw_id)) + assert body == { + "model": _MODEL, + "input": "compare owned response", + "stream": stream, + "prompt_cache_options": {"comparison_response_id": raw_id}, + }, request.body + if stream: + return Reply( + content_type="text/event-stream", + chunks=( + b"data: " + + json.dumps({"type": "response.completed", "response": json.loads(_response(raw_id))}).encode() + + b"\n\n", + b"data: [DONE]\n\n", + ), + ) + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + with _client(gateway, _key(scenario, team, user, model), prefix) as client: + created: Final = client.responses.create(model=model, input="create comparison response") + assert created.id != raw_id, created.model_dump_json() + assert len(_contract_requests(wire)) == 1 + + def compare_again() -> None: + result: Final = client.responses.create( + model=model, + input="compare owned response", + stream=stream, + extra_body=_comparison_body(created.id, nested), + ) + if isinstance(result, openai.Stream): + events: Final = tuple(result) + assert isinstance(events[-1], ResponseCompletedEvent), events + assert events[-1].response.status == "completed", events + else: + assert result.status == "completed", result.model_dump_json() + + for _ in range(2): + compare_again() + assert len(_contract_requests(wire)) == 2 + + +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize("stranger", ("user", "team")) +@pytest.mark.parametrize("retained", (False, True), ids=("ordinary", "client-internal-field")) +def test_regression_comparison_response_id_other_identity_is_denied_before_dispatch( + gateway: Gateway, nested: bool, stranger: str, retained: bool +) -> None: + raw_id: Final = f"resp_comparison_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team_a: Final = scenario.team(models=[model]) + team_b: Final = scenario.team(models=[model]) + user_a: Final = scenario.member(team_a) + user_b: Final = scenario.member(team_a) + gateway.post("/team/member_add", {"team_id": team_b, "member": {"role": "user", "user_id": user_a}}) + with ( + _client(gateway, _key(scenario, team_a, user_a, model), "/v1") as owner, + _client( + gateway, + _key( + scenario, team_a if stranger == "user" else team_b, user_b if stranger == "user" else user_a, model + ), + "/v1", + ) as other, + ): + created: Final = owner.responses.create(model=model, input="create private comparison") + assert created.id != raw_id + assert len(_contract_requests(wire)) == 1 + _refused( + lambda: other.responses.create( + model=model, + input="compare another identity", + extra_body={ + **_comparison_body(created.id, nested), + **( + {"_litellm_addressed_comparison_response_id": "resp_unmanaged_comparison"} + if retained + else {} + ), + }, + ), + _OTHER_USER_DETAIL if stranger == "user" else _OTHER_TEAM_DETAIL, + ) + assert _contract_requests(wire) == (), "A cross-identity comparison reached the provider" + + +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize("comparison_id", ("resp_unmanaged_comparison", "resp_!!!not-base64", "cmF3X3ZlbmRvcl9pZA==")) +def test_regression_unmanaged_comparison_response_id_is_denied_by_default( + gateway: Gateway, nested: bool, comparison_id: str +) -> None: + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + return Reply(body=_response(f"resp_result_{uuid.uuid4().hex}")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + with _client(gateway, _key(scenario, team, user, model), "/v1") as client: + _refused( + lambda: client.responses.create( + model=model, input="unmanaged comparison", extra_body=_comparison_body(comparison_id, nested) + ), + _UNMANAGED_DETAIL, + ) + assert _contract_requests(wire) == () + + +def test_comparison_extra_body_owned_id_overrides_foreign_top_level(gateway: Gateway) -> None: + foreign_raw_id: Final = f"resp_ignored_{uuid.uuid4().hex}" + owner_raw_id: Final = f"resp_effective_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + assert (request.method, request.target) == ("POST", "/v1/responses") + body: Final = _JSON_OBJECT.validate_json(request.body) + if body["input"] in ("seed ignored comparison", "seed effective comparison"): + assert body == {"model": _MODEL, "input": body["input"]}, request.body + return Reply(body=_response(foreign_raw_id if body["input"] == "seed ignored comparison" else owner_raw_id)) + assert body == { + "model": _MODEL, + "input": "compare effective options", + "prompt_cache_options": {"comparison_response_id": owner_raw_id}, + }, request.body + return Reply(body=_response(owner_raw_id)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + owner_user: Final = scenario.member(team) + foreign_user: Final = scenario.member(team) + with ( + _client(gateway, _key(scenario, team, owner_user, model), "/v1") as owner, + _client(gateway, _key(scenario, team, foreign_user, model), "/v1") as foreign, + ): + foreign_response: Final = foreign.responses.create(model=model, input="seed ignored comparison") + assert foreign_response.id != foreign_raw_id + owner_response: Final = owner.responses.create(model=model, input="seed effective comparison") + assert owner_response.id != owner_raw_id + assert len(_contract_requests(wire)) == 2 + compared: Final = owner.responses.create( + model=model, + input="compare effective options", + extra_body={ + "prompt_cache_options": { + "comparison_response_id": foreign_response.id, + "mode": "explicit", + "ttl": "12h", + }, + "extra_body": {"prompt_cache_options": {"comparison_response_id": owner_response.id}}, + }, + ) + assert compared.status == "completed", compared.model_dump_json() + assert len(_contract_requests(wire)) == 1 + + +def test_regression_comparison_extra_body_raw_id_overrides_foreign_top_level(gateway: Gateway) -> None: + raw_id: Final = "resp_raw_effective_comparison" + + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + return Reply(body=_response("resp_foreign_seed")) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + owner_user: Final = scenario.member(team) + foreign_user: Final = scenario.member(team) + with ( + _client(gateway, _key(scenario, team, owner_user, model), "/v1") as owner, + _client(gateway, _key(scenario, team, foreign_user, model), "/v1") as foreign, + ): + foreign_response: Final = foreign.responses.create(model=model, input="seed ignored comparison") + assert foreign_response.id != "resp_foreign_seed" + assert len(_contract_requests(wire)) == 1 + _refused( + lambda: owner.responses.create( + model=model, + input="compare effective options", + extra_body={ + "prompt_cache_options": { + "comparison_response_id": foreign_response.id, + "mode": "explicit", + "ttl": "12h", + }, + "extra_body": {"prompt_cache_options": {"comparison_response_id": raw_id}}, + }, + ), + _UNMANAGED_DETAIL, + ) + assert _contract_requests(wire) == () + + +@pytest.fixture(scope="module") +def allowed_comparison_gateway( + session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[tuple[Gateway, Wire]]: + directory: Final = tmp_path_factory.mktemp("allowed_comparison") + config: Final = directory / "config.yaml" + config.write_text("general_settings:\n allow_unmanaged_response_ids: true\n") + with wire_server(lambda request: Reply(body=_response("resp_decoy"))) as decoy: + with owned_proxy( + session_gateway, + directory, + { + "INTEGRATION_PROXY_WORKERS": "1", + "OPENAI_BASE_URL": f"{decoy.url}/v1", + "OPENAI_API_KEY": "synthetic-env-fallback-key", + }, + config=config, + ) as allowed: + yield allowed, decoy + + +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize("comparison_id", ("resp_unmanaged_comparison", "resp_!!!not-base64", "cmF3X3ZlbmRvcl9pZA==")) +def test_guard_opt_in_forwards_unmanaged_comparison_id( + allowed_comparison_gateway: tuple[Gateway, Wire], nested: bool, comparison_id: str +) -> None: + allowed, decoy = allowed_comparison_gateway + result_id: Final = f"resp_result_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + return Reply(body=_response(result_id)) + + with wire_server(respond) as wire, allowed.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team: Final = scenario.team(models=[model]) + user: Final = scenario.member(team) + with _client(allowed, _key(scenario, team, user, model), "/v1") as client: + compared: Final = client.responses.create( + model=model, input="unmanaged comparison", extra_body=_comparison_body(comparison_id, nested) + ) + assert compared.model_dump(mode="json", exclude_none=True) == { + "id": compared.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": model, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + }, compared.model_dump_json() + assert compared.id != result_id + requests: Final = _contract_requests(wire) + assert len(requests) == 1 + assert (requests[0].method, requests[0].target) == ("POST", "/v1/responses") + assert _JSON_OBJECT.validate_json(requests[0].body) == { + "model": _MODEL, + "input": "unmanaged comparison", + "prompt_cache_options": {"comparison_response_id": comparison_id}, + }, requests[0].body + assert decoy.drain() == () + + +@pytest.mark.parametrize("stranger", ("user", "team")) +def test_regression_echoed_raw_comparison_id_is_denied_to_other_identity(gateway: Gateway, stranger: str) -> None: + raw_id: Final = f"resp_echo_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + if request.target == "/v1/models": + return _model_discovery_reply() + body: Final = _JSON_OBJECT.validate_json(request.body) + if body["input"] == "create response": + assert body == {"model": _MODEL, "input": "create response"}, request.body + return Reply(body=_response(raw_id)) + assert body == { + "model": _MODEL, + "input": "compare owned response", + "prompt_cache_options": {"comparison_response_id": raw_id}, + }, request.body + return Reply( + body=json.dumps( + {**json.loads(_response(raw_id)), "prompt_cache_options": {"comparison_response_id": raw_id}} + ).encode() + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire) + team_a: Final = scenario.team(models=[model]) + team_b: Final = scenario.team(models=[model]) + user_a: Final = scenario.member(team_a) + user_b: Final = scenario.member(team_a) + gateway.post("/team/member_add", {"team_id": team_b, "member": {"role": "user", "user_id": user_a}}) + with ( + _client(gateway, _key(scenario, team_a, user_a, model), "/v1") as owner, + _client( + gateway, + _key( + scenario, team_a if stranger == "user" else team_b, user_b if stranger == "user" else user_a, model + ), + "/v1", + ) as other, + ): + created: Final = owner.responses.create(model=model, input="create response") + assert created.id != raw_id + assert len(_contract_requests(wire)) == 1 + echoed: Final = owner.responses.create( + model=model, input="compare owned response", extra_body=_comparison_body(created.id, False) + ) + echo_body: Final = _JSON_OBJECT.validate_json(echoed.model_dump_json(exclude_none=True)) + assert echo_body == { + "id": echoed.id, + "object": "response", + "created_at": 1.0, + "status": "completed", + "model": model, + "output": [], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "prompt_cache_options": {"comparison_response_id": raw_id}, + }, echo_body + options: Final = object_value(echo_body["prompt_cache_options"]) + leaked_id: Final = options["comparison_response_id"] + assert leaked_id == raw_id + assert isinstance(leaked_id, str) + assert len(_contract_requests(wire)) == 1 + _refused( + lambda: other.responses.create( + model=model, input="compare owned response", extra_body=_comparison_body(leaked_id, False) + ), + _UNMANAGED_DETAIL, + ) + assert _contract_requests(wire) == () + + @pytest.mark.parametrize("prefix", _ALIASES) def test_response_ids_are_scoped_to_the_issuing_user_and_team(gateway: Gateway, prefix: str) -> None: raw_id: Final = f"resp_owned_{uuid.uuid4().hex}" diff --git a/tests/integration/providers/test_responses_core_wire.py b/tests/integration/providers/test_responses_core_wire.py index 9a24903a20d..f9ff26045bc 100644 --- a/tests/integration/providers/test_responses_core_wire.py +++ b/tests/integration/providers/test_responses_core_wire.py @@ -1,10 +1,12 @@ from __future__ import annotations +import base64 import json import os import uuid -from collections.abc import Callable -from typing import Final +from collections.abc import AsyncIterator, Callable, Iterator +from copy import deepcopy +from typing import Final, Protocol, runtime_checkable import openai import pytest @@ -14,6 +16,10 @@ from integration._support.worker_readiness import wait_for_worker_readiness from openai.types.responses import ResponseCompletedEvent from pydantic import JsonValue, TypeAdapter +import litellm +from litellm.types.llms.openai import ResponseCompletedEvent as LiteLLMResponseCompletedEvent +from litellm.types.llms.openai import ResponsesAPIResponse + _MODEL: Final = "gpt-5" _API_KEY: Final = "synthetic-responses-key" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) @@ -21,6 +27,16 @@ _PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", " _ALIASES: Final = ("/v1", "", "/openai/v1") +@runtime_checkable +class _AsyncResponseEvents(Protocol): + def __aiter__(self) -> AsyncIterator[object]: ... + + +@runtime_checkable +class _ResponseEvents(Protocol): + def __iter__(self) -> Iterator[object]: ... + + def _deployments_in_group(gateway: Gateway, group: str) -> int: entries: Final = gateway.get("/model/info")["data"] assert isinstance(entries, list), entries @@ -69,6 +85,175 @@ def _stream(identity: str) -> tuple[bytes, ...]: ) + (b"data: [DONE]\n\n",) +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async")) +@pytest.mark.parametrize("stream", (False, True), ids=("nonstream", "stream")) +@pytest.mark.parametrize("extra", (False, True), ids=("top-level", "extra-body-precedence")) +async def test_regression_sdk_comparison_response_id_is_raw_on_the_wire( + asynchronous: bool, stream: bool, extra: bool +) -> None: + raw_id: Final = f"resp_sdk_comparison_{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/v1/responses") + body: Final = _JSON_OBJECT.validate_json(request.body) + if body["input"] == "seed comparison": + assert body == {"model": _MODEL, "input": "seed comparison"}, request.body + return Reply(body=_response(raw_id)) + assert body == { + "model": _MODEL, + "input": "compare response", + "stream": stream, + "previous_response_id": raw_id, + "prompt_cache_options": {"comparison_response_id": raw_id}, + }, request.body + return ( + Reply(content_type="text/event-stream", chunks=_stream(raw_id)) if stream else Reply(body=_response(raw_id)) + ) + + with wire_server(respond) as wire: + seeded: Final = litellm.responses( + model=f"openai/{_MODEL}", input="seed comparison", api_base=f"{wire.url}/v1", api_key=_API_KEY + ) + assert isinstance(seeded, ResponsesAPIResponse), seeded + assert seeded.id != raw_id, seeded + params: Final = { + "model": f"openai/{_MODEL}", + "input": "compare response", + "stream": stream, + "previous_response_id": seeded.id, + "api_base": f"{wire.url}/v1", + "api_key": _API_KEY, + "num_retries": 0, + **( + { + "prompt_cache_options": {"mode": "explicit", "ttl": "12h"}, + "extra_body": {"prompt_cache_options": {"comparison_response_id": seeded.id}}, + } + if extra + else {"prompt_cache_options": {"comparison_response_id": seeded.id}} + ), + } + original: Final = deepcopy(params) + result: Final = await litellm.aresponses(**params) if asynchronous else litellm.responses(**params) + if stream: + if asynchronous: + assert isinstance(result, _AsyncResponseEvents), result + async_events: Final = tuple([event async for event in result]) + async_completed: Final = async_events[-1] + assert isinstance(async_completed, LiteLLMResponseCompletedEvent), async_events + assert async_completed.response.status == "completed", async_events + else: + assert isinstance(result, _ResponseEvents), result + sync_events: Final = tuple(result) + sync_completed: Final = sync_events[-1] + assert isinstance(sync_completed, LiteLLMResponseCompletedEvent), sync_events + assert sync_completed.response.status == "completed", sync_events + else: + assert isinstance(result, ResponsesAPIResponse), result + assert result.status == "completed", result + assert params == original, "The caller's comparison options were mutated" + assert len(wire.drain()) == 2 + + +@pytest.mark.parametrize("extra", (False, True)) +@pytest.mark.parametrize( + "comparison_id", + ( + "resp_custom_vendor_id", + "resp_!!!not-base64", + "cmF3X3ZlbmRvcl9pZA==", + pytest.param( + "resp_" + + base64.b64encode(b"litellm:custom_llm_provider:xai;model_id:foreign;response_id:resp_foreign").decode(), + id="foreign-provider-wrapper", + ), + pytest.param("resp_" + base64.b64encode(b"custom;id;response").decode(), id="unmanaged-semicolons"), + pytest.param( + "resp_" + base64.b64encode(b"openai;foreign;resp_unmanaged").decode(), id="openai-without-envelope-markers" + ), + ), +) +def test_guard_sdk_custom_host_preserves_unmanaged_comparison_ids(extra: bool, comparison_id: str) -> None: + def respond(request: Request) -> Reply: + assert (request.method, request.target) == ("POST", "/v1/responses") + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _MODEL, + "input": "custom comparison", + "prompt_cache_options": {"comparison_response_id": comparison_id, "mode": "implicit", "ttl": "30m"}, + }, request.body + return Reply(body=_response("resp_custom_result")) + + with wire_server(respond) as wire: + options: Final = { + "comparison_response_id": comparison_id, + "mode": "implicit", + "ttl": "30m", + } + params: Final = ( + {"extra_body": {"prompt_cache_options": options}} if extra else {"prompt_cache_options": options} + ) + original: Final = deepcopy(params) + result: Final = litellm.responses( + model=f"openai/{_MODEL}", input="custom comparison", api_base=f"{wire.url}/v1", api_key=_API_KEY, **params + ) + assert isinstance(result, ResponsesAPIResponse), result + assert result.status == "completed", result + assert params == original + assert len(wire.drain()) == 1 + + +@pytest.mark.parametrize("provider", ("azure", "xai")) +@pytest.mark.parametrize("extra", (False, True)) +def test_guard_inherited_responses_providers_preserve_comparison_wrappers(provider: str, extra: bool) -> None: + raw_id: Final = f"resp_inherited_{uuid.uuid4().hex}" + + def seed(request: Request) -> Reply: + assert _JSON_OBJECT.validate_json(request.body) == {"model": _MODEL, "input": "seed inherited comparison"} + return Reply(body=_response(raw_id)) + + with wire_server(seed) as seed_wire: + seeded: Final = litellm.responses( + model=f"openai/{_MODEL}", + input="seed inherited comparison", + api_base=f"{seed_wire.url}/v1", + api_key=_API_KEY, + ) + assert isinstance(seeded, ResponsesAPIResponse), seeded + assert seeded.id != raw_id + assert len(seed_wire.drain()) == 1 + + def respond(request: Request) -> Reply: + assert request.method == "POST", request + expected_target: Final = ( + "/openai/responses?api-version=2025-03-01-preview" if provider == "azure" else "/v1/responses" + ) + assert request.target == expected_target, request.target + assert _JSON_OBJECT.validate_json(request.body) == { + "model": _MODEL, + "input": "inherited comparison", + "prompt_cache_options": {"comparison_response_id": seeded.id}, + }, request.body + return Reply(body=_response(raw_id)) + + with wire_server(respond) as wire: + options: Final = {"comparison_response_id": seeded.id} + params: Final = ( + {"extra_body": {"prompt_cache_options": options}} if extra else {"prompt_cache_options": options} + ) + result: Final = litellm.responses( + model=f"{provider}/{_MODEL}", + input="inherited comparison", + api_base=wire.url if provider == "azure" else f"{wire.url}/v1", + api_key=_API_KEY, + api_version="2025-03-01-preview" if provider == "azure" else None, + **params, + ) + assert isinstance(result, ResponsesAPIResponse), result + assert result.status == "completed", result + assert len(wire.drain()) == 1 + + def _register_in_group(scenario: Scenario, wire: Wire, group: str, order: int | None = None) -> str: existing: Final = _deployments_in_group(scenario.gateway, group) created: Final = scenario.gateway.post( diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 824f04aee75..cd6c1ff11b5 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -11157,6 +11157,7 @@ PROXY_STAMPED_NAMES: Final = frozenset( "secret_fields", "litellm_trusted_callback_vars", "_litellm_addressed_response_id", + "_litellm_addressed_comparison_response_id", "_litellm_strip_stream_usage", "_litellm_stream_usage_injected", "client_side_timeout", diff --git a/tests/unit/test_responses_id_security.py b/tests/unit/test_responses_id_security.py index 8dc50fa33c7..789fdbd453e 100644 --- a/tests/unit/test_responses_id_security.py +++ b/tests/unit/test_responses_id_security.py @@ -6,15 +6,17 @@ Tests the security hook that prevents user B from seeing response from user A. import datetime from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch import pytest from fastapi import HTTPException +from pydantic import TypeAdapter from litellm.caching.dual_cache import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.hooks.responses_id_security import ( ResponsesIDSecurity, _streams_response_ids, @@ -30,7 +32,15 @@ from litellm.types.llms.openai import ( ResponsesAPIStreamEvents, ) from litellm.types.responses.main import DeleteResponseResult -from litellm.types.utils import CallTypesLiteral, Choices, Message, ModelResponse, SpecialEnums, Usage +from litellm.types.utils import ( + ADDRESSED_COMPARISON_RESPONSE_ID_FIELD, + CallTypesLiteral, + Choices, + Message, + ModelResponse, + SpecialEnums, + Usage, +) @pytest.fixture @@ -760,7 +770,9 @@ def salt_key_env(monkeypatch): return _UNIT_TEST_SALT_KEY -def _hook(general_settings=None, signing_key=_UNIT_TEST_SALT_KEY): +def _hook( + general_settings: Mapping[str, object] | None = None, signing_key: str | None = _UNIT_TEST_SALT_KEY +) -> ResponsesIDSecurity: settings = general_settings if general_settings is not None else {} return ResponsesIDSecurity( general_settings_reader=lambda: settings, @@ -768,19 +780,254 @@ def _hook(general_settings=None, signing_key=_UNIT_TEST_SALT_KEY): ) -def _auth(user_id="owner-user", team_id="owner-team", user_role=None): - from litellm.proxy._types import UserAPIKeyAuth - +def _auth( + user_id: str = "owner-user", team_id: str = "owner-team", user_role: LitellmUserRoles | None = None +) -> UserAPIKeyAuth: return UserAPIKeyAuth(user_id=user_id, team_id=team_id, user_role=user_role) -def _issue_managed_id(hook, owner, provider_response_id=_FABRICATED_PROVIDER_RESPONSE_ID): +def _issue_managed_id( + hook: ResponsesIDSecurity, owner: UserAPIKeyAuth, provider_response_id: str = _FABRICATED_PROVIDER_RESPONSE_ID +) -> str: """Mint an id exactly the way the proxy hands one to a client on create.""" - issued = hook._encrypt_response_id( + issued = hook._encrypt_response_id( # pyright: ignore[reportPrivateUsage] # mints ids the way create does ResponsesAPIResponse(id=provider_response_id, created_at=1234567890, output=[], status="completed"), owner, ) - return issued.id + return ResponsesAPIResponse.model_validate(issued).id + + +def _comparison_options(data: object, nested: bool) -> Mapping[str, str]: + request: Final = TypeAdapter(dict[str, object]).validate_python(data) + container: Final = TypeAdapter(dict[str, object]).validate_python(request["extra_body"]) if nested else request + return TypeAdapter(dict[str, str]).validate_python(container["prompt_cache_options"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize("stranger", ("user", "team")) +@pytest.mark.parametrize("allow_unmanaged", (False, True)) +async def test_regression_comparison_id_reentry_keeps_ownership_after_owner_retry( + salt_key_env: str, nested: bool, stranger: str, allow_unmanaged: bool +) -> None: + hook: Final = _hook({"allow_unmanaged_response_ids": allow_unmanaged}, signing_key=salt_key_env) + owner: Final = UserAPIKeyAuth(user_id="owner-user", team_id="owner-team") + caller: Final = UserAPIKeyAuth( + user_id="stranger-user" if stranger == "user" else owner.user_id, + team_id="stranger-team" if stranger == "team" else owner.team_id, + ) + cache: Final = DualCache() + issued: Final = ResponsesAPIResponse.model_validate( + await hook.async_post_call_success_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + data={}, + user_api_key_dict=owner, + response=ResponsesAPIResponse( + id=_FABRICATED_PROVIDER_RESPONSE_ID, created_at=1234567890, output=[], status="completed" + ), + ) + ) + assert issued.id != _FABRICATED_PROVIDER_RESPONSE_ID + options: Final = {"comparison_response_id": issued.id} + request: Final = {"extra_body": {"prompt_cache_options": options}} if nested else {"prompt_cache_options": options} + first: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, cache=cache, data=request, call_type="aresponses" + ) + ) + retried: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, cache=cache, data=request, call_type="aresponses" + ) + ) + with pytest.raises(HTTPException) as refusal: + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=caller, cache=cache, data=request, call_type="aresponses" + ) + assert refusal.value.status_code == 403 + assert f"not associated with the {stranger}" in refusal.value.detail + assert _comparison_options(first, nested) == {"comparison_response_id": _FABRICATED_PROVIDER_RESPONSE_ID} + assert _comparison_options(retried, nested) == {"comparison_response_id": _FABRICATED_PROVIDER_RESPONSE_ID} + assert _comparison_options(request, nested) == {"comparison_response_id": issued.id} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize("replacement", ("resp_vendor_raw", "resp_!!!not-base64", "")) +async def test_guard_comparison_id_replaced_after_owner_hook_reaches_provider_unchanged( + salt_key_env: str, nested: bool, replacement: str +) -> None: + hook: Final = ResponsesIDSecurity( + general_settings_reader=lambda: {"allow_unmanaged_response_ids": True}, + signing_key_reader=lambda: salt_key_env, + ) + owner: Final = UserAPIKeyAuth(user_id="owner-user", team_id="owner-team") + cache: Final = DualCache() + issued: Final = ResponsesAPIResponse.model_validate( + await hook.async_post_call_success_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + data={}, + user_api_key_dict=owner, + response=ResponsesAPIResponse( + id=_FABRICATED_PROVIDER_RESPONSE_ID, created_at=1234567890, output=[], status="completed" + ), + ) + ) + first_options: Final = {"comparison_response_id": issued.id} + first_request: Final = ( + {"extra_body": {"prompt_cache_options": first_options}} if nested else {"prompt_cache_options": first_options} + ) + first: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, cache=cache, data=first_request, call_type="aresponses" + ) + ) + replaced_options: Final = {"comparison_response_id": replacement, "mode": "implicit"} + replaced: Final = ( + {**first, "extra_body": {"prompt_cache_options": replaced_options}} + if nested + else {**first, "prompt_cache_options": replaced_options} + ) + rerun: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, cache=cache, data=replaced, call_type="aresponses" + ) + ) + assert _comparison_options(rerun, nested) == replaced_options, rerun + + +def _comparison_request(comparison_response_id: str, nested: bool) -> dict[str, object]: + options: Final = {"comparison_response_id": comparison_response_id, "mode": "implicit"} + return {"extra_body": {"prompt_cache_options": options}} if nested else {"prompt_cache_options": options} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize("replacement", ("resp_vendor_raw", "resp_!!!not-base64", "")) +async def test_comparison_id_replaced_after_owner_hook_is_rejected_by_default( + salt_key_env: str, nested: bool, replacement: str +) -> None: + hook: Final = _hook() + owner: Final = _auth() + first: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, + cache=DualCache(), + data=_comparison_request(_issue_managed_id(hook, owner), nested), + call_type="aresponses", + ) + ) + replaced: Final = {**first, **_comparison_request(replacement, nested)} + with pytest.raises(HTTPException) as refusal: + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, cache=DualCache(), data=replaced, call_type="aresponses" + ) + assert refusal.value.status_code == 403 + assert "allow_unmanaged_response_ids" in refusal.value.detail + assert _comparison_options(replaced, nested)["comparison_response_id"] == replacement + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +async def test_raw_comparison_id_is_rejected_by_default(salt_key_env: str, nested: bool) -> None: + request: Final = _comparison_request(_FABRICATED_UNMANAGED_ID, nested) + with pytest.raises(HTTPException) as refusal: + await _hook().async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=_auth(), cache=DualCache(), data=request, call_type="aresponses" + ) + assert refusal.value.status_code == 403 + assert "allow_unmanaged_response_ids" in refusal.value.detail + assert _comparison_options(request, nested)["comparison_response_id"] == _FABRICATED_UNMANAGED_ID + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +@pytest.mark.parametrize( + ("general_settings", "signing_key", "user_role"), + [ + ({"allow_unmanaged_response_ids": True}, _UNIT_TEST_SALT_KEY, None), + ({"disable_responses_id_security": True}, _UNIT_TEST_SALT_KEY, None), + ({}, None, None), + ({}, _UNIT_TEST_SALT_KEY, LitellmUserRoles.PROXY_ADMIN), + ], +) +async def test_raw_comparison_id_follows_previous_response_id_escape_hatches( + salt_key_env: str, + nested: bool, + general_settings: dict[str, bool], + signing_key: str | None, + user_role: LitellmUserRoles | None, +) -> None: + hook: Final = _hook(general_settings=general_settings, signing_key=signing_key) + result: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=_auth(user_role=user_role), + cache=DualCache(), + data=_comparison_request(_FABRICATED_UNMANAGED_ID, nested), + call_type="aresponses", + ) + ) + assert _comparison_options(result, nested) == { + "comparison_response_id": _FABRICATED_UNMANAGED_ID, + "mode": "implicit", + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +async def test_retained_comparison_id_of_another_owner_cannot_authorize_its_raw_id( + salt_key_env: str, nested: bool +) -> None: + hook: Final = _hook() + victim_id: Final = _issue_managed_id(hook, _auth()) + request: Final = { + **_comparison_request(_FABRICATED_PROVIDER_RESPONSE_ID, nested), + ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: victim_id, + } + with pytest.raises(HTTPException) as refusal: + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=_auth(user_id="stranger-user", team_id="stranger-team"), + cache=DualCache(), + data=request, + call_type="aresponses", + ) + assert refusal.value.status_code == 403 + assert "not associated with the user" in refusal.value.detail + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +async def test_retained_own_comparison_id_cannot_carry_a_different_raw_id(salt_key_env: str, nested: bool) -> None: + hook: Final = _hook() + stranger: Final = _auth(user_id="stranger-user", team_id="stranger-team") + stranger_id: Final = _issue_managed_id(hook, stranger, provider_response_id="resp_strangerownprovideridcccccccc") + request: Final = { + **_comparison_request("resp_victimprovideriddddddddddddddddddddd", nested), + ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: stranger_id, + } + with pytest.raises(HTTPException) as refusal: + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=stranger, cache=DualCache(), data=request, call_type="aresponses" + ) + assert refusal.value.status_code == 403 + assert "allow_unmanaged_response_ids" in refusal.value.detail + + +@pytest.mark.asyncio +@pytest.mark.parametrize("nested", (False, True)) +async def test_newly_supplied_comparison_id_wins_over_the_retained_one(salt_key_env: str, nested: bool) -> None: + hook: Final = _hook() + owner: Final = _auth() + stale_id: Final = _issue_managed_id(hook, owner, provider_response_id="resp_staleownerprovideridbbbbbbbbbb") + fresh_id: Final = _issue_managed_id(hook, owner) + result: Final = TypeAdapter(dict[str, object]).validate_python( + await hook.async_pre_call_hook( # pyright: ignore[reportUnknownMemberType] # Legacy dict boundary + user_api_key_dict=owner, + cache=DualCache(), + data={**_comparison_request(fresh_id, nested), ADDRESSED_COMPARISON_RESPONSE_ID_FIELD: stale_id}, + call_type="aresponses", + ) + ) + assert _comparison_options(result, nested)["comparison_response_id"] == _FABRICATED_PROVIDER_RESPONSE_ID + assert result[ADDRESSED_COMPARISON_RESPONSE_ID_FIELD] == fresh_id class TestUnrecognizedResponseIdIsRejected: @@ -1299,7 +1546,6 @@ class TestStreamedReadAnswersWithTheIdTheClientHolds: "admin-team", ) - @pytest.mark.asyncio async def test_a_frame_naming_the_held_response_under_another_wrapping_still_gets_the_id_sent( self, mock_cache, salt_key_env diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 86a75ec65b9..d71791b6f11 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -270,6 +270,7 @@ INTERNAL_STATE_NAMES: Final = ( "github_copilot_user_session", "litellm_trusted_callback_vars", "_litellm_addressed_response_id", + "_litellm_addressed_comparison_response_id", "_litellm_strip_stream_usage", "_litellm_stream_usage_injected", "client_side_timeout", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 3aa7c888e98..8278fd9fff4 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -42058,6 +42058,8 @@ export interface components { }; /** PromptCacheOptions */ PromptCacheOptions: { + /** Comparison Response Id */ + comparison_response_id?: string; /** * Mode * @enum {string} From 2a94bb3835f1c3376e4ae948cf028c3ca409007a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:08:45 +0000 Subject: [PATCH 04/34] fix(spend): bill web search interception follow-ups once under the outer request (#49762) * fix(spend): bill web search interception follow-ups once under the outer request Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): keep web search rounds inside interception follow-ups billed Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(spend): keep the outer request as parent for nested interception follow-ups Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * revert(spend): drop parent override for nested interception follow-ups Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/_internal_context.py | 12 +++ .../websearch_interception/handler.py | 26 ++--- .../chat_completion_agentic_loop.py | 14 +-- litellm/llms/custom_httpx/llm_http_handler.py | 30 +++--- .../_support/interception_vendor.py | 32 +++++- ...t_interception_converted_stream_options.py | 98 ++++++++++++++++++- 6 files changed, 175 insertions(+), 37 deletions(-) diff --git a/litellm/_internal_context.py b/litellm/_internal_context.py index 40d3377bf66..f2f7869876d 100644 --- a/litellm/_internal_context.py +++ b/litellm/_internal_context.py @@ -52,6 +52,18 @@ def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]: internal_parent_request_id.reset(parent_token) +@contextmanager +def billed_separately() -> Generator[None]: + """A call that writes its own spend row even inside a sub-call whose parent bills the model response.""" + token: Final = is_internal_call.set(False) + parent_token: Final = internal_parent_request_id.set(None) + try: + yield + finally: + is_internal_call.reset(token) + internal_parent_request_id.reset(parent_token) + + @contextmanager def post_response_phase() -> Generator[None]: """Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns.""" diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 2bb89248be4..d41647fe5f1 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, cast from typing_extensions import Never, ReadOnly import litellm +from litellm._internal_context import billed_separately from litellm._logging import verbose_logger from litellm.anthropic_interface import messages as anthropic_messages from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME @@ -1628,19 +1629,20 @@ class WebSearchInterceptionLogger(CustomLogger): search_kwargs: Final = MappingProxyType( {**configured_search_kwargs, **parent_correlation.as_search_kwargs()} ) - result: Final = ( - await litellm.asearch( - query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs + with billed_separately(): + result: Final = ( + await litellm.asearch( + query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs + ) + if search_metadata is None + else await litellm.asearch( + query=query_arg, + search_provider=search_provider, + litellm_metadata=search_metadata, + **_NO_ASEARCH_NAMED, + **search_kwargs, + ) ) - if search_metadata is None - else await litellm.asearch( - query=query_arg, - search_provider=search_provider, - litellm_metadata=search_metadata, - **_NO_ASEARCH_NAMED, - **search_kwargs, - ) - ) # Format using transformation function search_result_text: Final = WebSearchTransformation.format_search_response(result) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 2ba47cbb400..1f29b5d7a4d 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -8,6 +8,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm._internal_context import internal_sub_call from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs @@ -204,12 +205,13 @@ async def _execute_chat_completion_agentic_plan( ) try: - response_followup = await litellm.acompletion( - model=full_model_name, - messages=patch.messages, - **optional_params_for_followup, - **kwargs_for_followup, - ) + with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObject) else None): + response_followup = await litellm.acompletion( + model=full_model_name, + messages=patch.messages, + **optional_params_for_followup, + **kwargs_for_followup, + ) if _post_hook_overridden(callback): try: response_followup = await callback.async_post_agentic_loop_response_hook( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 437944f8e03..879e4e02e3e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -37,6 +37,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm._internal_context import internal_sub_call from litellm._logging import redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import ( aload_beta_headers_config, @@ -6252,12 +6253,13 @@ class BaseLLMHTTPHandler: ) try: - response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses( - model=patch.model or model, - input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime - **optional_params, - **kwargs_for_followup, - ) + with internal_sub_call(logging_obj.litellm_call_id): + response: ResponsesAPIResponse | BaseResponsesAPIStreamingIterator = await litellm.aresponses( + model=patch.model or model, + input=patch.messages, # pyright: ignore[reportArgumentType] # pre-existing mismatch surfaced by the Router import; patch messages are valid response input at runtime + **optional_params, + **kwargs_for_followup, + ) if callback is not None: try: @@ -6346,6 +6348,7 @@ class BaseLLMHTTPHandler: max_loops: int, fingerprints: list[str], fingerprint: str, + logging_obj: Optional["LiteLLMLoggingObj"] = None, ) -> ModelResponse | CustomStreamWrapper: patch: Final = plan.request_patch or AgenticLoopRequestPatch() if patch.messages is None: @@ -6385,12 +6388,14 @@ class BaseLLMHTTPHandler: fingerprint=fingerprint, ) - return await litellm.acompletion( - model=full_model_name, - messages=patch.messages, - **optional_params_for_followup, - **kwargs_for_followup, - ) + with internal_sub_call(logging_obj.litellm_call_id if logging_obj is not None else None): + response: Final = await litellm.acompletion( + model=full_model_name, + messages=patch.messages, + **optional_params_for_followup, + **kwargs_for_followup, + ) + return response def _maybe_wrap_in_fake_stream( self, @@ -6739,6 +6744,7 @@ class BaseLLMHTTPHandler: messages=messages, optional_params=optional_params, kwargs=kwargs_with_provider, + logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, depth=depth, max_loops=max_loops, diff --git a/tests/integration/_support/interception_vendor.py b/tests/integration/_support/interception_vendor.py index 3ca40087441..b46eb998f7a 100644 --- a/tests/integration/_support/interception_vendor.py +++ b/tests/integration/_support/interception_vendor.py @@ -27,7 +27,7 @@ JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) _CHAT_TARGET: Final = re.compile(r"^/openai/deployments/([^/]+)/chat/completions$") -Mode: TypeAlias = Literal["search", "plain", "search-down", "followup-fails", "missing"] +Mode: TypeAlias = Literal["search", "multi-search", "plain", "search-down", "followup-fails", "missing"] def deployment(mode: Mode) -> str: @@ -42,8 +42,8 @@ def plain(name: str) -> str: return f"plain-{name}" -def query(name: str) -> str: - return f"q-{name}" +def query(name: str, turn: int = 1) -> str: + return f"q-{name}" if turn == 1 else f"q-{name}-{turn}" def _minted(prefix: str) -> str: @@ -98,9 +98,20 @@ def _chat(name: str, body: Mapping[str, JsonValue]) -> Reply: content_type="text/event-stream", chunks=chat_stream(_minted(f"chatcmpl-{name}-s-"), "gpt-4o", plain(name)) ) messages: Final = _ITEMS.validate_python(body["messages"]) - if any(message.get("role") == "tool" for message in messages): + tool_results: Final = tuple(message for message in messages if message.get("role") == "tool") + if tool_results: if name.endswith("-followup-fails"): return _azure_error(500, FOLLOWUP_FAILURE) + if name.endswith("-multi-search") and len(tool_results) == 1: + call: Final = { + "id": f"call_{name[4:36]}-2", + "type": "function", + "function": {"name": SEARCH_TOOL, "arguments": json.dumps({"query": query(name, turn=2)})}, + } + message: Final = JSON_OBJECT.validate_python({"role": "assistant", "content": None, "tool_calls": [call]}) + return _chat_completion( + _minted(f"chatcmpl-{name}-2-"), message, "tool_calls", FOLLOWUP_USAGE + ) return _chat_completion( _minted(f"chatcmpl-{name}-2-"), {"role": "assistant", "content": answer(name)}, "stop", FOLLOWUP_USAGE ) @@ -153,9 +164,20 @@ def _responses(body: Mapping[str, JsonValue]) -> Reply: content_type="text/event-stream", chunks=responses_stream(_minted(f"resp_{name}_s_"), name, plain(name)) ) items: Final = _ITEMS.validate_python(body["input"]) if isinstance(body.get("input"), list) else [] - if any(item.get("type") == "function_call_output" for item in items): + function_call_outputs: Final = tuple(item for item in items if item.get("type") == "function_call_output") + if function_call_outputs: if name.endswith("-followup-fails"): return _azure_error(500, FOLLOWUP_FAILURE) + if name.endswith("-multi-search") and len(function_call_outputs) == 1: + call: Final = { + "type": "function_call", + "id": f"fc_{name[4:36]}-2", + "call_id": f"call_{name[4:36]}-2", + "name": SEARCH_TOOL, + "arguments": json.dumps({"query": query(name, turn=2)}), + "status": "completed", + } + return _response(_minted(f"resp_{name}_2_"), name, (call,), RESPONSES_FOLLOWUP_USAGE) return _response( _minted(f"resp_{name}_2_"), name, (_text_item(f"{name}_2", answer(name)),), RESPONSES_FOLLOWUP_USAGE ) diff --git a/tests/integration/streaming/test_interception_converted_stream_options.py b/tests/integration/streaming/test_interception_converted_stream_options.py index 0ca62d2f014..5a7855102c0 100644 --- a/tests/integration/streaming/test_interception_converted_stream_options.py +++ b/tests/integration/streaming/test_interception_converted_stream_options.py @@ -5,6 +5,7 @@ import uuid from collections.abc import Callable, Generator, Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass +from hashlib import sha256 from pathlib import Path from queue import SimpleQueue from types import MappingProxyType @@ -216,10 +217,13 @@ def scenario(rig: _Rig) -> Iterator[Scenario]: yield opened -def _deploy(rig: _Rig, scenario: Scenario, mode: iv.Mode, *, api_key: str = iv.AZURE_KEY) -> _Deployment: +def _deploy( + rig: _Rig, scenario: Scenario, mode: iv.Mode, *, api_key: str = iv.AZURE_KEY, priced: bool = False +) -> _Deployment: name: Final = iv.deployment(mode) + pricing: Final = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} if priced else {} model: Final = scenario.model( - model=f"azure/{name}", api_base=rig.wire.url, api_key=api_key, api_version=iv.API_VERSION + model=f"azure/{name}", api_base=rig.wire.url, api_key=api_key, api_version=iv.API_VERSION, **pricing ) return _Deployment(model, name) @@ -325,6 +329,46 @@ def _assert_response_landed_once(model: str, identity: str) -> None: assert row["status"] == "success" and rv.same_response(string_value(row["request_id"]), identity), row +def _assert_single_model_spend_for_key(key: str, call_type: str, *, expected_searches: int = 1) -> None: + digest: Final = sha256(key.encode()).hexdigest() + observed: Final = eventually( + lambda: ( + read_rows( + "SELECT status, spend, request_id, call_type, model_group, session_id " + 'FROM "LiteLLM_SpendLogs" WHERE api_key = %s AND status = %s', + (digest, "success"), + ), + read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', (digest,)), + ), + lambda values: ( + bool(values[0]) + and ( + len(values[0]) > expected_searches + 1 + or ( + len(values[0]) == expected_searches + 1 + and len(values[1]) == 1 + and values[1][0]["spend"] is not None + and float(str(values[1][0]["spend"])) + == pytest.approx(sum(float(str(row["spend"])) for row in values[0])) + ) + ) + ), + seconds=70, + ) + rows, balance = observed + model_rows: Final = [row for row in rows if row["call_type"] == call_type] + search_rows: Final = [row for row in rows if row["call_type"] == "asearch"] + spend: Final = sum(float(str(row["spend"])) for row in rows) + assert ( + len(rows) == expected_searches + 1 + and len(model_rows) == 1 + and len(search_rows) == expected_searches + and float(str(model_rows[0]["spend"])) > 0 + and all(float(str(row["spend"])) > 0 for row in search_rows) + and float(str(balance[0]["spend"])) == pytest.approx(spend) + ), (rows, balance) + + def _openai(rig: _Rig) -> openai.OpenAI: return openai.OpenAI(base_url=f"{rig.base_url}/v1", api_key=rig.gateway.key, max_retries=0) @@ -402,6 +446,31 @@ def test_raw_chat_stream_without_stream_options_answers_without_a_usage_chunk(ri _assert_search_turn(rig, deployment, identity) +def test_chat_stream_through_web_search_has_single_spend_owner(rig: _Rig, scenario: Scenario) -> None: + deployment: Final = _deploy(rig, scenario, "search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/chat/completions", _chat_body(deployment), key=key) + identity: Final = _assert_restreamed(stream, iv.answer(deployment.name), usage=None) + _assert_search_turn(rig, deployment, identity) + _assert_single_model_spend_for_key(key, "acompletion") + + +def test_multi_round_chat_stream_through_web_search_bills_each_search( + rig: _Rig, scenario: Scenario +) -> None: + deployment: Final = _deploy(rig, scenario, "multi-search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/chat/completions", _chat_body(deployment), key=key) + identity: Final = _assert_restreamed(stream, iv.answer(deployment.name), usage=None) + received: Final = _upstream(rig, deployment) + _assert_upstream(received, model_calls=3, searches=2) + assert [search["query"] for search in received.searches] == [ + iv.query(deployment.name), + iv.query(deployment.name, turn=2), + ], received.searches + _assert_single_model_spend_for_key(key, "acompletion", expected_searches=2) + + def test_openai_sdk_responses_stream_through_web_search_keeps_stream_options_off_azure( rig: _Rig, scenario: Scenario ) -> None: @@ -454,6 +523,31 @@ def test_raw_responses_stream_without_stream_options_answers(rig: _Rig, scenario _assert_response_landed_once(deployment.model, identity) +def test_responses_stream_through_web_search_has_single_spend_owner(rig: _Rig, scenario: Scenario) -> None: + deployment: Final = _deploy(rig, scenario, "search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/responses", _responses_body(deployment), key=key) + _assert_responses_restreamed(stream, iv.answer(deployment.name)) + _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) + _assert_single_model_spend_for_key(key, "aresponses") + + +def test_multi_round_responses_stream_through_web_search_bills_each_search( + rig: _Rig, scenario: Scenario +) -> None: + deployment: Final = _deploy(rig, scenario, "multi-search", priced=True) + key: Final = scenario.key(models=[deployment.model]) + stream: Final = _post(rig, "/v1/responses", _responses_body(deployment), key=key) + _assert_responses_restreamed(stream, iv.answer(deployment.name)) + received: Final = _upstream(rig, deployment) + _assert_upstream(received, model_calls=3, searches=2) + assert [search["query"] for search in received.searches] == [ + iv.query(deployment.name), + iv.query(deployment.name, turn=2), + ], received.searches + _assert_single_model_spend_for_key(key, "aresponses", expected_searches=2) + + def test_openai_sdk_chat_without_streaming_through_web_search_answers_in_json(rig: _Rig, scenario: Scenario) -> None: deployment: Final = _deploy(rig, scenario, "search") completion: Final = _openai(rig).chat.completions.create( From 08b951a81412240d845f672970d26dc113ac9248 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:13:28 +0000 Subject: [PATCH 05/34] fix(vllm): list hosted_vllm wildcard models when the server has no api key (#40066) * fix(proxy): expand wildcard models for providers with model info but no cost map entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): stop requiring an api key to list vllm models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): preserve model ids during wildcard expansion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): forward api key for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): support bearer auth for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): list hosted_vllm models from the server chat uses, with a bounded wait The hosted_vllm listing now resolves HOSTED_VLLM_API_BASE and HOSTED_VLLM_API_KEY like hosted_vllm chat, instead of the /vllm pass-through's VLLM_API_BASE, and waits at most NON_LLM_CONNECTION_TIMEOUT rather than the completion timeout. vllm/* keeps listing nothing, since its chat runs in process. The router pass-through now forwards the deployment's api_key. * fix(vllm): skip the placeholder key, honor deployment TLS and headers, and retry failed listings after a minute * fix(discovery): cache only timed-out model listings, so a fast failure is tried again on the next call * fix(proxy): list wildcard models off the event loop, so a hung provider no longer stalls other requests * fix(utils): keep a concurrent call's model listing when another call times out * fix(proxy): run model discovery on its own thread pool and keep credential lookups off the event loop * fix(proxy): stop the file credential lookup at the first match before listing wildcard models The fallback lookup now walks the allowed model names lazily, in the same order as before, so a hung vLLM server is listed only when no configured name serves the requested provider. The vLLM auth headers no longer mutate the caller's headers dict. * fix(proxy): try every configured wildcard route before listing any wildcard's models in the file credential lookup * fix(proxy): answer model lists from cached listings first and share one listing per server * test(vllm): give the self-signed cert fixture fixed validity dates * fix(proxy): skip wildcard servers a key cannot reach on /v1/model/info /v1/model/info listed every wildcard deployment and filtered by the key afterwards, so a key limited to one model waited on every wildcard server and sent listing calls to servers it cannot use. The listing now skips a wildcard whose patterns match none of the key's models before it asks the server. A key or team granted hosted_vllm/* with no deployment of that name now lists no models, like openrouter/*, since chat answers 400 for every id the environment server would list. * fix(proxy): match a wildcard's own row pattern before asking its server The /v1/model/info pre-filter matched a key's models against both the deployment name and its litellm_params.model, so a key granted hosted_vllm/* still asked a vllm-secure/* -> hosted_vllm/* server whose rows it can never see. It now matches only the pattern expansion names the rows with, picked by one shared helper. * test(vllm): cover the vLLM listings and no-slash wildcard aliases the patch left unrun Adds tests for hosted_vllm get_models with and without a server, vllm/* never asking a configured server, and /model/info pre-filtering of wildcard aliases whose name has no slash. Drops a test comment that only repeated a method signature. --------- Co-authored-by: jesus Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 92 +++-- .../llms/vllm/passthrough/transformation.py | 3 - litellm/proxy/auth/model_checks.py | 38 +- litellm/proxy/proxy_server.py | 11 +- litellm/utils.py | 10 +- .../unit/llms/vllm/test_vllm_common_utils.py | 27 ++ tests/unit/proxy/auth/test_model_checks.py | 353 +++++++++++++++++- .../proxy_server/test_routes_model_info.py | 50 +++ tests/unit/test_utils.py | 17 + 9 files changed, 555 insertions(+), 46 deletions(-) create mode 100644 tests/unit/llms/vllm/test_vllm_common_utils.py diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 014fabd5187..41db96914f5 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -2,10 +2,12 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter -import litellm +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import get_httpx_client from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base @@ -37,17 +39,31 @@ class VLLMError(BaseLLMException): ) +_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def _auth_headers(api_key: str | None) -> Mapping[str, str]: + if api_key is None or api_key == PLACEHOLDER_API_KEY: + return {} + return {"Authorization": f"Bearer {api_key}"} + + +def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) else None + + class VLLMModelInfo(BaseLLMModelInfo): def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], optional_params: dict, litellm_params: dict, api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: upstream_key: Final = self.upstream_api_key(api_key, api_base) return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers @@ -58,7 +74,7 @@ class VLLMModelInfo(BaseLLMModelInfo): return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key} @staticmethod - def get_api_base(api_base: str | None = None) -> str | None: + def get_api_base(api_base: str | None = None) -> str: api_base = api_base or get_secret_str("VLLM_API_BASE") if api_base is None: raise ValueError( @@ -74,25 +90,55 @@ class VLLMModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> str | None: return model - def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: - api_base = VLLMModelInfo.get_api_base(api_base) - api_key = VLLMModelInfo.get_api_key(api_key) - endpoint: Final = "/v1/models" - if api_base is None or api_key is None: - raise ValueError( - "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint." - ) - - url: Final = add_path_to_api_base(api_base, endpoint) - response: Final = litellm.module_level_client.get( - url=url, - ) - - response.raise_for_status() - - models: Final = response.json()["data"] - - return [model["id"] for model in models] + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: mirrors override contract + return [] def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VLLMError(status_code=status_code, message=error_message, headers=headers) + + +class HostedVLLMModelInfo(VLLMModelInfo): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") + if resolved_api_base is None: + raise ValueError("HOSTED_VLLM_API_BASE is not set.") + return resolved_api_base + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("HOSTED_VLLM_API_KEY") + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None) + + def discover_models( + self, litellm_params: Mapping[str, object] | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + params: Final = litellm_params or {} + ssl_verify: Final = params.get("ssl_verify") + return self._list_models( + api_key=_str_param(params, "api_key"), + api_base=_str_param(params, "api_base"), + extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}), + ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None, + ) + + def _list_models( + self, + *, + api_key: str | None, + api_base: str | None, + extra_headers: Mapping[str, str], + ssl_verify: bool | str | None, + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models") + response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get( + url=url, + headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))}, + timeout=NON_LLM_CONNECTION_TIMEOUT, + ) + response.raise_for_status() + return [model["id"] for model in response.json()["data"]] diff --git a/litellm/llms/vllm/passthrough/transformation.py b/litellm/llms/vllm/passthrough/transformation.py index 2b253e7fa38..10d740e49e0 100644 --- a/litellm/llms/vllm/passthrough/transformation.py +++ b/litellm/llms/vllm/passthrough/transformation.py @@ -67,9 +67,6 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig): ) -> tuple["URL", str]: base_target_url: Final = self.get_api_base(api_base) - if base_target_url is None: - raise Exception("VLLM api base not found") - return ( self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params), base_target_url, diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 4bcf020a9c4..e41c9354a8a 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -6,7 +6,7 @@ import copy import functools import itertools import re -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from types import MappingProxyType from typing import Final, TypeVar @@ -35,7 +35,9 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) _MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing") -_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value}) +_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset( + {LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value} +) _Listed: Final = TypeVar("_Listed") @@ -562,9 +564,27 @@ def _names_no_deployment_serves_by_name( return tuple(name for name in names if not deployed_by_name(name, team_id)) +def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None: + if not _check_wildcard_routing(model_name): + return None + if "/" in model_name or model_name == "*": + return model_name + if _check_wildcard_routing(litellm_model): + return litellm_model + return model_name + + +def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool: + if allowed_model_names is None: + return True + pattern: Final = _wildcard_name_regex(wildcard_pattern) + return any(pattern.match(name) for name in allowed_model_names) + + def expand_wildcard_deployments_for_model_info( deployments: list[dict[str, object]], llm_router: Router | None = None, + allowed_model_names: Collection[str] | None = None, ) -> list[dict[str, object]]: """Expand wildcard deployments into one row per known provider model. @@ -581,19 +601,11 @@ def expand_wildcard_deployments_for_model_info( raw_params = deployment.get("litellm_params") litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {} litellm_model = str(litellm_params_dict.get("model") or "") - - # Determine the wildcard pattern to expand. - # Branch order matters: only fall to litellm_model when model_name is - # also a wildcard, so a concrete model_name is never overwritten. - if _check_wildcard_routing(model_name) and ("/" in model_name or model_name == "*"): - wildcard_pattern = model_name - elif _check_wildcard_routing(model_name) and _check_wildcard_routing(litellm_model): - wildcard_pattern = litellm_model - elif _check_wildcard_routing(model_name): - wildcard_pattern = model_name - else: + if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None: expanded.append(deployment) continue + if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names): + continue try: litellm_params = LiteLLM_Params.model_validate(litellm_params_dict) if litellm_params_dict else None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ed65ab71220..ae4f5072a26 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -18409,8 +18409,6 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(registry.deployments(source="alias")) all_models.extend(alias_models) - all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router)) - allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, llm_router=llm_router, @@ -18419,6 +18417,15 @@ async def model_info_v1( proxy_logging_obj=proxy_logging_obj, ) + all_models = await run_model_listing( + partial( + expand_wildcard_deployments_for_model_info, + all_models, + llm_router, + allowed_model_names=allowed_model_names, + ) + ) + all_models = _filter_v1_model_info_deployments( all_models=all_models, allowed_model_names=allowed_model_names, diff --git a/litellm/utils.py b/litellm/utils.py index 5a0fc18dfdc..4d3399d1d0c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10558,12 +10558,14 @@ class ProviderConfigManager: from litellm.llms.ollama.common_utils import OllamaModelInfo return OllamaModelInfo() - elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.vllm.common_utils import ( - VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py - ) + elif LlmProviders.VLLM == provider: + from litellm.llms.vllm.common_utils import VLLMModelInfo return VLLMModelInfo() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + + return HostedVLLMModelInfo() elif LlmProviders.LEMONADE == provider: return litellm.LemonadeChatConfig() elif LlmProviders.CLARIFAI == provider: diff --git a/tests/unit/llms/vllm/test_vllm_common_utils.py b/tests/unit/llms/vllm/test_vllm_common_utils.py new file mode 100644 index 00000000000..aa8a4852273 --- /dev/null +++ b/tests/unit/llms/vllm/test_vllm_common_utils.py @@ -0,0 +1,27 @@ +from typing import Final + +import httpx +import pytest +import respx + +from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + + +@respx.mock +def test_hosted_vllm_get_models_lists_the_server_ids_sending_the_key_as_a_bearer_token() -> None: + respx.get("http://vllm-sdk.test:8000/v1/models", headers={"Authorization": "Bearer vllm-sdk-key"}).mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}, {"id": "meta-llama/Llama-3.1-8B-Instruct"}]}) + ) + + listed: Final = HostedVLLMModelInfo().get_models(api_key="vllm-sdk-key", api_base="http://vllm-sdk.test:8000/v1") + + assert listed == ["qwen2.5", "meta-llama/Llama-3.1-8B-Instruct"] + + +def test_hosted_vllm_get_models_without_a_server_names_the_setting_that_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False) + + with pytest.raises(ValueError, match="HOSTED_VLLM_API_BASE"): + HostedVLLMModelInfo().get_models() diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index d0156ff0195..eef7e51982a 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1,11 +1,22 @@ -from collections.abc import Generator, Mapping +import datetime +import ipaddress +import json +import ssl +import threading +from collections.abc import Generator, Iterator, Mapping from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Final from unittest.mock import patch import httpx import pytest import respx +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID from litellm import Router @@ -430,6 +441,262 @@ def test_wildcard_credential_hydration_preserves_deployment_params( } +@respx.mock +def test_hosted_vllm_wildcard_lists_the_server_models_without_an_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get("http://vllm-without-key.test:8000/v1/models").mock( + return_value=httpx.Response( + 200, + json={ + "data": [ + {"id": "meta-llama/Llama-3.1-8B-Instruct"}, + {"id": "qwen2.5"}, + {"id": "openai/gpt-oss-20b"}, + ] + }, + ) + ) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-without-key.test:8000/v1"), + ) + + assert listed == [ + "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct", + "hosted_vllm/qwen2.5", + "hosted_vllm/openai/gpt-oss-20b", + ] + assert "authorization" not in models_route.calls.last.request.headers + + +@respx.mock +def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get( + "http://vllm-with-key.test:8000/v1/models", headers={"Authorization": "Bearer vllm-test-key"} + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-with-key.test:8000/v1", api_key="vllm-test-key"), + ) + + assert listed == ["hosted_vllm/qwen2.5"] + assert models_route.call_count == 1 + + +@respx.mock +def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key") + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000") + passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]}) + ) + respx.get( + "http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"} + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm") + ) + + assert listed == ["hosted_vllm/qwen2.5"] + assert not passthrough_server_route.called + + +@respx.mock +def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.constants import NON_LLM_CONNECTION_TIMEOUT + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + + get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1") + ) + + read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"] + assert read_timeout is not None + assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT + + +@respx.mock +def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get( + "http://vllm-behind-gateway.test:8000/v1/models", + headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"}, + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params( + model="hosted_vllm/*", + api_base="http://vllm-behind-gateway.test:8000/v1", + api_key="vllm-test-key", + extra_headers={"X-Tenant": "team-a"}, + ), + ) + + assert listed == ["hosted_vllm/qwen2.5"] + + + +@respx.mock +def test_vllm_wildcard_never_lists_the_ids_of_a_configured_vllm_server(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-library-env.test:8000") + monkeypatch.setenv("VLLM_API_KEY", "vllm-library-env-key") + server_route: Final = respx.get(url__regex=r"http://vllm-library(-env)?\.test:8000/.*").mock( + return_value=httpx.Response(200, json={"data": [{"id": "model-only-a-server-serves"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "vllm/*", + LiteLLM_Params(model="vllm/*", api_base="http://vllm-library.test:8000", api_key="vllm-library-key"), + ) + + assert "vllm/model-only-a-server-serves" not in listed + assert not server_route.called + +_SELF_SIGNED_CERT_NOT_VALID_BEFORE: Final = datetime.datetime(2000, 1, 1, tzinfo=datetime.UTC) +_SELF_SIGNED_CERT_NOT_VALID_AFTER: Final = datetime.datetime(9999, 12, 31, tzinfo=datetime.UTC) + + +def _self_signed_cert(directory: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(_SELF_SIGNED_CERT_NOT_VALID_BEFORE) + .not_valid_after(_SELF_SIGNED_CERT_NOT_VALID_AFTER) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False + ) + .sign(key, hashes.SHA256()) + ) + cert_path: Final = directory / "cert.pem" + key_path: Final = directory / "key.pem" + cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + return cert_path, key_path + + +@pytest.fixture +def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]: + cert_path, key_path = _self_signed_cert(tmp_path) + + class Handler(BaseHTTPRequestHandler): + timeout = 5 + + def do_GET(self) -> None: + body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_path, key_path) + server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"https://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + server.server_close() + + +def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify( + monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str +) -> None: + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx import http_handler + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + + verified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base) + ) + unverified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False), + ) + + assert verified == [] + assert unverified == ["hosted_vllm/qwen2.5"] + + +@respx.mock +def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get("http://upstream-proxy.test:4000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "team-proxy/*", + LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"), + ) + + assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"] + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. @@ -834,6 +1101,56 @@ def test_expand_wildcard_invalid_litellm_params_passthrough(): assert result == [deployment] + +@pytest.mark.parametrize( + ("allowed_model_names", "expected_names", "expected_listings"), + [ + pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"), + pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"), + pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"), + ], +) +@respx.mock +def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern( + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, + allowed_model_names: list[str] | None, + expected_names: list[str], + expected_listings: int, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + api_base: Final = f"http://vllm-alias-{request.node.callspec.id}.test:8000/v1" + listing: Final = respx.get(f"{api_base}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + deployment: Final = {"model_name": "qwen-*", "litellm_params": {"model": "hosted_vllm/*", "api_base": api_base}} + + rows: Final = expand_wildcard_deployments_for_model_info([deployment], allowed_model_names=allowed_model_names) + + assert ([row["model_name"] for row in rows], listing.call_count) == (expected_names, expected_listings) + + +@pytest.mark.parametrize( + ("allowed_model_names", "keeps_the_alias"), + [ + pytest.param(["gpt-4o-mini"], True, id="granted-a-name-the-alias-matches"), + pytest.param(["claude-sonnet"], False, id="granted-only-other-names"), + ], +) +def test_expand_wildcard_alias_over_one_model_is_kept_only_for_keys_with_a_name_it_matches( + allowed_model_names: list[str], keeps_the_alias: bool +) -> None: + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info + + deployment: Final = {"model_name": "gpt-4o*", "litellm_params": {"model": "openai/gpt-4o"}} + + rows: Final = expand_wildcard_deployments_for_model_info([deployment], allowed_model_names=allowed_model_names) + + assert rows == ([deployment] if keeps_the_alias else []) + def test_get_complete_model_list_excludes_wildcard_routes_by_default(): """Regression (LIT-4108): a wildcard with a matching router deployment leaked into /v1/models.""" from litellm import Router @@ -1327,6 +1644,40 @@ def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_ assert (listed, catalog.call_count) == ([], 0) +@respx.mock +def test_key_granted_hosted_vllm_wildcard_without_a_hosted_vllm_deployment_lists_no_environment_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm import Router + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-environment-only.test:8000/v1") + environment_server: Final = respx.get("http://hosted-vllm-environment-only.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + router: Final = Router( + model_list=[ + { + "model_name": "vllm-team/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-team.test:8000/v1"}, + } + ] + ) + + listed: Final = get_complete_model_list( + key_models=["hosted_vllm/*"], + team_models=[], + proxy_model_list=router.model_names, + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert (listed, environment_server.call_count) == ([], 0) + + @contextmanager def _openai_catalog_behind_a_catch_all( monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str] diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index bffa4612964..6ad5ef1a4eb 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import respx from fastapi.testclient import TestClient import litellm @@ -576,6 +577,55 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke assert router.registry().names("gpt-4o", kind="wildcard") == frozenset({"openai/gpt-4o"}) +@pytest.mark.parametrize( + ("key_models", "expected_names", "expected_listings"), + [(["qwen-open"], ["qwen-open"], 0), (["hosted_vllm/*"], ["hosted_vllm/qwen2.5"], 1)], + ids=["key-without-the-wildcard", "key-granted-the-wildcard"], +) +def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it( + client, auth_as, monkeypatch, request, key_models, expected_names, expected_listings +): + from litellm.proxy._types import LitellmUserRoles + + wildcard_server: Final = f"http://vllm-{request.node.callspec.id}.test:8000/v1" + other_alias_server: Final = f"http://vllm-secure-{request.node.callspec.id}.test:8000/v1" + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "qwen-open", + "litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"}, + }, + {"model_name": "hosted_vllm/*", "litellm_params": {"model": "hosted_vllm/*", "api_base": wildcard_server}}, + { + "model_name": "vllm-secure/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": other_alias_server}, + }, + ] + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list()) + monkeypatch.setattr(proxy_server, "user_model", None) + monkeypatch.setattr(proxy_server, "prisma_client", None) + + with respx.mock(assert_all_called=False) as upstream: + listing: Final = upstream.get(f"{wildcard_server}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + other_alias_listing: Final = upstream.get(f"{other_alias_server}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + with auth_as(role=LitellmUserRoles.INTERNAL_USER, models=key_models): + response: Final = client.get("/v1/model/info") + + assert response.status_code == 200, response.text + assert ( + [model["model_name"] for model in response.json()["data"]], + listing.call_count, + other_alias_listing.call_count, + ) == (expected_names, expected_listings, 0) + + # --------------------------------------------------------------------------- # GET /model/info — team BYOK scoping (issue #30983) # --------------------------------------------------------------------------- diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 19b3f317927..ed2f023dd11 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -9350,6 +9350,23 @@ def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_li assert (listings, catalog.call_count) == (([], []), 1) +@respx.mock +def test_get_valid_models_does_not_ask_a_timed_out_hosted_vllm_server_again_on_the_next_listing( + reset_mock_cache: None, +) -> None: + models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( + side_effect=httpx.ReadTimeout("vLLM did not answer") + ) + deployment: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1") + + listings: Final = tuple( + get_valid_models(check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=deployment) + for _ in range(2) + ) + + assert (listings, models_route.call_count) == (([], []), 1) + + @respx.mock def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None: respx.get("https://openrouter.ai/api/v1/models/user").mock( From 216041e24378243a104c4ab871a5fd8a73d5fcbe Mon Sep 17 00:00:00 2001 From: Devin Gupta <55953391+devin-gupta@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:47:02 -0800 Subject: [PATCH 06/34] feat(decisions): add Parallel AI web-grounded decisions (#49188) * feat(decisions): add Parallel AI web-grounded decisions * fix(decisions): name the Parallel model decision * fix(parallel_ai): restrict environment credentials to trusted decision endpoints * fix(decisions): keep Parallel evidence fields off other providers and handle missing confidence Only fields a provider names reach the OpenAI response, so existing providers keep their base shape. A malformed api_base answers 400 on both credential paths. The complexity router classifier and the Playground handle a provider that omits confidence. Regenerates the lazy OpenAPI snapshot and the dashboard API types * chore(cost-map): declare the Decisions routes on parallel_ai/decision The provider endpoint files are generated from the cost map, so the row must name both routes for the generated System One flag to hold * fix(decisions): report no classifier confidence when a provider omits it A choice answer with no confidence is a valid classifier answer, so the complexity router picks its tier and leaves classifier_confidence and the confidence signal out instead of reporting a different metric in its place. Also show the Parallel AI logo in the provider picker and add a Parallel translation fixture captured from the live API for both Decisions routes. * fix(decisions): keep a token-priced row on token billing when it also sets a query price A plain input_cost_per_query bills flat only on a row with no token price, and input_cost_per_query_decisions always bills flat, so a custom Decisions row that already bills tokens costs the same as before this PR * fix(decisions): bill a deployment's query price and accept Parallel bases that end in a route A proxy request logs through an object built before the router picks the deployment, so a query price on litellm_params never reached Decisions billing. Parallel Search accepts a base ending in /v1/search and shares PARALLEL_AI_API_BASE with Decisions, so Decisions now accepts that base and one ending in /v1/decisions. * fix(decisions): keep per-query billing and optional confidence on Parallel only A query price on any other Decisions provider moved that provider off token billing, and a choice or score answer with no confidence became a 200 with a null for every provider. Both now apply to Parallel alone, so other providers bill and fail as they do on main. A Parallel deployment with explicit zero token rates and a query price read as free, so every budget check was skipped while it billed per call. It now counts as priced. The dashboard key for Parallel is now PARALLEL_AI so a served bundle that predates the provider still sends parallel_ai as the provider slug, and the README table lists the provider. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- README.md | 1 + litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + litellm/cost_calculator.py | 9 +- litellm/decisions/call.py | 10 +- .../litellm_core_utils/decisions_pricing.py | 12 + .../llms/base_llm/decisions/transformation.py | 15 +- .../llms/openai/decisions/transformation.py | 8 +- .../llms/parallel_ai/decisions/__init__.py | 0 .../parallel_ai/decisions/transformation.py | 89 ++++ ...odel_prices_and_context_window_backup.json | 13 + .../provider_endpoints_support_backup.json | 2 +- litellm/proxy/_lazy_openapi_snapshot.json | 76 +++- litellm/proxy/auth/auth_checks.py | 17 +- .../provider_create_fields.json | 28 ++ .../complexity_router/complexity_router.py | 8 +- .../complexity_router/jev_classifier.py | 4 +- litellm/types/decisions.py | 12 +- litellm/types/utils.py | 1 + litellm/utils.py | 9 +- model_prices_and_context_window.json | 13 + provider_endpoints_support.json | 2 +- tests/integration/proxy_config.yaml | 5 + .../decisions/bases/parallel_ai.py | 257 +++++++++++ .../basic/test_decisions_basic_parallel_ai.py | 11 + .../test_decisions_pricing.py | 60 +++ .../test_base_decisions_transformation.py | 45 ++ .../test_openai_decisions_transformation.py | 37 +- .../llms/parallel_ai/decisions/__init__.py | 0 .../decisions/test_transformation.py | 423 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 1 + .../test_zero_cost_model_budget_bypass.py | 23 + .../router_strategy/test_complexity_router.py | 46 ++ .../ResponseView.integration.test.tsx | 14 + .../components/systemOneUI/ResponseView.tsx | 4 +- .../systemOneUI/lib/openAIDecisions.ts | 4 +- .../components/systemOneUI/lib/schemas.ts | 4 +- .../components/provider_info_helpers.test.tsx | 16 + .../src/components/provider_info_helpers.tsx | 5 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 8 +- 40 files changed, 1242 insertions(+), 55 deletions(-) create mode 100644 litellm/llms/parallel_ai/decisions/__init__.py create mode 100644 litellm/llms/parallel_ai/decisions/transformation.py create mode 100644 tests/integration/translation/decisions/bases/parallel_ai.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py create mode 100644 tests/unit/llms/parallel_ai/decisions/__init__.py create mode 100644 tests/unit/llms/parallel_ai/decisions/test_transformation.py diff --git a/README.md b/README.md index 3b5710fa0c5..c4aeee80fb4 100644 --- a/README.md +++ b/README.md @@ -444,6 +444,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) | | | | ✅ | | | | | | | | [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | | | | | | | | | [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ | | | | | | | | +| [Parallel AI (`parallel_ai`)](https://docs.litellm.ai/docs/search/parallel_ai) | | | | | | | | | | | | [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | | | | | | | | | [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | | | | | | | | | [Pinstripes (`pinstripes`)](https://docs.litellm.ai/docs/providers/pinstripes) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index baee4e9f7ba..041e3fc7837 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1754,6 +1754,9 @@ if TYPE_CHECKING: from .llms.typesafe.decisions.transformation import ( TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, ) + from .llms.parallel_ai.decisions.transformation import ( + ParallelAIDecisionsConfig as ParallelAIDecisionsConfig, + ) from .llms.openrouter.decisions.transformation import ( OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index c9949613303..65074093dbb 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -163,6 +163,7 @@ LLM_CONFIG_NAMES: Final = ( "HostedVLLMRerankConfig", "PerplexityDecisionsConfig", "TypeSafeDecisionsConfig", + "ParallelAIDecisionsConfig", "NaceDecisionsConfig", "OpenRouterDecisionsConfig", "FireworksAIDecisionsConfig", @@ -740,6 +741,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), "NaceDecisionsConfig": (".llms.nace.decisions.transformation", "NaceDecisionsConfig"), "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "ParallelAIDecisionsConfig": (".llms.parallel_ai.decisions.transformation", "ParallelAIDecisionsConfig"), "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 3c028278b2d..ef89a5ca8e7 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,7 +25,11 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount -from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type +from litellm.litellm_core_utils.decisions_pricing import ( + decisions_pricing_fields, + decisions_request_price, + is_decisions_call_type, +) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, @@ -605,6 +609,9 @@ def cost_per_token( decisions_overrides: Final = decisions_pricing_fields( litellm.model_cost.get(decisions_model_info["key"], {}) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # model cost map is dynamically loaded ) + request_price: Final = decisions_request_price(decisions_model_info, decisions_overrides) + if request_price is not None: + return request_price, 0.0 if decisions_overrides: return generic_cost_per_token( model=model, diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index 871fa8227ea..9c0e51e14ac 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -13,6 +13,7 @@ from typing_extensions import assert_never import litellm from litellm.cost_calculator import cost_map_key_prices_anything from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params +from litellm.litellm_core_utils.decisions_pricing import QUERY_PRICED_DECISIONS_PROVIDERS from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.image_handling import ( @@ -555,7 +556,7 @@ def prepare_call( resolved_api_base, static_api_key = provider_config.resolve_connection( dynamic_api_base or call_api_base, dynamic_api_key or api_key ) - except ValueError as error: + except (ValueError, httpx.InvalidURL) as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error if resolved_api_base is None: raise litellm.BadRequestError( @@ -613,6 +614,9 @@ def prepare_call( litellm_params={ "litellm_call_id": kwargs.get("litellm_call_id"), "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + "input_cost_per_query": ( + kwargs.get("input_cost_per_query") if provider in QUERY_PRICED_DECISIONS_PROVIDERS else None + ), }, custom_llm_provider=provider, ) @@ -787,7 +791,9 @@ def _formatted_response( case DecisionsRequestBody(): return ir_to_systemone_response(caller_response, caller_request) case OpenAIDecisionRequestBody(): - return ir_to_openai_response(caller_response, caller_request, call.requested_model) + return ir_to_openai_response( + caller_response, caller_request, call.requested_model, call.provider_config.openai_evidence_fields + ) case _: assert_never(call.request) diff --git a/litellm/litellm_core_utils/decisions_pricing.py b/litellm/litellm_core_utils/decisions_pricing.py index 9199a8097cc..accd315da23 100644 --- a/litellm/litellm_core_utils/decisions_pricing.py +++ b/litellm/litellm_core_utils/decisions_pricing.py @@ -12,6 +12,7 @@ from typing import Final DECISIONS_PRICING_SUFFIX: Final = "_decisions" DECISIONS_CALL_TYPES: Final = frozenset({"decisions", "adecisions", "systemone", "asystemone"}) +QUERY_PRICED_DECISIONS_PROVIDERS: Final = frozenset({"parallel_ai"}) def is_decisions_call_type(call_type: str) -> bool: @@ -27,6 +28,17 @@ def decisions_pricing_fields(row: Mapping[str, object]) -> Mapping[str, object]: } +def decisions_request_price(model_info: Mapping[str, object], overrides: Mapping[str, object]) -> float | None: + if model_info.get("litellm_provider") not in QUERY_PRICED_DECISIONS_PROVIDERS: + return None + rates: Final = {**model_info, **overrides} + request_price: Final = rates.get("input_cost_per_query") + if not isinstance(request_price, (int, float)): + return None + bills_tokens: Final = any(rates.get(field) for field in ("input_cost_per_token", "output_cost_per_token")) + return None if bills_tokens and "input_cost_per_query" not in overrides else float(request_price) + + def apply_decisions_pricing(row: Mapping[str, object]) -> Mapping[str, object]: """Return ``row`` priced for a Decisions call: ``_decisions`` values replace their plain fields.""" plain: Final = {key: value for key, value in row.items() if not key.endswith(DECISIONS_PRICING_SUFFIX)} diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index c562c8d0f04..254783edf39 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -431,6 +431,10 @@ class DecisionsResponseValidationError(BaseLLMException): pass +def _lacks_confidence(answer: DecisionsIRAnswer) -> bool: + return isinstance(answer, (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) and answer.confidence is None + + def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: return systemone_response_to_ir( _SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request @@ -534,6 +538,8 @@ class BaseDecisionsConfig(ABC): max_total_image_bytes: int | None = None image_media_types: frozenset[str] | None = None inlines_remote_images: bool = False + openai_evidence_fields: frozenset[str] = frozenset() + confidence_optional: bool = False health_check_questions: Mapping[str, Mapping[str, object]] = MappingProxyType( {"reachable": MappingProxyType({"type": "noul", "instructions": "Is the service reachable?"})} ) @@ -682,13 +688,20 @@ class BaseDecisionsConfig(ABC): ) -> DecisionsIRResponse: try: payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) - return self.parse_response(payload, request) + response: Final = self.parse_response(payload, request) except ValidationError as error: raise DecisionsResponseValidationError( status_code=500, message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " f"{error.title} {_validation_summary(error)}", ) from error + if not self.confidence_optional and any(_lacks_confidence(answer) for answer in response.answers): + raise DecisionsResponseValidationError( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: " + "a choice or score answer has no confidence", + ) + return response def get_error_class( self, diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 369b6959411..5aa72579d6a 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -307,10 +307,13 @@ def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> def ir_to_openai_response( - response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str + response: DecisionsIRResponse, + request: DecisionsIRRequest, + requested_model: str, + evidence_fields: frozenset[str] = frozenset(), ) -> OpenAIDecisionResponse: usage: Final = response.usage or DecisionsIRUsage() - return OpenAIDecisionResponse( + formatted: Final = OpenAIDecisionResponse( model=response.model or requested_model, answers=tuple( _openai_answer(question, answer) @@ -327,6 +330,7 @@ def ir_to_openai_response( total_tokens=usage.input_tokens + usage.output_tokens, ), ) + return formatted.model_copy(update={name: response.extra[name] for name in evidence_fields & response.extra.keys()}) class OpenAIDecisionsConfig(BaseDecisionsConfig): diff --git a/litellm/llms/parallel_ai/decisions/__init__.py b/litellm/llms/parallel_ai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/parallel_ai/decisions/transformation.py b/litellm/llms/parallel_ai/decisions/transformation.py new file mode 100644 index 00000000000..34d03047301 --- /dev/null +++ b/litellm/llms/parallel_ai/decisions/transformation.py @@ -0,0 +1,89 @@ +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict + +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, ir_to_systemone_request +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsIRResponse, + UnsupportedDecisionsRequest, +) + + +class _ParallelResponse(BaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + mode: str + answers: Mapping[str, Mapping[str, object]] + + +class ParallelAIDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PARALLEL_AI_API_KEY", "PARALLEL_API_KEY") + api_base_env = ("PARALLEL_AI_API_BASE",) + confidence_optional = True + openai_evidence_fields = frozenset({"citations", "decision_id", "mode"}) + + def get_default_api_base(self) -> str: + return "https://api.parallel.ai" + + def resolve_connection(self, api_base: str | None, api_key: str | None) -> tuple[str | None, str | None]: + configured_base: Final = self.resolve_api_base(None) + resolved_base: Final = api_base or configured_base + destination: Final = self._decisions_url(resolved_base) + if api_key: + return resolved_base, api_key + if destination not in {self._decisions_url(None), self._decisions_url(configured_base)}: + raise ValueError( + "Parallel Decisions requires an explicit api_key for a custom api_base unless " + "the destination is configured in PARALLEL_AI_API_BASE" + ) + return resolved_base, self.resolve_api_key(None) + + def get_complete_url(self, api_base: str, model: str) -> str: + return super().get_complete_url(api_base.rstrip("/").removesuffix("/v1/search").removesuffix(self.path), model) + + def _decisions_url(self, api_base: str | None) -> httpx.URL: + return httpx.URL(self.get_complete_url(api_base or self.get_default_api_base(), "decision")) + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + return { + **{ + name: value + for name, value in headers.items() + if name.lower() not in {"authorization", "x-api-key", "content-type"} + }, + **({"x-api-key": api_key} if api_key is not None else {}), + "Content-Type": "application/json", + } + + def transform_decisions_request( + self, model: str, request: DecisionsIRRequest, custom_llm_provider: str + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if model != "decision": + return UnsupportedDecisionsRequest(reason="Parallel Decisions supports the decision model") + if len(request.questions) > 100: + return UnsupportedDecisionsRequest(reason="Parallel Decisions supports at most 100 questions") + body: Final = ir_to_systemone_request(model, request) + if isinstance(body, UnsupportedDecisionsRequest): + return body + return {"mode": "fast", **{key: value for key, value in body.items() if key != "model"}} + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + response: Final = _ParallelResponse.model_validate(payload) + return super().parse_response( + { + **response.model_dump(), + "model": "decision", + "answers": {key: answer for key, answer in response.answers.items() if answer.get("type") != "refusal"}, + }, + request, + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 032b7d17423..75364a382f7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -46412,6 +46412,19 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "parallel_ai/decision": { + "input_cost_per_query": 0.001, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "max_input_tokens": 32000, + "mode": "evaluation", + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api", + "supported_endpoints": [ + "/v1/decisions", + "/v1/systemone" + ] + }, "parallel_ai/search": { "input_cost_per_query": 0.005, "litellm_provider": "parallel_ai", diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 700a35edb06..40ea9d15ff4 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -3158,7 +3158,7 @@ "batches": false, "messages": false, "interactions": false, - "systemone": false, + "systemone": true, "a2a": false, "generateContent": false } diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 5a80583b207..ba6494c3bda 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10237,8 +10237,15 @@ "type": "string" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "probabilities": { "additionalProperties": { @@ -10256,7 +10263,6 @@ "required": [ "type", "choice", - "confidence", "probabilities" ], "title": "ChoiceAnswer", @@ -10687,8 +10693,15 @@ "title": "Choice" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "name": { "anyOf": [ @@ -10718,8 +10731,7 @@ "required": [ "name", "choice", - "probabilities", - "confidence" + "probabilities" ], "title": "OpenAIChoiceAnswer", "type": "object" @@ -10924,8 +10936,15 @@ "OpenAIScoreAnswer": { "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "name": { "anyOf": [ @@ -10959,8 +10978,7 @@ "required": [ "name", "score", - "probabilities", - "confidence" + "probabilities" ], "title": "OpenAIScoreAnswer", "type": "object" @@ -10992,8 +11010,15 @@ "additionalProperties": true, "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "legend": { "additionalProperties": { @@ -11034,7 +11059,6 @@ "required": [ "type", "score", - "confidence", "legend", "probabilities" ], @@ -23542,8 +23566,15 @@ "type": "string" }, "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "probabilities": { "additionalProperties": { @@ -23561,7 +23592,6 @@ "required": [ "type", "choice", - "confidence", "probabilities" ], "title": "ChoiceAnswer", @@ -29400,8 +29430,15 @@ "additionalProperties": true, "properties": { "confidence": { - "title": "Confidence", - "type": "number" + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" }, "legend": { "additionalProperties": { @@ -29442,7 +29479,6 @@ "required": [ "type", "score", - "confidence", "legend", "probabilities" ], diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fdc8aa82268..9d98e2c9973 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -43,7 +43,9 @@ from litellm.constants import ( REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.dd_tracing import tracer +from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, decisions_request_price from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.tool_fields import DropPolicy @@ -730,7 +732,7 @@ def _is_cost_explicitly_configured(deployments: Sequence[DeploymentTypedDict]) - def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> bool: - """Whether any of the deployments carries an explicit positive per-token price. + """Whether any of the deployments carries an explicit positive per-token or per-query price. The group's price is read from the deployments named after the routed group and from that group's own alias target, while an alias chain is served from the deployments named @@ -741,14 +743,21 @@ def _has_explicit_positive_cost(deployments: Sequence[DeploymentTypedDict]) -> b model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id") if model_id is None: continue - raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY) - if _is_positive_cost(raw_entry.get("input_cost_per_token")) or _is_positive_cost( - raw_entry.get("output_cost_per_token") + raw_entry = as_str_mapping(litellm.model_cost.get(model_id)) or _EMPTY_COST_ENTRY + if ( + _is_positive_cost(raw_entry.get("input_cost_per_token")) + or _is_positive_cost(raw_entry.get("output_cost_per_token")) + or _is_positive_cost(_deployment_query_price(deployment["litellm_params"], raw_entry)) ): return True return False +def _deployment_query_price(litellm_params: Mapping[str, object], cost_entry: Mapping[str, object]) -> float | None: + provider: Final = litellm_params.get("custom_llm_provider") or str(litellm_params.get("model")).partition("/")[0] + return decisions_request_price({**cost_entry, "litellm_provider": provider}, decisions_pricing_fields(cost_entry)) + + _EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({}) diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 044b8a1de6e..aa7554c4a5a 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2641,6 +2641,34 @@ ], "default_model_placeholder": "nace/drex-latest" }, + { + "provider": "PARALLEL_AI", + "provider_display_name": "Parallel AI", + "litellm_provider": "parallel_ai", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.parallel.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "parallel_ai/decision" + }, { "provider": "NEBIUS", "provider_display_name": "Nebius", diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 4f4ed76c170..05dd4706b62 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -1219,9 +1219,11 @@ def _with_classifier_forecast( forecasted_decision: Final[StandardLoggingRoutingDecision] = { **with_input, "classifier_probabilities": outcome.jev_verdict.probabilities, - "classifier_confidence": outcome.jev_verdict.confidence, } - return forecasted_decision + confidence: Final = outcome.jev_verdict.confidence + return ( + forecasted_decision if confidence is None else {**forecasted_decision, "classifier_confidence": confidence} + ) if outcome.llm_v2_forecast is not None: return _with_llm_v2_forecast(with_input, outcome.llm_v2_forecast) forecast: Final = outcome.capability_forecast @@ -2468,7 +2470,7 @@ class ComplexityRouter(CustomLogger): score=None, signals=( f"{config.provider}-classifier:{tier_name}", - f"{config.provider}-confidence={answer.confidence:.6f}", + *(() if answer.confidence is None else (f"{config.provider}-confidence={answer.confidence:.6f}",)), *( f"tier-probability:{label}={probability:.6f}" for label, probability in answer.probabilities.items() diff --git a/litellm/router_strategy/complexity_router/jev_classifier.py b/litellm/router_strategy/complexity_router/jev_classifier.py index bb7d2e91cd3..5cd4f00a941 100644 --- a/litellm/router_strategy/complexity_router/jev_classifier.py +++ b/litellm/router_strategy/complexity_router/jev_classifier.py @@ -210,7 +210,7 @@ class JevChoiceAnswer(LiteLLMBaseModel): type: Literal["choice"] choice: str probabilities: Mapping[str, JevProbability] - confidence: JevProbability + confidence: JevProbability | None = None class JevUsage(LiteLLMBaseModel): @@ -425,7 +425,7 @@ class HttpJevClassifierClient: class JevVerdict(NamedTuple): label: str probabilities: Mapping[str, float] - confidence: float + confidence: float | None model: str cost: float | None provider: str = "typesafe" diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 33b2c677d31..3b28225f0ff 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -210,7 +210,7 @@ class NoulAnswer(LiteLLMPydanticObjectBase): class ChoiceAnswer(LiteLLMPydanticObjectBase): type: Literal["choice"] choice: str - confidence: float + confidence: float | None = None probabilities: Mapping[str, float] model_config = ConfigDict(extra="allow", frozen=True) @@ -219,7 +219,7 @@ class ChoiceAnswer(LiteLLMPydanticObjectBase): class ScoreAnswer(LiteLLMPydanticObjectBase): type: Literal["score"] score: float - confidence: float + confidence: float | None = None legend: Mapping[str, DecisionsJSON] probabilities: Mapping[str, float] @@ -431,7 +431,7 @@ class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): name: str | None choice: str | bool probabilities: tuple[OpenAIChoiceProbability, ...] - confidence: float + confidence: float | None = None model_config = ConfigDict(frozen=True) @@ -449,7 +449,7 @@ class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): name: str | None score: float probabilities: tuple[OpenAIScoreProbability, ...] - confidence: float + confidence: float | None = None model_config = ConfigDict(frozen=True) @@ -595,7 +595,7 @@ class DecisionsIRChoiceProbability: @dataclass(frozen=True, slots=True) class DecisionsIRChoiceAnswer: choice: str | bool - confidence: float + confidence: float | None probabilities: tuple[DecisionsIRChoiceProbability, ...] extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) @@ -610,7 +610,7 @@ class DecisionsIRScoreProbability: @dataclass(frozen=True, slots=True) class DecisionsIRScoreAnswer: score: float - confidence: float + confidence: float | None probabilities: tuple[DecisionsIRScoreProbability, ...] extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 080ea9848b6..e633e557bb2 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4580,6 +4580,7 @@ class LlmProviders(str, Enum): CLF_AI_GATEWAY = "clf_ai_gateway" PERPLEXITY = "perplexity" TYPESAFE = "typesafe" + PARALLEL_AI = "parallel_ai" NACE = "nace" STRANDS_DECIDER = "strands_decider" LAYA = "laya" diff --git a/litellm/utils.py b/litellm/utils.py index 4d3399d1d0c..ddfc38ca677 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10105,12 +10105,19 @@ class ProviderConfigManager: return litellm.LayaDecisionsConfig() if provider == LlmProviders.BESPOKE: return litellm.BespokeDecisionsConfig() - if provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI, LlmProviders.EU_JEV): + if provider in ( + LlmProviders.OPENAI, + LlmProviders.AZURE, + LlmProviders.AZURE_AI, + LlmProviders.EU_JEV, + LlmProviders.PARALLEL_AI, + ): configs: Final = { LlmProviders.OPENAI: litellm.OpenAIDecisionsConfig, LlmProviders.AZURE: litellm.AzureDecisionsConfig, LlmProviders.AZURE_AI: litellm.AzureAIDecisionsConfig, LlmProviders.EU_JEV: litellm.EuJevDecisionsConfig, + LlmProviders.PARALLEL_AI: litellm.ParallelAIDecisionsConfig, } return configs[provider]() return None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 032b7d17423..75364a382f7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -46412,6 +46412,19 @@ "supports_response_schema": true, "supports_tool_choice": false }, + "parallel_ai/decision": { + "input_cost_per_query": 0.001, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "parallel_ai", + "max_input_tokens": 32000, + "mode": "evaluation", + "source": "https://docs.parallel.ai/getting-started/pricing#decisions-api", + "supported_endpoints": [ + "/v1/decisions", + "/v1/systemone" + ] + }, "parallel_ai/search": { "input_cost_per_query": 0.005, "litellm_provider": "parallel_ai", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 700a35edb06..40ea9d15ff4 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -3158,7 +3158,7 @@ "batches": false, "messages": false, "interactions": false, - "systemone": false, + "systemone": true, "a2a": false, "generateContent": false } diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 4e2cab56d27..c552e11c22a 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -587,6 +587,11 @@ model_list: model: perplexity/pplx-decider-v1-27b api_base: http://127.0.0.1:8191 api_key: synthetic-perplexity-key + - model_name: parallel_ai/decision + litellm_params: + model: parallel_ai/decision + api_base: http://127.0.0.1:8191 + api_key: synthetic-parallel-key - model_name: typesafe/jev-1.13.0 litellm_params: model: typesafe/jev-1.13.0 diff --git a/tests/integration/translation/decisions/bases/parallel_ai.py b/tests/integration/translation/decisions/bases/parallel_ai.py new file mode 100644 index 00000000000..14e9cd6fc43 --- /dev/null +++ b/tests/integration/translation/decisions/bases/parallel_ai.py @@ -0,0 +1,257 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase +from pydantic import JsonValue + +"""Provider request and reply shape from https://docs.parallel.ai/api-reference/decisions/make-web-grounded-typed-decisions. Mock reply captured live on 2026-10-31. +""" +_CITATIONS: Final[list[JsonValue]] = [ + { + "url": "https://access.redhat.com/solutions/7108129", + "title": 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg. - Red Hat Customer Portal', + "excerpts": [ + 'NFS client hangs and "nfs4_handle_delegation_recall_error" found in dmesg.\n\nSolution Verified - Updated 2026-06-18T01:07:40+00:00 -\nEnglish\n* No translations currently exist.\n\nIssue\n\n* NFS client hangs while returning delegations.\n* Below delegation recall errors may be reported in dmesg:\nNFS: nfs4_handle_delegation_recall_error: unhandled error -512\nNFS: nfs4_handle_delegation_recall_error: unhandled error -10024\n* We have a hanging process, it\'s using data ontop of a mounted NFS export. The fcntl() system call issued by a user process has not returned, The user process processing does not finish - usually it finishes within 2 minutes.\n\nEnvironment\n\n* Red Hat Enterprise Linux (RHEL) 8, all minor releases\n* kernel:\n* rhel8.4: kernels before 4.18.0-305.194.1.el8_4 affected\n* rhel8.10: kernels before 4.18.0-553.37.1.el8_10 affected\n* NFSv4' + ], + }, + { + "url": "https://github.com/moby/buildkit/issues/2804", + "title": "GitHub cache exporter (`type=gha`) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. · Issue #2804 · moby/buildkit · GitHub", + "excerpts": [ + 'Copy link\nOpen\nOpen\nGitHub cache exporter ( type=gha ) hangs reletively low number of cache reads/writes, satirating api limits/triggering throttling. #2804\nCopy link\n\nDescription\n\n@bendavies\nbendavies\nopened on Apr 13, 2022\nIssue body actions\n\nBehaviour\n\nCache operations using type=gha will start hanging after a relavatively low number of cache reads/writes.\nWe are a small team (5 devs) and are hitting these limits constantly on our main repository. Not exactly high throughput.\n\n...\n\nCache reads and writes go towards saturating the limit, and both are affected by this throttling.\n@dhadka mentions there is a limit of about "150 cache requests in 5 minutes", and seems about right: #2276 (comment)\n\nExpected behaviour\n\nI don\'t know, but the current behaviour of hanging is not great. there is no feedback as to what is hanging resulting in the user just being confused.' + ], + }, + { + "url": "https://github.com/huginn/huginn/issues/2963", + "title": "WebsiteAgent hangs on a specific url, never finishes job or times out · Issue #2963 · huginn/huginn · GitHub", + "excerpts": [ + 'WebsiteAgent hangs on a specific url, never finishes job or times out\n\n* Page: GitHub issue\n* URL: https://github.com/huginn/huginn/issues/2963\n* State: open\n* Author: BoKKeR\n* Created: 2021-03-14T16:42:01Z\n* Updated: 2021-03-16T19:09:24Z\n* Repository: huginn/huginn\n* Number: #2963\n* Comments: 1\nI know it might be controversial but I am trying to get a new GPU for a while. Wanted to setup a monitor for when the stock on amd.com changes. But the site just won\'t load. The WebsiteAgent gets stuck in\nFetching https://www.amd.com/de/direct-buy/se\nThe only way to stop it is by stopping the container. As I am running the service in a container I cant find any of the debug log locations. Can anyone else try if they are encountering the same error?\n{\n "debug": "true",\n "expected_update_period_in_days": "2",\n "url": "https://www.amd.com/de/direct-buy/se",\n "type": "html",' + ], + }, + { + "url": "https://github.com/mozilla-firefox/firefox-crb/blob/main/docs/bug-mgmt/guides/severity.md", + "title": "firefox-crb/docs/bug-mgmt/guides/severity.md at main · mozilla-firefox/firefox-crb · GitHub", + "excerpts": [ + "We use the severity field in Bugzilla to indicate the scope of a bug's effect on Firefox.\nThe field is display alongside the bug's priority.\n\nValues\n\nSeverity levels and their definitions are enumerated at https://wiki.mozilla.org/BMO/UserGuide/BugFields#bug_severity.\nBy default, new bugs have a severity of --.\n\nExamples of S1 bugs\n\n* WebExtensions disabled for all users\n* Web search not working from URL bar\n* Crashes with data loss\n\nExamples of S2 bugs\n\nBugs that could reasonably be expected to cause a Firefox user to switch browsers, either because the severity is bad enough, or the frequency of occurrence is high enough.\n* Bug 1640441 - Slack hangs indefinitely in a onResize loop" + ], + }, + { + "url": "https://github.com/n0-computer/iroh/issues/4604", + "title": "Endpoint::close hangs forever if the tokio runtime gets dropped while the connection is open · Issue #4604 · n0-computer/iroh · GitHub", + "excerpts": [ + "@alpergundogdu\nalpergundogdu\nopened on Oct 14, 2026\nIssue body actions\nDescribe the bug\nAfter opening a connection from inside a short-lived tokio runtime, and dropping the runtime while the connection is still open, Endpoint::close() never returns.\nIn my repro it hangs reliably; when I keep the runtime open, it closes instantly.\nLocal repro/test (has the control tests too): https://github.com/decdn/iroh-tests/blob/main/tests/bug_endpoint_close_after_runtime_drop.rs\n\n...\n\nIf spawning on the caller's runtime is intended, a note on Endpoint::connect saying the driver lives there would have saved us a lot of time. (Maybe it's an expectation/documentation issue?)\nIroh\nVersion: iroh 1.3.0, noq 1.3.0\nEndpoint configuration: bound to 127.0.0.1, no relays.\nPlatform(s)\nDesktop:\n\n...\n\nSorry @flub , that was unclear: the endpoint's runtime stays alive, and close() is called from it. Only a separate short-lived runtime that runs connect() gets dropped." + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1640441", + "title": "1640441 - Slack hangs indefinitely in a onResize loop", + "excerpts": [ + "Closed Bug 1640441 Opened 6 years ago Closed 6 years ago\n\nSlack hangs indefinitely in a on Resize loop\n\nCategories\n\n(Web Compatibility :: Site Reports, defect, P1)\n\nProduct:\nWeb Compatibility ▾\nComponent:\nSite Reports ▾\nType:\ndefect\nPriority:\nP1 Severity:\nS2\n\nTracking\n\n(Not tracked)\n\nStatus:\nRESOLVED FIXED\n\nPeople\n\n(Reporter: Harald, Assigned: emilio)\n\nReferences\n\n(Depends on 1 open bug)\n\nDetails\n\n(Keywords: enterprise, perf, webcompat:site-wait)\n\nAttachments\n\n(1 file, 1 obsolete file)\n\n|relevant unminified code.\n6 years ago Emilio Cobos Álvarez [:emilio]\n* 13.50 KB,\n\n...\n\n* Hover over the link\n* ... tab immediately becomes unresponsive ...\nThis may well be a bug in slack, but IMO it's still on us to try and fix what's going on. When it happens, it affects not only Slack but my entire Firefox session. This is a serious usability issue -- I'm now at the point of restarting Firefox several times a day because of this bug, if" + ], + }, + { + "url": "https://www.ibm.com/support/pages/cognos-business-intelligence-1022-fix-lists", + "title": "Cognos Business Intelligence 10.2.2 Fix Lists", + "excerpts": [ + 'IBM Support \n\nCognos Business Intelligence 10.2.2 Fix Lists\n\nRelease Notes\n\nAbstract\n\nA comprehensive list of defect corrections for major releases, refresh packs and fix packs of Cognos Business Intelligence 10.2.2.\nDetails of the APARs listed below can be accessed by clicking the link for the APAR number. If you have questions about a particular defect, please contact Customer Support.\n\n...\n\n* PI05143 | MULTIPLE VALUES IN CONTEXT FILTERS CAUSE A CROSSTAB REPORT TO SHOW DASH "-" INSTEAD OF CORRECT VALUES\n* PI05166 | QE-DEF-0359 The query contains a reference to at least one object that does not exist\n* PI05233 | Export deployments from 8.4.1 with export folder or package tasks in Content Administration contain unknown content steps\n\n...\n\n* PI08523 | Progressive Chart - Incorrect Colours when "Reverse Catagory Order" = True\n* PI08585 | Cognos hangs when starting against a Content Store from previous version (upgrade) and using SSL gateway UR' + ], + }, + { + "url": "https://josm.eu/ticket/8175", + "title": "\n #8175 ([PATCH] Cadastre grab query sometimes hangs completely)\n – JOSM\n\n ", + "excerpts": [ + 'Opened 14 years ago\nClosed 14 years ago\nLast modified 14 years ago\n\n#8175 closed defect ( fixed )\n\n[PATCH] Cadastre grab query sometimes hangs completely\n\n* Reported by: | Don-vip | Owned by: | pieren\n* Priority: | normal | Milestone: | \n* Component: | Plugin cadastre-fr | Version: | \n* Keywords: | | Cc: | \n\nDescription\n\nDuring cadastre grab, the HTTP connection with the WMS server sometimes completely hangs (the progress dialog is frozen and cancel button does not work).\nThe only solution is to restart JOSM to have a clean state and grab the cadastre again. We had a similar problem with history request ( #7640 ) when dealing with the API and solved it in [r5324](https://josm.eu/changeset/5324/josm "fix #7640 - History query will not cancel or timeout\n\n...\n\nby Don-vip , 14 years ago\n\n* Attachment: | 8175.patch \u200b added\n\ncomment:1 by Don-vip , 14 years ago' + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1749908", + "title": "1749908 - Infinite loop in HTTP3 hangs socket thread - Bugzilla", + "excerpts": [ + "Closed Bug 1749908 Opened 4 years ago Closed 4 years ago\n\nInfinite loop in HTTP3 hangs socket thread\n\nCategories\n\n(Core :: Networking, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking ▾\nPlatform:\nDesktop\nAll\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nVERIFIED DUPLICATE of bug 1749910\n\nPeople\n\n(Reporter: heftig, Unassigned)\n\nDetails\n\nBottom ↓\nTags ▾\nTimeline ▾\n| |Jan Alexander Steffens [:heftig]\n* Reporter | \n* ### Description\n•\n* 4 years ago\n* \nWhile browsing, pages suddenly stopped loading.\n\n...\n\nIt's not related to a specific version, we're getting reports ESR is even affected. Suspicion is around some long existing HTTP3 bug that's being triggered by an external service updating.\n\n...\n\nI agree it is not the root cause but it certainly caused a massive problem for many users of Firefox, the majority of whom are opted-in by default. If they had not been, they would not have been affected." + ], + }, + { + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1748414", + "title": "1748414 - Firefox 94 sometimes hangs on TLSv1.3 requests", + "excerpts": [ + "Closed Bug 1748414 Opened 4 years ago Closed 4 years ago\n\nFirefox 94 sometimes hangs on TLSv1 .3 requests\n\nCategories\n\n(Core :: Networking: HTTP, defect)\n\nProduct:\nCore ▾\nComponent:\nNetworking: HTTP ▾\nVersion:\nFirefox 94\nType:\ndefect\nPriority:\nNot set Severity:\n--\n\nTracking\n\n( )\n\nStatus:\nRESOLVED DUPLICATE of bug 1740856\n\nPeople\n\n(Reporter: code, Unassigned)\n\nDetails\n\nBottom ↓\n\n...\n\nSteps to reproduce:\nIn the beginning of November, colleagues started to experience hangs of 1 to 1.5 minutes when working with our own web app. We use latest official releases for Windows, Ubuntu and FreeBSD. The problems exist for all plattforms and very likely started with Firefox 94.\nActual results:\nSometimes while interacting normally with our web app, a request hangs for a long time (up to 1.5 minutes; but usually succeeds). This typically happens on POST requests." + ], + }, +] +DECISION_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "parallel_ai/decision", + "input": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": [ + {"type": "predicate", "name": "defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "severity", + "instructions": "How severe is it?", + "choices": [ + {"value": "low", "description": "cosmetic"}, + {"value": "high", "description": "blocks users"}, + ], + }, + { + "type": "score", + "name": "confidence", + "instructions": "How sure are you?", + "levels": [{"label": "unsure"}, {"label": "sure"}], + }, + ], + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"}, + expected_provider_request={ + "mode": "fast", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "mode": "fast", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, + expected_litellm_response={ + "model": "decision", + "answers": [ + {"type": "predicate", "name": "defect", "probability": 0.6442559258537284}, + { + "type": "choice", + "name": "severity", + "choice": "high", + "probabilities": [ + {"value": "low", "probability": 0.1616964510265985}, + {"value": "high", "probability": 0.8383035489734014}, + ], + "confidence": 0.6766070979468028, + }, + { + "type": "score", + "name": "confidence", + "score": 0.25195637565890455, + "probabilities": [ + {"value": 0, "label": "unsure", "probability": 0.7480436243410955}, + {"value": 1, "label": "sure", "probability": 0.25195637565890455}, + ], + "confidence": 0.4960872486821909, + }, + ], + "usage": { + "input_tokens": 3166, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 3166, + }, + "mode": "fast", + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, +) +DECISION_SYSTEMONE_TEST_CASE: Final = TranslationTestCase( + scenario="systemone", + litellm_endpoint="/v1/systemone", + litellm_request={ + "model": "parallel_ai/decision", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"x-api-key": "synthetic-parallel-key", "content-type": "application/json"}, + expected_provider_request={ + "mode": "fast", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "mode": "fast", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, + expected_litellm_response={ + "model": "decision", + "answers": { + "defect": {"type": "noul", "noul": 0.6442559258537284}, + "severity": { + "type": "choice", + "choice": "high", + "probabilities": {"low": 0.1616964510265985, "high": 0.8383035489734014}, + "confidence": 0.6766070979468028, + }, + "confidence": { + "type": "score", + "score": 0.25195637565890455, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.7480436243410955, "1": 0.25195637565890455}, + "confidence": 0.4960872486821909, + }, + }, + "usage": {"input_tokens": 3166, "output_tokens": 0}, + "mode": "fast", + "citations": _CITATIONS, + "decision_id": "decisions_9fd99def328ed16256c11df297dea899", + }, +) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py new file mode 100644 index 00000000000..47eb057323b --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_parallel_ai.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.parallel_ai import DECISION_SYSTEMONE_TEST_CASE, DECISION_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [DECISION_TEST_CASE, DECISION_SYSTEMONE_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_parallel_ai(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/litellm_core_utils/test_decisions_pricing.py b/tests/unit/litellm_core_utils/test_decisions_pricing.py index e9814b980d1..d9603230319 100644 --- a/tests/unit/litellm_core_utils/test_decisions_pricing.py +++ b/tests/unit/litellm_core_utils/test_decisions_pricing.py @@ -62,6 +62,66 @@ def test_apply_decisions_pricing_leaves_rows_without_decisions_fields_unchanged( assert apply_decisions_pricing(plain_row) == plain_row +@pytest.mark.parametrize("rate", [0.0, 0.002]) +def test_flat_decisions_rate_overrides_catalog_without_changing_chat( + monkeypatch: pytest.MonkeyPatch, rate: float +) -> None: + model: Final = "parallel_ai/decision" + monkeypatch.setitem( + litellm.model_cost, + model, + {**_ROW, "litellm_provider": "parallel_ai", "input_cost_per_query_decisions": rate}, + ) + assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (rate, 0.0) + assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="completion") == pytest.approx( + (100 * 1e-07, 0.0) + ) + + +@pytest.mark.parametrize("query_field", ["input_cost_per_query", "input_cost_per_query_decisions"]) +def test_query_price_on_a_provider_that_bills_tokens_stays_unbilled( + monkeypatch: pytest.MonkeyPatch, query_field: str +) -> None: + model: Final = "fireworks_ai/accounts/example/models/query-priced" + monkeypatch.setitem( + litellm.model_cost, + model, + {query_field: 0.5, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0} + | {"litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + assert cost_calculator.cost_per_token(model=model, prompt_tokens=100, call_type="decisions") == (0.0, 0.0) + + +@pytest.mark.parametrize("call_type", ["adecisions", "asystemone"]) +@pytest.mark.parametrize( + ("token_rates", "expected_costs"), + [ + ({"input_cost_per_token": 1e-05, "output_cost_per_token": 0.0}, (100 * 1e-05, 0.0)), + ({"input_cost_per_token": 0.0, "output_cost_per_token": 1e-05}, (0.0, 10 * 1e-05)), + ], +) +def test_plain_query_price_leaves_a_token_priced_row_billing_tokens( + monkeypatch: pytest.MonkeyPatch, + call_type: str, + token_rates: dict[str, float], + expected_costs: tuple[float, float], +) -> None: + model: Final = "fireworks_ai/accounts/example/models/priced-both" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_query": 0.5, **token_rates, "litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + + assert cost_calculator.cost_per_token( + model=model, + custom_llm_provider="fireworks_ai", + prompt_tokens=100, + completion_tokens=10, + call_type=cast(CallTypesLiteral, call_type), + ) == pytest.approx(expected_costs) + + _LUNA_USAGE: Final = Usage( prompt_tokens=100, completion_tokens=10, diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index 041c625eaef..2ef202762ff 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -12,6 +12,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.decisions.transformation import ( BaseDecisionsConfig, DecisionsInputLimits, + DecisionsResponseValidationError, ir_to_systemone_request, ir_to_systemone_response, parse_systemone_response, @@ -21,6 +22,7 @@ from litellm.llms.openai.decisions.transformation import openai_request_to_ir from litellm.llms.typesafe.decisions.transformation import TypeSafeDecisionsConfig from litellm.types.decisions import ( MAX_DECISION_QUESTIONS, + DecisionsIRChoiceAnswer, DecisionsIRRefusal, DecisionsIRRequest, DecisionsIRScoreAnswer, @@ -347,6 +349,49 @@ def test_a_box_answer_without_confidence_reaches_the_caller_unchanged() -> None: assert response.model_dump(mode="json") == payload +@pytest.mark.parametrize( + ("question", "answer"), + [ + ( + {"type": "choice", "criteria": {"yes": "Yes", "no": "No"}}, + {"type": "choice", "choice": "yes", "probabilities": {"yes": 0.9, "no": 0.1}}, + ), + ( + {"type": "score", "criteria": ["low", "high"]}, + {"type": "score", "score": 0.5, "legend": {"0": "low", "1": "high"}, "probabilities": {"0": 0.5, "1": 0.5}}, + ), + ], +) +@pytest.mark.parametrize("confidence", ({}, {"confidence": None})) +@pytest.mark.parametrize( + ("provider", "optional"), (("nace", False), ("fireworks_ai", False), ("parallel_ai", True)) +) +def test_only_a_provider_without_confidence_may_omit_it( + question: Mapping[str, object], + answer: Mapping[str, object], + confidence: Mapping[str, object], + provider: str, + optional: bool, +) -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders(provider)) + assert config is not None + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate({"state": {"text": "Example"}, "questions": {"asked": question}}) + ) + upstream: Final = httpx.Response( + 200, json={"mode": "fast", "model": "decision", "answers": {"asked": {**answer, **confidence}}} + ) + + if optional: + parsed: Final = config.transform_decisions_response("decision", provider, upstream, request) + assert isinstance(parsed.answers[0], (DecisionsIRChoiceAnswer, DecisionsIRScoreAnswer)) + assert parsed.answers[0].confidence is None + return + with pytest.raises(DecisionsResponseValidationError) as raised: + config.transform_decisions_response("decision", provider, upstream, request) + assert raised.value.status_code == 500 + + @pytest.mark.parametrize("provider", ("nace", "openrouter", "fireworks_ai", "typesafe")) @pytest.mark.parametrize("legend", (["low", "high"], {"0": "low", "1": "high"})) def test_score_legends_preserve_labels_probabilities_and_metadata(provider: str, legend: object) -> None: diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py index eddfe578256..186ee545323 100644 --- a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -1,10 +1,16 @@ from collections.abc import Mapping +from dataclasses import replace from typing import Final +import httpx import pytest from pydantic import TypeAdapter -from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir +from litellm.llms.base_llm.decisions.transformation import ( + DecisionsResponseValidationError, + ir_to_systemone_response, + systemone_request_to_ir, +) from litellm.llms.openai.decisions.transformation import ( OpenAIDecisionsConfig, ir_to_openai_request, @@ -106,6 +112,35 @@ def test_an_openai_response_reaches_the_caller_unchanged() -> None: assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE +@pytest.mark.parametrize("answer_type", ["choice", "score"]) +def test_a_choice_or_score_answer_without_confidence_stays_a_provider_error(answer_type: str) -> None: + answers: Final = [ + {key: value for key, value in answer.items() if answer["type"] != answer_type or key != "confidence"} + for answer in _OPENAI_RESPONSE["answers"] + ] + with pytest.raises(DecisionsResponseValidationError) as raised: + OpenAIDecisionsConfig().transform_decisions_response( + "gpt-6-luna", + "openai", + httpx.Response(200, json={**_OPENAI_RESPONSE, "answers": answers}), + _openai_ir(_OPENAI_REQUEST), + ) + assert raised.value.status_code == 500 + + +def test_provider_extras_stay_off_the_openai_response_unless_the_provider_names_them() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + parsed: Final = replace( + OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir), extra={"provider": "upstream", "citations": []} + ) + + assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE + assert ir_to_openai_response(parsed, ir, "requested", frozenset({"citations"})).model_dump(mode="json") == { + **_OPENAI_RESPONSE, + "citations": [], + } + + def test_answers_openai_did_not_return_are_refusals() -> None: ir: Final = _openai_ir(_OPENAI_REQUEST) payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} diff --git a/tests/unit/llms/parallel_ai/decisions/__init__.py b/tests/unit/llms/parallel_ai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/parallel_ai/decisions/test_transformation.py b/tests/unit/llms/parallel_ai/decisions/test_transformation.py new file mode 100644 index 00000000000..9cd404f61f4 --- /dev/null +++ b/tests/unit/llms/parallel_ai/decisions/test_transformation.py @@ -0,0 +1,423 @@ +import json +from collections.abc import Mapping +from dataclasses import replace +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.base_llm.decisions.transformation import ( + DecisionsResponseValidationError, + ir_to_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir +from litellm.llms.parallel_ai.decisions.transformation import ParallelAIDecisionsConfig +from litellm.types.decisions import ( + DecisionsIRRefusal, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, Rules, function_setup + +QUESTIONS: Final = { + "available": {"type": "noul", "instructions": "Is this product available?"}, + "tier": {"type": "choice", "instructions": "Which tier is offered?", "criteria": {"free": "Free", "paid": "Paid"}}, + "fit": {"type": "score", "instructions": "How well does it fit?", "criteria": ["Poor", "Good", "Excellent"]}, + "unknown": {"type": "noul", "instructions": "Is there enough evidence?"}, +} +REQUEST: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate({"state": {"product": "Example"}, "questions": QUESTIONS}) +) +PAYLOAD: Final = { + "mode": "fast", + "answers": { + "available": {"type": "noul", "noul": 0.9}, + "tier": {"type": "choice", "choice": "paid", "probabilities": {"free": 0.2, "paid": 0.8}}, + "fit": { + "type": "score", + "score": 1.7, + "legend": {"0": "Poor", "1": "Good", "2": "Excellent"}, + "probabilities": {"0": 0.1, "1": 0.1, "2": 0.8}, + }, + "unknown": {"type": "refusal"}, + }, + "usage": {"input_tokens": 4212, "output_tokens": 0}, + "citations": [{"url": "https://example.com/product", "title": "Product", "excerpts": ["Available now"]}], + "decision_id": "decisions_example", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("surface", ["decisions", "systemone"]) +async def test_sdk_sends_parallel_wire_format_and_preserves_evidence( + use_async: bool, surface: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + route: Final = respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) + connection: Final = {"model": "parallel_ai/decision", "api_key": "test-key", "timeout": 5, "num_retries": 0} + fields: Final = ( + {"state": "Example", "questions": QUESTIONS} + if surface == "systemone" + else { + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions + async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + response: Final = await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields) + sent: Final = route.calls.last.request + body: Final = json.loads(sent.content) + assert body["mode"] == "fast" + assert "model" not in body + assert body["state"] == "Example" + assert sent.headers["x-api-key"] == "test-key" + assert "authorization" not in sent.headers + assert response.model == "decision" + assert response.model_dump()["citations"] == PAYLOAD["citations"] + assert response.model_dump()["decision_id"] == PAYLOAD["decision_id"] + await GLOBAL_LOGGING_WORKER.clear_queue() + + +def test_provider_routes_systemone_questions_to_parallel_without_a_model_field() -> None: + config: Final = ProviderConfigManager.get_provider_decisions_config("decision", LlmProviders.PARALLEL_AI) + assert config is not None + assert config.transform_decisions_request("decision", REQUEST, "parallel_ai") == { + "mode": "fast", + "state": {"product": "Example"}, + "questions": QUESTIONS, + } + assert config.get_complete_url("https://gateway.example/v1/", "decision") == "https://gateway.example/v1/decisions" + + +@pytest.mark.parametrize("suffix", ["", "/", "/v1", "/v1/", "/v1/decisions", "/v1/decisions/", "/v1/search", "/v1/search/"]) +def test_base_carrying_a_parallel_route_reaches_the_decisions_endpoint(suffix: str) -> None: + complete_url: Final = ParallelAIDecisionsConfig().get_complete_url(f"https://api.parallel.ai{suffix}", "decision") + assert complete_url == "https://api.parallel.ai/v1/decisions" + + +def test_auth_replaces_conflicting_headers_without_forwarding_bearer_credentials() -> None: + headers: Final = ParallelAIDecisionsConfig().validate_environment( + {"Authorization": "Bearer old", "X-API-Key": "old", "CONTENT-TYPE": "text/plain", "X-Trace": "test"}, + "decision", + "test-key", + {}, + ) + assert headers == {"x-api-key": "test-key", "Content-Type": "application/json", "X-Trace": "test"} + + +def test_openai_predicate_is_translated_to_a_named_noul_question() -> None: + request: Final = openai_request_to_ir( + OpenAIDecisionRequestBody.model_validate( + { + "input": "Example product", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + ) + assert ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai") == { + "mode": "fast", + "state": "Example product", + "questions": {"available": {"type": "noul", "instructions": "Available?"}}, + } + + +@pytest.mark.parametrize("surface", ["systemone", "decisions"]) +def test_all_answers_usage_and_evidence_survive_response_translation(surface: str) -> None: + response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) + assert isinstance(response.answers[3], DecisionsIRRefusal) + formatted: Final = ( + ir_to_systemone_response(response, REQUEST) + if surface == "systemone" + else ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + ) + result: Final = formatted.model_dump(mode="json") + assert result["citations"] == PAYLOAD["citations"] + assert result["decision_id"] == PAYLOAD["decision_id"] + assert result["mode"] == "fast" + assert result["usage"]["input_tokens"] == 4212 + assert result["usage"]["output_tokens"] == 0 + if surface == "systemone": + assert result["answers"]["available"]["noul"] == 0.9 + assert result["answers"]["tier"]["choice"] == "paid" + assert result["answers"]["fit"]["score"] == 1.7 + assert "unknown" not in result["answers"] + else: + assert result["answers"][3]["type"] == "refusal" + + +def test_rejects_more_than_one_hundred_questions() -> None: + request: Final = systemone_request_to_ir( + DecisionsRequestBody.model_validate( + { + "state": "Example", + "questions": {str(index): QUESTIONS["available"] for index in range(101)}, + } + ) + ) + assert isinstance( + ParallelAIDecisionsConfig().transform_decisions_request("decision", request, "parallel_ai"), + UnsupportedDecisionsRequest, + ) + + +@pytest.mark.parametrize("model", ["fast", "unknown"]) +def test_rejects_unregistered_models_instead_of_forwarding_them_as_modes(model: str) -> None: + assert isinstance( + ParallelAIDecisionsConfig().transform_decisions_request(model, REQUEST, "parallel_ai"), + UnsupportedDecisionsRequest, + ) + + +@pytest.mark.parametrize("confidence", [None, 0.8]) +@pytest.mark.parametrize("surface", ["systemone", "decisions"]) +def test_optional_confidence_is_preserved_without_inventing_a_value(confidence: float | None, surface: str) -> None: + payload: Final = { + **PAYLOAD, + "answers": {name: {**answer, "confidence": confidence} for name, answer in PAYLOAD["answers"].items()}, + } + response: Final = ParallelAIDecisionsConfig().parse_response(payload, REQUEST) + formatted: Final = ( + ir_to_systemone_response(response, REQUEST) + if surface == "systemone" + else ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + ) + answers: Final = formatted.model_dump(mode="json")["answers"] + values: Final = answers.values() if isinstance(answers, dict) else answers + assert [answer["confidence"] for answer in values if answer["type"] in {"choice", "score"}] == [confidence] * 2 + + +def test_provider_extras_cannot_replace_normalized_response_fields() -> None: + response: Final = ParallelAIDecisionsConfig().parse_response(PAYLOAD, REQUEST) + with_collision: Final = replace(response, extra={**response.extra, "model": "wrong", "answers": [], "usage": {}}) + formatted: Final = ir_to_openai_response( + with_collision, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + assert formatted.model == "decision" + assert len(formatted.answers) == 4 + assert formatted.usage.input_tokens == 4212 + assert formatted.model_dump()["citations"] == PAYLOAD["citations"] + + +def test_undocumented_parallel_fields_stay_off_the_openai_response() -> None: + response: Final = ParallelAIDecisionsConfig().parse_response({**PAYLOAD, "internal_trace_id": "trace"}, REQUEST) + formatted: Final = ir_to_openai_response( + response, REQUEST, "parallel_ai/decision", ParallelAIDecisionsConfig.openai_evidence_fields + ) + assert "internal_trace_id" not in formatted.model_dump() + + +@pytest.mark.parametrize("tokens", [0, 100, 32000]) +@pytest.mark.parametrize("call_type", ["decisions", "adecisions", "systemone", "asystemone"]) +def test_request_pricing_is_independent_of_token_count(tokens: int, call_type: str) -> None: + rate: Final = litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"] + cost: Final = litellm.cost_per_token( + model="parallel_ai/decision", + prompt_tokens=tokens, + completion_tokens=0, + call_type=call_type, + ) + assert cost == (rate, 0.0) + + +@pytest.mark.asyncio +async def test_deployment_query_price_replaces_the_catalog_price( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + respx_mock.post("https://api.parallel.ai/v1/decisions").respond(json=PAYLOAD) + price: Final = 25 * litellm.model_cost["parallel_ai/decision"]["input_cost_per_query"] + router: Final = litellm.Router( + model_list=[ + { + "model_name": "webjev", + "litellm_params": { + "model": "parallel_ai/decision", + "api_key": "test-key", + "input_cost_per_query": price, + }, + } + ] + ) + fields: Final = { + "model": "webjev", + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + "litellm_call_id": "query-price-call", + } + logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields) + response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj) + assert response._hidden_params["response_cost"] == pytest.approx(price) + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.asyncio +async def test_query_price_on_another_provider_leaves_its_token_billing_alone( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + input_rate: Final = 1e-06 + tokens: Final = 1000 + model: Final = "fireworks_ai/accounts/example/models/token-billed" + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_token": input_rate, "output_cost_per_token": 0.0} + | {"litellm_provider": "fireworks_ai", "mode": "evaluation"}, + ) + respx_mock.post("https://api.fireworks.ai/inference/v1/systemone").respond( + json={ + "model": "token-billed", + "answers": {"available": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": tokens, "output_tokens": 0}, + } + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "token-billed", + "litellm_params": {"model": model, "api_key": "test-key", "input_cost_per_query": 0.025}, + } + ] + ) + fields: Final = { + "model": "token-billed", + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + "litellm_call_id": "token-billed-call", + } + logging_obj, data = function_setup("adecisions", Rules(), datetime.now(), **fields) + response: Final = await router.adecisions(**data, litellm_logging_obj=logging_obj) + assert response._hidden_params["response_cost"] == pytest.approx(tokens * input_rate) + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.parametrize( + "payload", [{"answers": {}}, {"mode": "fast", "answers": {"available": {"type": "noul", "noul": "invalid"}}}] +) +def test_malformed_provider_responses_raise_a_provider_error(payload: Mapping[str, object]) -> None: + with pytest.raises(DecisionsResponseValidationError): + ParallelAIDecisionsConfig().transform_decisions_response( + "decision", "parallel_ai", httpx.Response(200, json=payload), REQUEST + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("surface", ["decisions", "systemone"]) +@pytest.mark.parametrize("key_env", ["PARALLEL_AI_API_KEY", "PARALLEL_API_KEY"]) +async def test_untrusted_base_cannot_receive_environment_key( + use_async: bool, surface: str, key_env: str, respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.delenv("PARALLEL_AI_API_KEY", raising=False) + monkeypatch.delenv("PARALLEL_API_KEY", raising=False) + monkeypatch.setenv(key_env, "server-secret") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://attacker.example/v1/decisions").respond(json=PAYLOAD) + connection: Final = {"model": "parallel_ai/decision", "api_base": "https://attacker.example", "num_retries": 0} + fields: Final = ( + {"state": "Example", "questions": QUESTIONS} + if surface == "systemone" + else { + "input": "Example", + "questions": [{"type": "predicate", "name": "available", "instructions": "Available?"}], + } + ) + sync_call: Final = litellm.systemone if surface == "systemone" else litellm.decisions + async_call: Final = litellm.asystemone if surface == "systemone" else litellm.adecisions + + async def call() -> object: + return await async_call(**connection, **fields) if use_async else sync_call(**connection, **fields) + + with pytest.raises(litellm.BadRequestError, match="explicit api_key"): + await call() + assert not route.called + await GLOBAL_LOGGING_WORKER.clear_queue() + + +@pytest.mark.parametrize( + "api_base", + [ + "https://api.parallel.ai.attacker.example", + "https://api.parallel.ai@attacker.example", + "http://api.parallel.ai", + "https://api.parallel.ai:8443", + "https://attacker.example/api.parallel.ai", + "https://api.parallel.ai/other-tenant", + "https://api.parallel.ai/other-tenant/v1/search", + "https://attacker.example/v1/search", + "https://attacker.example/v1/decisions", + ], +) +def test_environment_key_rejects_untrusted_destination_variants(api_base: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + with pytest.raises(ValueError, match="explicit api_key"): + ParallelAIDecisionsConfig().resolve_connection(api_base, None) + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.parallel.ai", + "https://API.PARALLEL.AI:443/v1/", + "https://api.parallel.ai/v1/decisions", + "https://api.parallel.ai/v1/search", + ], +) +def test_default_destination_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PARALLEL_AI_API_BASE", raising=False) + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None) + assert resolved_base == (api_base or "https://api.parallel.ai") + assert key == "server-secret" + + +@pytest.mark.parametrize( + "api_base", [None, "https://gateway.example/parallel/v1/", "https://gateway.example/parallel/v1/search"] +) +def test_operator_base_retains_environment_key(api_base: str | None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PARALLEL_AI_API_BASE", "https://gateway.example/parallel") + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + resolved_base, key = ParallelAIDecisionsConfig().resolve_connection(api_base, None) + assert resolved_base == (api_base or "https://gateway.example/parallel") + assert key == "server-secret" + + +@pytest.mark.parametrize("api_key", [None, "caller-key"]) +@pytest.mark.parametrize("api_base", ["https://api.parallel.ai\t", "https://api.parallel.ai\r\nHost: attacker.example"]) +def test_malformed_destination_is_a_bad_request(api_base: str, api_key: str | None) -> None: + with pytest.raises(litellm.BadRequestError): + litellm.decisions( + model="parallel_ai/decision", + api_base=api_base, + api_key=api_key, + input="Example", + questions=[{"type": "predicate", "name": "available", "instructions": "Available?"}], + num_retries=0, + ) + + +def test_custom_destination_uses_explicit_key_instead_of_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PARALLEL_AI_API_KEY", "server-secret") + assert ParallelAIDecisionsConfig().resolve_connection("https://gateway.example", "caller-key") == ( + "https://gateway.example", + "caller-key", + ) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 72755633376..d399adaea7d 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -536,6 +536,7 @@ def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: ( ("TypeSafe", "TypeSafe", LlmProviders.TYPESAFE), ("Nace", "Nace", LlmProviders.NACE), + ("PARALLEL_AI", "Parallel AI", LlmProviders.PARALLEL_AI), ("EU_JEV", "eu/jev", LlmProviders.EU_JEV), ), ) diff --git a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py index c2931470482..ce1dfce1646 100644 --- a/tests/unit/proxy/test_zero_cost_model_budget_bypass.py +++ b/tests/unit/proxy/test_zero_cost_model_budget_bypass.py @@ -32,6 +32,29 @@ from litellm.router import Router from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo +@pytest.mark.parametrize( + ("model", "expected_free"), + [("parallel_ai/decision", False), ("fireworks_ai/accounts/fireworks/routers/rune", True)], +) +def test_query_price_keeps_budgets_only_where_it_is_billed(model: str, expected_free: bool) -> None: + router = Router( + model_list=[ + { + "model_name": "query-priced", + "litellm_params": { + "model": model, + "api_key": "test-key", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_query": 0.025, + }, + } + ] + ) + + assert is_model_cost_zero("query-priced", router) is expected_free + + @pytest.fixture def mock_router_with_zero_cost_model(): """Create a mock router with a zero-cost model.""" diff --git a/tests/unit/router_strategy/test_complexity_router.py b/tests/unit/router_strategy/test_complexity_router.py index ef7b9e485d4..d0e2b381253 100644 --- a/tests/unit/router_strategy/test_complexity_router.py +++ b/tests/unit/router_strategy/test_complexity_router.py @@ -359,6 +359,52 @@ class TestComplexityRouterInit: "tier-probability:MEDIUM=0.900000", ) + @pytest.mark.asyncio + async def test_jev_choice_without_confidence_routes_and_reports_no_confidence(self, mock_router_instance): + client = _StaticJevClient( + JevSystemOneResponse( + answers={ + "tier": JevChoiceAnswer( + type="choice", + choice="MEDIUM", + probabilities={"SIMPLE": 0.1, "MEDIUM": 0.9}, + ) + }, + ) + ) + router = ComplexityRouter( + "test-router", + mock_router_instance, + { + "classifier_type": "jev", + "jev_classifier_config": {"api_key": "test", "timeout_ms": 100}, + "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}, + }, + derive_savings_baseline=False, + jev_client=client, + ) + + outcome = await router.aclassify("Explain this") + result = await router.async_pre_routing_hook( + model="test-router", + request_kwargs={}, + messages=[{"role": "user", "content": "Explain this"}], + ) + + assert outcome.tier == ComplexityTier.MEDIUM + assert outcome.cause == "jev_classifier" + assert outcome.jev_verdict is not None + assert outcome.jev_verdict.confidence is None + assert outcome.signals == ( + "jev-classifier:MEDIUM", + "tier-probability:SIMPLE=0.100000", + "tier-probability:MEDIUM=0.900000", + ) + assert result is not None + assert result.routing_decision is not None + assert result.routing_decision["classifier_probabilities"] == {"SIMPLE": 0.1, "MEDIUM": 0.9} + assert "classifier_confidence" not in result.routing_decision + @pytest.mark.asyncio async def test_jev_pre_routing_hook_exposes_routing_decision_provenance( self, mock_router_instance, monkeypatch: pytest.MonkeyPatch diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx index f6c5794fa5c..9c78eff1a2b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.integration.test.tsx @@ -109,6 +109,20 @@ describe("ResponseView integration", () => { expect(screen.getByText("378 in / 63 out")).toBeInTheDocument(); }); + it("leaves out the confidence badge when the provider reports no confidence", () => { + const withoutConfidence: SystemOneResponse = { + ...sampleResponse, + answers: { + dept: { type: "choice", choice: "technical", confidence: null, probabilities: { technical: 1 } }, + frustration: { type: "score", score: 2, confidence: null, probabilities: { "2": 1 } }, + }, + }; + + render(); + + expect(screen.queryByText(/% confidence/)).not.toBeInTheDocument(); + }); + it("selects the score level nearest a fractional calibrated score", () => { const fractional: SystemOneResponse = { answers: { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx index 572bf765ac1..89d998b2632 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/ResponseView.tsx @@ -71,7 +71,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
Selected choice {answer.choice} - {answer.confidence !== undefined && ( + {typeof answer.confidence === "number" && ( {Math.round(answer.confidence * 100)}% confidence )}
@@ -90,7 +90,7 @@ function AnswerDetails({ answer }: { answer: SystemOneAnswer }) {
Score {SCORE_FORMAT.format(answer.score)} - {answer.confidence !== undefined && ( + {typeof answer.confidence === "number" && ( {Math.round(answer.confidence * 100)}% confidence )}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts index 0e06d733ea3..1b906a8d975 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/openAIDecisions.ts @@ -62,14 +62,14 @@ const choiceAnswerFields = { ...answerFields, type: z.literal("choice"), choice: choiceValue, - confidence: probability, + confidence: probability.nullish(), probabilities: z.array(z.looseObject({ value: choiceValue, probability })), }; const scoreAnswerFields = { ...answerFields, type: z.literal("score"), score: z.number().finite(), - confidence: probability, + confidence: probability.nullish(), probabilities: z.array(z.looseObject({ value: z.number().int(), label: z.string(), probability })), }; const usageFields = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts index 7c923fe8e8b..108da052893 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts @@ -125,7 +125,7 @@ const noulAnswerSchema = z.looseObject({ type: z.literal("noul"), noul: probabil const choiceAnswerShape = { type: z.literal("choice"), choice: z.string(), - confidence: probability.optional(), + confidence: probability.nullish(), probabilities, }; const choiceAnswerSchema = z.looseObject(choiceAnswerShape); @@ -133,7 +133,7 @@ const choiceAnswerSchema = z.looseObject(choiceAnswerShape); const scoreAnswerShape = { type: z.literal("score"), score: z.number().finite(), - confidence: probability.optional(), + confidence: probability.nullish(), legend: z.record(z.string(), decisionsJson).optional(), probabilities, }; diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 4c483434831..c88e44f21d4 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -73,6 +73,12 @@ describe("provider_info_helpers", () => { }); describe("getProviderLogoAndName", () => { + it("shows the Parallel AI logo for the parallel_ai slug", () => { + const { logo, displayName } = getProviderLogoAndName("parallel_ai"); + expect(displayName).toBe(Providers.PARALLEL_AI); + expect(logo).toContain("parallel_ai"); + }); + it("should return empty logo and dash display name when providerValue is empty", () => { const result = getProviderLogoAndName(""); expect(result).toEqual({ logo: "", displayName: "-" }); @@ -577,6 +583,16 @@ describe("provider_info_helpers", () => { expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); }); + it("lists Parallel decision models with their provider and model placeholder", () => { + const modelMap = { + "parallel_ai/decision": { litellm_provider: "parallel_ai", mode: "evaluation" }, + }; + + expect(getProviderModels("PARALLEL_AI", modelMap)).toEqual(["parallel_ai/decision"]); + expect(getProviderLogoAndName("parallel_ai").displayName).toBe("Parallel AI"); + expect(getPlaceholder("PARALLEL_AI")).toBe("parallel_ai/decision"); + }); + it("lists Nace decision models and resolves the provider name and model placeholder", () => { const modelMap = { "nace/drex-latest": { litellm_provider: "nace", mode: "evaluation" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index a8879eb570b..1680258e10b 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -50,6 +50,7 @@ import ollamaLogo from "../../public/assets/logos/ollama.svg"; import openaiSmallLogo from "../../public/assets/logos/openai_small.svg"; import openrouterLogo from "../../public/assets/logos/openrouter.svg"; import oracleLogo from "../../public/assets/logos/oracle.svg"; +import parallelAiLogo from "../../public/assets/logos/parallel_ai.png"; import perplexityAiLogo from "../../public/assets/logos/perplexity-ai.svg"; import qwenLogo from "../../public/assets/logos/qwen.png"; import recraftLogo from "../../public/assets/logos/recraft.svg"; @@ -191,6 +192,7 @@ export enum Providers { Triton = "Triton", TypeSafe = "TypeSafe", Nace = "Nace", + PARALLEL_AI = "Parallel AI", V0 = "V0", VERCEL_AI_GATEWAY = "Vercel Ai Gateway", Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", @@ -325,6 +327,7 @@ export const provider_map: Record = { Triton: "triton", TypeSafe: "typesafe", Nace: "nace", + PARALLEL_AI: "parallel_ai", V0: "v0", VERCEL_AI_GATEWAY: "vercel_ai_gateway", Vertex_AI: "vertex_ai", @@ -419,6 +422,7 @@ export const providerLogoMap: Partial> = { [Providers.OpenAI_Compatible]: openaiSmallLogo.src, [Providers.Openrouter]: openrouterLogo.src, [Providers.Oracle]: oracleLogo.src, + [Providers.PARALLEL_AI]: parallelAiLogo.src, [Providers.Perplexity]: perplexityAiLogo.src, [Providers.Qwen_AI_Platform]: qwenLogo.src, [Providers.QwenCloud]: qwenLogo.src, @@ -514,6 +518,7 @@ const providerPlaceholderMap: Partial> = { [Providers.Tencent]: "tencent/deepseek-v4-pro", [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Nace]: "nace/drex-latest", + [Providers.PARALLEL_AI]: "parallel_ai/decision", [Providers.StreamLake]: "streamlake/", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 8278fd9fff4..79680f096e3 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -29404,7 +29404,7 @@ export interface components { /** Choice */ choice: string; /** Confidence */ - confidence: number; + confidence?: number | null; /** Probabilities */ probabilities: { [key: string]: number; @@ -40226,7 +40226,7 @@ export interface components { /** Choice */ choice: string | boolean; /** Confidence */ - confidence: number; + confidence?: number | null; /** Name */ name: string | null; /** Probabilities */ @@ -40322,7 +40322,7 @@ export interface components { /** OpenAIScoreAnswer */ OpenAIScoreAnswer: { /** Confidence */ - confidence: number; + confidence?: number | null; /** Name */ name: string | null; /** Probabilities */ @@ -45564,7 +45564,7 @@ export interface components { /** ScoreAnswer */ ScoreAnswer: { /** Confidence */ - confidence: number; + confidence?: number | null; /** Legend */ legend: { [key: string]: string | { From 5e08cb0e632f0fd36ab8be0c7d4584fadbdffd38 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 2 Nov 2026 21:02:52 -0800 Subject: [PATCH 07/34] test(messages): add rust vs python messages parity characterization suite (#49610) Runs every case through the Python /v1/messages path and the Rust path against one recording upstream and compares dispatch, upstream request, outcome, hidden params, standard logging fields and callback kwargs. Known Rust differences are strict xfails tagged with their inventory ID. --- .../strategies/e2e_parity/__init__.py | 2 +- .../e2e_parity/sdk/messages/conftest.py | 34 + .../sdk/messages/test_native_parity.py | 1200 +++++++++++++++++ .../support/recording_server.py | 5 +- 4 files changed, 1239 insertions(+), 2 deletions(-) create mode 100644 tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py create mode 100644 tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py diff --git a/tests/rust-python-harness/strategies/e2e_parity/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/__init__.py index 298c75f2af1..a2ea98f19ab 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/__init__.py +++ b/tests/rust-python-harness/strategies/e2e_parity/__init__.py @@ -20,7 +20,7 @@ CASES: Final[tuple[CaseDefinition, ...]] = ( CaseDefinition( "messages", NotImplementedCaseSpec( - reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered." + reason="Parity runs as the pytest suite in sdk/messages/test_native_parity.py, not as a runner case." ), surface="sdk", ), diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py new file mode 100644 index 00000000000..4bb20713c59 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/conftest.py @@ -0,0 +1,34 @@ +from collections.abc import AsyncIterator, Generator +from importlib import import_module +from pathlib import Path +from typing import Final + +import pytest +import pytest_asyncio + +from tests.test_litellm_rust.support.isolation import isolated_test_state +from tests.test_litellm_rust.support.recording_server import RecordingServer, recording_service + +PARITY_ROOT: Final = Path(__file__).parent + + +@pytest_asyncio.fixture(autouse=True, loop_scope="function") +async def isolate_test_state() -> AsyncIterator[None]: + async with isolated_test_state(): + yield + + +@pytest.fixture +def recording_server() -> Generator[RecordingServer]: + with recording_service() as server: + yield server + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + try: + import_module("litellm.rust_bridge._native") + except ImportError: + skip: Final = pytest.mark.skip(reason="requires a compiled litellm.rust_bridge._native extension") + for item in items: + if PARITY_ROOT in item.path.parents: + item.add_marker(skip) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py new file mode 100644 index 00000000000..cdfba773cf6 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -0,0 +1,1200 @@ +"""Native Anthropic Messages calls must be indistinguishable from the Python implementation. + +Each case sends one `/v1/messages` SDK call through the Python handler and through the native route against the same +scripted upstream, then compares one aspect of what happened: + +- dispatch: the native leg was served natively instead of falling back to Python +- request: every request that reached the upstream (method, path, body, headers) +- outcome: what the caller got back (response body, stream bytes, or the raised exception) +- hidden_params: the `_hidden_params` keys the proxy builds its response headers from +- logged: which terminal callbacks fired and the stable standard logging payload fields +- logged_metadata, logged_model_info, logged_response_headers, logged_api_key, logged_call: single callback kwargs + fields that proxy integrations read + +Both legs run once per case and every aspect of that case reads the same pair + +The native leg prepends an unconstrained Rust rule, so feature detectors that keep a call on Python today do not +apply: a case for a detector-covered feature shows what Rust still lacks before that detector can go. Every known +difference is a strict xfail naming the gap, with an id from the Rust vs Python Messages inventory (A routing, +R request, S response, C callbacks, F covered by a fallback detector today) +""" + +import asyncio +import json +import os +import re +from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field, replace +from types import MappingProxyType +from typing import Final, cast + +import pytest +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict +from litellm.messages import dispatch +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.configuration import Rollout +from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging +from tests.test_litellm_rust.support.isolation import isolated_callback_registries, rebound +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + +SONNET: Final = "anthropic/claude-sonnet-4-5" +OPUS_ADAPTIVE: Final = "anthropic/claude-opus-4-7" +OPUS_FAST: Final = "anthropic/claude-opus-5-5" +API_KEY: Final = "sk-ant-parity-test" +USER: Final[Mapping[str, JsonValue]] = MappingProxyType({"role": "user", "content": "What is the capital of France?"}) + +_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_LOGGED: Final = TypeAdapter(dict[str, object]) +_TRANSPORT_HEADERS: Final = frozenset( + {"accept", "accept-encoding", "connection", "content-length", "host", "user-agent"} +) +_COMPARED_LOG_FIELDS: Final = ( + "call_type", + "custom_llm_provider", + "model", + "model_group", + "model_id", + "api_base", + "stream", + "cache_hit", + "prompt_tokens", + "completion_tokens", + "total_tokens", + "response_cost", + "status", +) +_TERMINAL_EVENTS: Final = ( + "log_success_event", + "async_log_success_event", + "log_failure_event", + "async_log_failure_event", +) +_VOLATILE: Final = ( + (re.compile(r"time taken=[0-9.]+"), "time taken="), + (re.compile(r"Timeout passed=[0-9.]+"), "Timeout passed="), + (re.compile(r"127\.0\.0\.1:[0-9]+"), "127.0.0.1:"), +) + +TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "id": "msg_01ParityText", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": "Paris"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 14, "output_tokens": 4}, + } +) + +TOOL_USE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParityTool", + "content": [ + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {"city": "Paris"}}, + ], + "stop_reason": "tool_use", + } +) + +THINKING_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "id": "msg_01ParityThinking", + "content": [ + {"type": "thinking", "thinking": "The capital of France is Paris.", "signature": "EqQBCkYIARgCKkB"}, + {"type": "text", "text": "Paris"}, + ], + } +) + +CACHED_USAGE_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "usage": { + "input_tokens": 14, + "output_tokens": 4, + "cache_creation_input_tokens": 2048, + "cache_read_input_tokens": 1024, + "service_tier": "standard", + }, + } +) + +WEATHER_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "name": "get_weather", + "description": "Current weather for a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } +) + +TOOL_TURNS: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "Weather in Paris?"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {"city": "Paris"}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01Parity", "content": "18C, sunny"}]}, +) + +THINKING_REPLAY: Final[tuple[JsonValue, ...]] = ( + {"role": "user", "content": "What is the capital of France?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Paris.", "signature": "EqQBCkYIARgCKkB"}, + {"type": "text", "text": "Paris"}, + ], + }, + {"role": "user", "content": "And Germany?"}, +) + +NO_INPUT: Final[Mapping[str, JsonValue]] = MappingProxyType({}) +PNG_PIXEL: Final = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + +def _events(*events: tuple[str, JsonValue]) -> tuple[tuple[str, object], ...]: + return tuple((name, data) for name, data in events) + + +def _message_start(message_id: str) -> tuple[str, JsonValue]: + return ( + "message_start", + { + "type": "message_start", + "message": { + **TEXT_RESPONSE, + "id": message_id, + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 14, "output_tokens": 1}, + }, + }, + ) + + +def _text_block(index: int, *pieces: str) -> tuple[tuple[str, JsonValue], ...]: + return ( + ( + "content_block_start", + {"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}}, + ), + *( + ( + "content_block_delta", + {"type": "content_block_delta", "index": index, "delta": {"type": "text_delta", "text": piece}}, + ) + for piece in pieces + ), + ("content_block_stop", {"type": "content_block_stop", "index": index}), + ) + + +def _message_end(stop_reason: str, output_tokens: int) -> tuple[tuple[str, JsonValue], ...]: + return ( + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": {"output_tokens": output_tokens}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ) + + +TEXT_STREAM: Final = _events( + _message_start("msg_01ParityStream"), *_text_block(0, "Pa", "ris"), *_message_end("end_turn", 4) +) + +TOOL_STREAM: Final = _events( + _message_start("msg_01ParityToolStream"), + *_text_block(0, "Checking."), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_01Parity", "name": "get_weather", "input": {}}, + }, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '{"city": '}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": '"Paris"}'}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + *_message_end("tool_use", 22), +) + +THINKING_STREAM: Final = _events( + _message_start("msg_01ParityThinkingStream"), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "Paris."}}, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "signature_delta", "signature": "EqQBCkYIARgCKkB"}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + *_text_block(1, "Paris"), + *_message_end("end_turn", 12), +) + +ERROR_MID_STREAM: Final = _events( + _message_start("msg_01ParityOverloaded"), + *_text_block(0, "Pa"), + ("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), +) + +SLOW_STREAM: Final = _events( + _message_start("msg_01ParitySlow"), + *_text_block(0, "a", "b", "c", "d", "e", "f"), + *_message_end("end_turn", 6), +) + + +def _upstream_error(kind: str, message: str) -> Mapping[str, JsonValue]: + return MappingProxyType({"type": "error", "error": {"type": kind, "message": message}}) + + +def _ok(body: Mapping[str, JsonValue] = TEXT_RESPONSE, **headers: str) -> ResponseSpec: + return ResponseSpec(body=dict(body), headers=dict(headers)) + + +def _stream(events: tuple[tuple[str, object], ...], chunk_delay: float = 0) -> ResponseSpec: + return ResponseSpec(body={}, events=events, chunk_delay=chunk_delay) + + +def _failure(status: int, kind: str, message: str, **headers: str) -> ResponseSpec: + return ResponseSpec(body=dict(_upstream_error(kind, message)), status=status, headers=dict(headers)) + + +CALLBACK_ASPECTS: Final = ( + "logged_metadata", + "logged_model_info", + "logged_response_headers", + "logged_api_key", + "logged_call", +) +ASPECTS: Final = ("dispatch", "request", "outcome", "hidden_params", "logged", *CALLBACK_ASPECTS) + + +@dataclass(frozen=True, slots=True) +class Case: + name: str + kwargs: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + upstream: tuple[ResponseSpec, ...] = (ResponseSpec(body=dict(TEXT_RESPONSE)),) + model: str = SONNET + sync: bool = False + env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + settings: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + private_keys: bool = False + gaps: Mapping[str, str | None] = field(default_factory=lambda: MappingProxyType({})) + + def params(self) -> Iterator[object]: + for aspect in ASPECTS: + gap = {**KNOWN_GAPS.get(self.name, {}), **self.gaps}.get(aspect) + marks = () if not gap else (pytest.mark.xfail(reason=gap, strict=True),) + yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks) + + +def _gaps(**gaps: str | None) -> Mapping[str, str | None]: + return MappingProxyType(gaps) + + +def _kw(**kwargs: object) -> Mapping[str, object]: + return MappingProxyType(kwargs) + + +A2_BARE_MODEL: Final = "A2: native resolves the provider from the model prefix only, so a bare Claude model name fails" +A10_NO_MAX_TOKENS: Final = "A10: native admission declines a call without an int max_tokens and Python serves it" +R3_LEGACY_THINKING: Final = "R3: native maps a float legacy thinking budget to effort low instead of Python's tier" +R4_BINDING_BETA: Final = "R4: native omits the thinking-binding-controls beta for block_binding" +R5_DEPLOYMENT_EFFORT: Final = "R5: native ignores the deployment's reasoning_effort_levels from model_info" +R7_DROP_ORDER: Final = "R7: native applies additional_drop_params before Python's unsupported-param check" +R8_VERSION_HEADER: Final = "R8: Python sends anthropic-version twice when the caller capitalises it; native once" +R9_QUERY_PATH: Final = "R9: native appends /v1/messages again to an api_base that ends in /v1/messages?query" +R11_PARAM_ERROR: Final = "R11: native raises BadRequestError where Python raises UnsupportedParamsError" +R12_METADATA_ERROR: Final = ( + "R12: Python raises a pydantic ValidationError for a non-string metadata.user_id; native a BadRequestError" +) +S1_S2_ERROR_HEADERS: Final = ( + "S1/S2: native drops upstream error headers and wraps the error body in a (status, body) tuple" +) +S2_ERROR_MESSAGE: Final = ( + "S2: native wraps the upstream error body in a (status, body) tuple and strips the provider from model" +) +S2_S12_ERROR_STATUS: Final = "S2/S12: native keeps the upstream status where Python maps it (413 to 400, 529 to 500)" +S3_TIMEOUT: Final = "S3: native raises APIConnectionError (500) for a timeout where Python raises Timeout (408)" +S5_STREAM_DEADLINE: Final = "S5: native applies the timeout to the whole stream instead of to each read" +S6_PRIVATE_KEYS: Final = "S6: native writes _hidden_params into the response dict" +S8_TYPED_RESPONSE: Final = "S8: native re-serializes the response and drops null usage and container" +S9_SYNC: Final = "S9: Python's sync handler raises ValueError for native providers; native serves the call" +S13_INVALID_JSON: Final = "S13: native reports an unparseable success body as an internal error, not AnthropicException" +S14_MISSING_KEY: Final = ( + "S14: native raises APIConnectionError for a missing API key where Python raises AuthenticationError" +) +C4_METADATA: Final = "C4: native callbacks get the raw Anthropic metadata, not Python's merged litellm metadata" +C6_RESPONSE_HEADERS: Final = "C6: native success logs carry no response_headers" +C7_API_KEY: Final = "C7: native callbacks see the caller's api_key where Python logs an empty string" +C8_STREAM_HIDDEN: Final = "C8: native streams lack the call metadata @client stamps into _hidden_params" +F_MOCK: Final = "F: mock_response is Python-only (detector mock_response)" +F_THINKING_RETRY: Final = ( + "F: retry without thinking after a rejected signature is Python-only (detector thinking_signature_retry)" +) +F_CACHE_INJECTION: Final = "F: prompt cache_control injection is Python-only (detector cache_control_injection)" +F_TOOL_CUSTOM: Final = "F: hoisting a tool's custom field is Python-only (detector tool_custom_field)" +F_STREAM_COST: Final = "F: cost injection into streamed usage is Python-only (detector stream_cost_injection)" + +KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType( + { + "plain_text": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "system_string": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "system_blocks_with_cache_control": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "multi_turn_tool_result": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "tools_with_tool_choice": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "tool_choice_none": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "tool_input_examples": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "sampling_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "stop_sequences": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "metadata_user_id": _gaps( + logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "metadata_extra_keys_are_filtered": _gaps( + logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "metadata_user_id_not_a_string": _gaps(outcome=R12_METADATA_ERROR, logged=R12_METADATA_ERROR), + "service_tier": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "thinking_enabled": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "thinking_disabled": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "legacy_thinking_on_adaptive_model_keeps_display": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "legacy_thinking_float_budget_on_adaptive_model": _gaps( + request=R3_LEGACY_THINKING, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "adaptive_thinking_with_block_binding": _gaps( + request=R4_BINDING_BETA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "reasoning_effort": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "reasoning_effort_on_adaptive_model": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "output_config_effort": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "output_format_json_schema": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "speed_on_model_without_speed": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR), + "speed_on_model_without_speed_drop_params": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "speed_on_fast_model": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "temperature_on_adaptive_model": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR), + "top_k_on_adaptive_model": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR), + "top_k_on_adaptive_model_drop_params": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "additional_drop_params_hides_rejected_temperature": _gaps( + request=R7_DROP_ORDER, + outcome=R7_DROP_ORDER, + hidden_params=R7_DROP_ORDER, + logged=R7_DROP_ORDER, + logged_metadata=R7_DROP_ORDER, + logged_api_key=R7_DROP_ORDER, + logged_call=R7_DROP_ORDER, + ), + "additional_drop_params_nested_path": _gaps( + logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "context_management_edits": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "container_skills": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "remote_mcp_servers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "inference_geo": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "top_level_cache_control": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "null_safeguards_are_stripped": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "empty_text_blocks_are_stripped": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "tool_use_ids_are_sanitized": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "provider_specific_fields_are_stripped": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "image_block": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "document_block": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "server_web_search_tool": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "extra_headers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "anthropic_beta_from_headers_and_extra_headers": _gaps( + logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "caller_anthropic_version_capitalised": _gaps( + request=R8_VERSION_HEADER, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "provider_specific_header": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "explicit_custom_llm_provider": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "bare_model_name": _gaps(**dict.fromkeys(ASPECTS[1:], A2_BARE_MODEL) | {"logged_model_info": None}), + "api_base_with_messages_path": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "api_base_with_messages_path_and_query": _gaps( + request=R9_QUERY_PATH, + logged=R9_QUERY_PATH, + logged_response_headers=C6_RESPONSE_HEADERS, + logged_api_key=C7_API_KEY, + ), + "api_key_from_environment": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "auth_token_from_environment": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "reasoning_auto_summary": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "global_drop_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "request_timeout_setting": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "litellm_metadata_and_metadata": _gaps( + logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "model_info_with_id": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "deployment_effort_levels": _gaps( + request=R5_DEPLOYMENT_EFFORT, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "response_with_tool_use": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_with_thinking": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_with_cache_usage": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_with_null_usage_and_container": _gaps(outcome=S8_TYPED_RESPONSE), + "response_without_stop_sequence": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_with_unknown_field": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_key_order": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_upstream_headers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "response_private_keys": _gaps( + outcome=S6_PRIVATE_KEYS, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "upstream_400": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_401": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_403": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_404": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_413": _gaps(outcome=S2_S12_ERROR_STATUS, logged_api_key=C7_API_KEY), + "upstream_429_with_retry_after": _gaps(outcome=S1_S2_ERROR_HEADERS, logged_api_key=C7_API_KEY), + "upstream_500": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_529_overloaded": _gaps(outcome=S2_S12_ERROR_STATUS, logged_api_key=C7_API_KEY), + "upstream_400_context_window": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_non_json_error": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "upstream_invalid_json_success": _gaps( + outcome=S13_INVALID_JSON, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "upstream_timeout": _gaps(outcome=S3_TIMEOUT, logged=S3_TIMEOUT, logged_api_key=C7_API_KEY), + "missing_max_tokens": _gaps(dispatch=A10_NO_MAX_TOKENS), + "missing_api_key": _gaps(outcome=S14_MISSING_KEY, logged=S14_MISSING_KEY), + "stream_text": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), + "stream_tool_use": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), + "stream_thinking": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), + "stream_error_event_mid_stream": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), + "stream_longer_than_timeout": _gaps( + outcome=S5_STREAM_DEADLINE, + hidden_params=S5_STREAM_DEADLINE, + logged=S5_STREAM_DEADLINE, + logged_metadata=S5_STREAM_DEADLINE, + logged_api_key=S5_STREAM_DEADLINE, + ), + "stream_upstream_429": _gaps(outcome=S1_S2_ERROR_HEADERS, logged_api_key=C7_API_KEY), + "stream_false_explicit": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "sync_text": _gaps( + request=S9_SYNC, + outcome=S9_SYNC, + hidden_params=S9_SYNC, + logged=S9_SYNC, + logged_metadata=S9_SYNC, + logged_response_headers=S9_SYNC, + logged_api_key=S9_SYNC, + logged_call=S9_SYNC, + ), + "sync_stream": _gaps( + request=S9_SYNC, + outcome=S9_SYNC, + hidden_params=S9_SYNC, + logged=S9_SYNC, + logged_metadata=S9_SYNC, + logged_response_headers=S9_SYNC, + logged_api_key=S9_SYNC, + logged_call=S9_SYNC, + ), + "sync_upstream_429": _gaps( + request=S9_SYNC, + outcome=S9_SYNC, + logged=S9_SYNC, + logged_response_headers=S9_SYNC, + logged_api_key=S9_SYNC, + logged_call=S9_SYNC, + ), + "mock_response": _gaps( + request=F_MOCK, + outcome=F_MOCK, + hidden_params=F_MOCK, + logged=F_MOCK, + logged_metadata=F_MOCK, + logged_api_key=F_MOCK, + logged_call=F_MOCK, + ), + "mock_response_rate_limit": _gaps( + request=F_MOCK, outcome=F_MOCK, logged=F_MOCK, logged_api_key=F_MOCK, logged_call=F_MOCK + ), + "thinking_replay": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "thinking_signature_rejected_then_retried": _gaps( + request=F_THINKING_RETRY, + outcome=F_THINKING_RETRY, + hidden_params=F_THINKING_RETRY, + logged=F_THINKING_RETRY, + logged_metadata=C4_METADATA, + logged_response_headers=C6_RESPONSE_HEADERS, + logged_api_key=C7_API_KEY, + ), + "cache_control_injection_points": _gaps( + request=F_CACHE_INJECTION, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "tool_custom_field": _gaps( + request=F_TOOL_CUSTOM, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "strict_tools_rejected_then_retried": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), + "modify_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "stream_cost_injection": _gaps(outcome=F_STREAM_COST, hidden_params=F_STREAM_COST, logged_api_key=C7_API_KEY), + "enable_prompt_caching": _gaps( + request=F_CACHE_INJECTION, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY + ), + "request_timeout_kwarg": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), + "stream_timeout_kwarg": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), + } +) + + +REQUEST_CASES: Final = ( + Case("plain_text"), + Case("system_string", _kw(system="Answer in one word.")), + Case( + "system_blocks_with_cache_control", + _kw(system=[{"type": "text", "text": "Answer in one word.", "cache_control": {"type": "ephemeral"}}]), + ), + Case("multi_turn_tool_result", _kw(messages=list(TOOL_TURNS), tools=[dict(WEATHER_TOOL)])), + Case( + "tools_with_tool_choice", + _kw(tools=[dict(WEATHER_TOOL)], tool_choice={"type": "tool", "name": "get_weather"}), + upstream=(_ok(TOOL_USE_RESPONSE),), + ), + Case("tool_choice_none", _kw(tools=[dict(WEATHER_TOOL)], tool_choice={"type": "none"})), + Case("tool_input_examples", _kw(tools=[{**WEATHER_TOOL, "input_examples": [{"city": "Paris"}]}])), + Case("sampling_params", _kw(temperature=0.3, top_p=0.9, top_k=40)), + Case("stop_sequences", _kw(stop_sequences=["\n\nHuman:"])), + Case("metadata_user_id", _kw(metadata={"user_id": "user-123"})), + Case("metadata_extra_keys_are_filtered", _kw(metadata={"user_id": "user-123", "session": "s-1"})), + Case("metadata_user_id_not_a_string", _kw(metadata={"user_id": 123})), + Case("service_tier", _kw(service_tier="auto"), gaps=_gaps(request="R1: native drops service_tier")), + Case("thinking_enabled", _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096)), + Case("thinking_disabled", _kw(thinking={"type": "disabled"})), + Case( + "legacy_thinking_on_adaptive_model_keeps_display", + _kw(thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + gaps=_gaps(request="R3: native replaces legacy thinking with a bare adaptive block and drops display"), + ), + Case( + "legacy_thinking_float_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": 8192.0}, max_tokens=16000), + model=OPUS_ADAPTIVE, + ), + Case( + "adaptive_thinking_with_block_binding", + _kw(thinking={"type": "adaptive", "block_binding": "strict"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + ), + Case("reasoning_effort", _kw(reasoning_effort="high", max_tokens=8192)), + Case("reasoning_effort_on_adaptive_model", _kw(reasoning_effort="low", max_tokens=8192), model=OPUS_ADAPTIVE), + Case("output_config_effort", _kw(output_config={"effort": "medium"}), model=OPUS_ADAPTIVE), + Case( + "output_format_json_schema", + _kw( + output_format={ + "type": "json_schema", + "schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + } + ), + ), + Case("speed_on_model_without_speed", _kw(speed="fast")), + Case("speed_on_model_without_speed_drop_params", _kw(speed="fast", drop_params=True)), + Case("speed_on_fast_model", _kw(speed="fast"), model=OPUS_FAST), + Case("temperature_on_adaptive_model", _kw(temperature=0.5), model=OPUS_ADAPTIVE), + Case("top_k_on_adaptive_model", _kw(top_k=5), model=OPUS_ADAPTIVE), + Case("top_k_on_adaptive_model_drop_params", _kw(top_k=5, drop_params=True), model=OPUS_ADAPTIVE), + Case( + "additional_drop_params_hides_rejected_temperature", + _kw(temperature=0.5, additional_drop_params=["temperature"]), + model=OPUS_ADAPTIVE, + ), + Case( + "additional_drop_params_nested_path", + _kw( + tools=[{**WEATHER_TOOL, "input_examples": [{"city": "Paris"}]}], + additional_drop_params=["tools[*].input_examples", "metadata"], + metadata={"user_id": "user-123"}, + ), + ), + Case( + "context_management_edits", + _kw(context_management={"edits": [{"type": "clear_tool_uses_20250919"}]}), + ), + Case( + "container_skills", _kw(container={"skills": [{"type": "anthropic", "skill_id": "pptx", "version": "latest"}]}) + ), + Case( + "remote_mcp_servers", + _kw(mcp_servers=[{"type": "url", "url": "https://mcp.example.com/sse", "name": "example"}]), + ), + Case("inference_geo", _kw(inference_geo="us")), + Case("top_level_cache_control", _kw(cache_control={"type": "ephemeral"})), + Case("null_safeguards_are_stripped", _kw(safeguards=None)), + Case( + "empty_text_blocks_are_stripped", + _kw( + messages=[ + {"role": "user", "content": [{"type": "text", "text": ""}, {"type": "text", "text": "hi"}]}, + ] + ), + ), + Case( + "tool_use_ids_are_sanitized", + _kw( + messages=[ + {"role": "user", "content": "Weather?"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "call:abc/1", "name": "get_weather", "input": dict(NO_INPUT)} + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call:abc/1", "content": "sunny"}]}, + ], + tools=[dict(WEATHER_TOOL)], + ), + ), + Case( + "provider_specific_fields_are_stripped", + _kw(messages=[{"role": "user", "content": "hi", "provider_specific_fields": {"x": 1}}]), + ), + Case( + "image_block", + _kw( + messages=[ + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_PIXEL}}, + {"type": "text", "text": "Describe it"}, + ], + } + ] + ), + ), + Case( + "document_block", + _kw( + messages=[ + { + "role": "user", + "content": [ + {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "Paris."}}, + {"type": "text", "text": "Summarize"}, + ], + } + ] + ), + ), + Case("server_web_search_tool", _kw(tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 2}])), + Case("extra_headers", _kw(extra_headers={"x-trace-id": "trace-1", "anthropic-beta": "files-api-2025-04-14"})), + Case( + "anthropic_beta_from_headers_and_extra_headers", + _kw(headers={"anthropic-beta": "client-beta-1"}, extra_headers={"anthropic-beta": "deploy-beta-2"}), + gaps=_gaps(request="R2: native keeps the last anthropic-beta source instead of the union"), + ), + Case("caller_anthropic_version_capitalised", _kw(extra_headers={"Anthropic-Version": "2023-06-01"})), + Case( + "provider_specific_header", + _kw(provider_specific_header={"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "yes"}}), + ), + Case("explicit_custom_llm_provider", _kw(custom_llm_provider="anthropic"), model="claude-sonnet-4-5"), + Case("bare_model_name", model="claude-sonnet-4-5"), + Case("api_base_with_messages_path", _kw(api_base_suffix="/v1/messages")), + Case( + "api_base_with_messages_path_and_query", + _kw(api_base_suffix="/v1/messages?beta=true"), + ), + Case("api_key_from_environment", _kw(api_key=None), env=MappingProxyType({"ANTHROPIC_API_KEY": API_KEY})), + Case( + "auth_token_from_environment", + _kw(api_key=None), + env=MappingProxyType({"ANTHROPIC_AUTH_TOKEN": "oauth-token"}), + ), + Case( + "reasoning_auto_summary", + _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + env=MappingProxyType({"LITELLM_REASONING_AUTO_SUMMARY": "true"}), + ), + Case("global_drop_params", _kw(top_k=5), model=OPUS_ADAPTIVE, settings=MappingProxyType({"drop_params": True})), + Case("request_timeout_setting", settings=MappingProxyType({"request_timeout": 30})), + Case( + "litellm_metadata_and_metadata", + _kw( + metadata={"user_id": "user-123"}, + litellm_metadata={"user_api_key": "hashed-key", "user_api_key_team_id": "t1"}, + ), + ), + Case("model_info_with_id", _kw(model_info={"id": "deployment-1", "db_model": False})), + Case( + "deployment_effort_levels", + _kw( + output_config={"effort": "max"}, + model_info={"id": "deployment-1", "reasoning_effort_levels": ["low", "high"]}, + ), + model=OPUS_ADAPTIVE, + ), +) + +RESPONSE_CASES: Final = ( + Case("response_with_tool_use", _kw(tools=[dict(WEATHER_TOOL)]), upstream=(_ok(TOOL_USE_RESPONSE),)), + Case("response_with_thinking", upstream=(_ok(THINKING_RESPONSE),)), + Case("response_with_cache_usage", upstream=(_ok(CACHED_USAGE_RESPONSE),)), + Case( + "response_with_null_usage_and_container", + upstream=(_ok({**TEXT_RESPONSE, "usage": None, "container": None}),), + ), + Case( + "response_without_stop_sequence", + upstream=(_ok({k: v for k, v in TEXT_RESPONSE.items() if k != "stop_sequence"}),), + ), + Case("response_with_unknown_field", upstream=(_ok({**TEXT_RESPONSE, "future_field": {"a": 1}}),)), + Case("response_key_order", upstream=(_ok({"usage": TEXT_RESPONSE["usage"], **TEXT_RESPONSE}),)), + Case( + "response_upstream_headers", + upstream=( + _ok(TEXT_RESPONSE, **{"request-id": "req_01Parity", "anthropic-ratelimit-requests-remaining": "99"}), + ), + ), + Case("response_private_keys", private_keys=True), +) + +FAILURE_CASES: Final = ( + Case("upstream_400", upstream=(_failure(400, "invalid_request_error", "max_tokens: too large"),)), + Case("upstream_401", upstream=(_failure(401, "authentication_error", "invalid x-api-key"),)), + Case("upstream_403", upstream=(_failure(403, "permission_error", "not allowed"),)), + Case("upstream_404", upstream=(_failure(404, "not_found_error", "model not found"),)), + Case("upstream_413", upstream=(_failure(413, "request_too_large", "too large"),)), + Case( + "upstream_429_with_retry_after", + upstream=( + _failure(429, "rate_limit_error", "slow down", **{"retry-after": "7", "request-id": "req_01Parity"}), + ), + ), + Case("upstream_500", upstream=(_failure(500, "api_error", "internal"),)), + Case("upstream_529_overloaded", upstream=(_failure(529, "overloaded_error", "Overloaded"),)), + Case( + "upstream_400_context_window", + upstream=(_failure(400, "invalid_request_error", "prompt is too long: 250000 tokens > 200000 maximum"),), + ), + Case("upstream_non_json_error", upstream=(ResponseSpec(body=b"bad gateway", status=502),)), + Case("upstream_invalid_json_success", upstream=(ResponseSpec(body=b"not json", status=200),)), + Case("upstream_timeout", _kw(timeout=0.5), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), delay=1.5),)), + Case("missing_max_tokens", _kw(max_tokens=None), upstream=()), + Case("missing_api_key", _kw(api_key=None), upstream=()), +) + +STREAM_CASES: Final = ( + Case("stream_text", _kw(stream=True), upstream=(_stream(TEXT_STREAM),)), + Case("stream_tool_use", _kw(stream=True, tools=[dict(WEATHER_TOOL)]), upstream=(_stream(TOOL_STREAM),)), + Case( + "stream_thinking", + _kw(stream=True, thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + upstream=(_stream(THINKING_STREAM),), + ), + Case("stream_error_event_mid_stream", _kw(stream=True), upstream=(_stream(ERROR_MID_STREAM),)), + Case( + "stream_longer_than_timeout", + _kw(stream=True, timeout=1), + upstream=(_stream(SLOW_STREAM, chunk_delay=0.25),), + ), + Case( + "stream_upstream_429", + _kw(stream=True), + upstream=(_failure(429, "rate_limit_error", "slow down", **{"retry-after": "3"}),), + ), + Case("stream_false_explicit", _kw(stream=False)), +) + +SYNC_CASES: Final = ( + Case("sync_text", sync=True), + Case("sync_stream", _kw(stream=True), upstream=(_stream(TEXT_STREAM),), sync=True), + Case("sync_upstream_429", upstream=(_failure(429, "rate_limit_error", "slow down"),), sync=True), +) + +FALLBACK_CASES: Final = ( + Case("mock_response", _kw(mock_response="Hi from the mock"), upstream=()), + Case("mock_response_rate_limit", _kw(mock_response="litellm.RateLimitError"), upstream=()), + Case( + "thinking_replay", + _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + ), + Case( + "thinking_signature_rejected_then_retried", + _kw(messages=list(THINKING_REPLAY), thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096), + upstream=( + _failure(400, "invalid_request_error", "messages.1.content.0: Invalid `signature` in `thinking` block"), + _ok(), + ), + ), + Case( + "cache_control_injection_points", + _kw(system="Answer in one word.", cache_control_injection_points=[{"location": "message", "role": "system"}]), + ), + Case("tool_custom_field", _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": True}}])), + Case( + "strict_tools_rejected_then_retried", + _kw(tools=[{**WEATHER_TOOL, "strict": True}], drop_params=True), + upstream=( + _failure(400, "invalid_request_error", "tools.0.strict: too many strict tools"), + _ok(), + ), + ), + Case("modify_params", _kw(max_tokens=1), settings=MappingProxyType({"modify_params": True})), + Case( + "stream_cost_injection", + _kw(stream=True), + upstream=(_stream(TEXT_STREAM),), + settings=MappingProxyType({"include_cost_in_streaming_usage": True}), + ), + Case("enable_prompt_caching", _kw(system="Answer in one word.", enable_prompt_caching=True)), + Case("request_timeout_kwarg", _kw(request_timeout=30)), + Case("stream_timeout_kwarg", _kw(stream=True, stream_timeout=30), upstream=(_stream(TEXT_STREAM),)), +) + +CASES: Final = (*REQUEST_CASES, *RESPONSE_CASES, *FAILURE_CASES, *STREAM_CASES, *SYNC_CASES, *FALLBACK_CASES) + + +@dataclass(frozen=True, slots=True) +class Leg: + fell_back: bool + requests: tuple[Mapping[str, object], ...] + outcome: Mapping[str, object] + hidden_params: Mapping[str, object] + logged: Mapping[str, object] + callback_kwargs: Mapping[str, object] + + +@dataclass(slots=True) +class FallbackProbe: + calls: int = 0 + + +_PYTHON_AMESSAGES: Final = cast(Callable[..., Awaitable[object]], dispatch._PYTHON_AMESSAGES) # pyright: ignore[reportPrivateUsage] # the Python baseline under comparison +_PYTHON_MESSAGES: Final = cast(Callable[..., object], dispatch._PYTHON_MESSAGES) # pyright: ignore[reportPrivateUsage] # the Python baseline under comparison +_PUBLIC_AMESSAGES: Final = cast(Callable[..., Awaitable[object]], dispatch.anthropic_messages) +_PUBLIC_MESSAGES: Final = cast(Callable[..., object], dispatch.anthropic_messages_handler) + + +@pytest.fixture +def native_messages() -> Iterator[FallbackProbe]: + probe: Final = FallbackProbe() + + async def python_afallback(*args: object, **kwargs: object) -> object: + probe.calls += 1 + return await _PYTHON_AMESSAGES(*args, **kwargs) + + def python_fallback(*args: object, **kwargs: object) -> object: + probe.calls += 1 + return _PYTHON_MESSAGES(*args, **kwargs) + + with ( + rebound(catalog, "RULES", (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_OUT), *catalog.RULES)), + rebound(dispatch, "_PYTHON_AMESSAGES", python_afallback), + rebound(dispatch, "_PYTHON_MESSAGES", python_fallback), + ): + yield probe + + +@contextmanager +def _environment(env: Mapping[str, str]) -> Generator[None]: + # LITELLM_RUST overrides the catalog rule, so a shell that sets it would route the native leg to Python. + names: Final = ( + *env, + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "LITELLM_REASONING_AUTO_SUMMARY", + "LITELLM_RUST", + ) + with pytest.MonkeyPatch.context() as patch: + for name in names: + patch.delenv(name, raising=False) + for name, value in env.items(): + patch.setenv(name, value) + yield + + +@contextmanager +def _settings(settings: Mapping[str, object]) -> Generator[None]: + saved: Final[dict[str, object]] = {name: cast(object, getattr(litellm, name)) for name in settings} + for name, value in settings.items(): + setattr(litellm, name, value) + try: + yield + finally: + for name, value in saved.items(): + setattr(litellm, name, value) + + +def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: + supplied: Final = dict(case.kwargs) + suffix: Final = supplied.pop("api_base_suffix", "") + kwargs: Final[dict[str, object]] = { + "model": case.model, + "max_tokens": 64, + "messages": [dict(USER)], + "api_key": API_KEY, + "api_base": f"{server.base_url}{suffix}", + **supplied, + } + return {name: value for name, value in kwargs.items() if value is not None or name in ("safeguards",)} + + +def _mapping(value: object) -> Mapping[str, object]: + return _LOGGED.validate_python(value) if isinstance(value, dict) else {} + + +def _normalized(text: str) -> str: + for pattern, replacement in _VOLATILE: + text = pattern.sub(replacement, text) # rebind-ok: successive substitutions + return text + + +def _error(error: BaseException, phase: str, chunks: Iterable[object] = ()) -> Mapping[str, object]: + header_items: Final = { + name.lower(): value for name, value in _mapping(getattr(error, "litellm_response_headers", None)).items() + } + return { + "kind": "error", + "phase": phase, + "class": type(error).__name__, + "status_code": getattr(error, "status_code", None), + "llm_provider": getattr(error, "llm_provider", None), + "model": getattr(error, "model", None), + "message": _normalized(str(error)), + "headers": {name: header_items.get(name) for name in ("retry-after", "request-id") if name in header_items}, + "partial": _stream_text(chunks), + } + + +def _stream_text(chunks: Iterable[object]) -> str: + return b"".join(chunk if isinstance(chunk, bytes) else str(chunk).encode() for chunk in chunks).decode() + + +def _response(result: object, private_keys: bool) -> Mapping[str, object]: + assert isinstance(result, dict), f"expected a dict response, got {type(result).__name__}" + body: Final = { + name: value + for name, value in cast(dict[str, object], result).items() + if private_keys or not name.startswith("_") + } + return {"kind": "response", "body": _OBJECT.validate_python(body)} + + +def _hidden_params(result: object) -> Mapping[str, object]: + hidden: Final = dict(_mapping(get_hidden_params_dict(result))) + headers: Final = _mapping(hidden.pop("additional_headers", None)) + header_names: Final = sorted(name for name in headers if name not in ("x-litellm-rust", "x-litellm-cache-key")) + return {"keys": sorted(hidden), "additional_headers": header_names} + + +async def _consume_async(stream: AsyncIterator[object]) -> tuple[list[object], BaseException | None]: + chunks: Final[list[object]] = [] + try: + async for chunk in stream: + chunks.append(chunk) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return chunks, error + return chunks, None + + +def _consume_sync(stream: Iterator[object]) -> tuple[list[object], BaseException | None]: + chunks: Final[list[object]] = [] + try: + for chunk in stream: + chunks.append(chunk) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return chunks, error + return chunks, None + + +async def _invoke( + case: Case, kwargs: Mapping[str, object], native: bool +) -> tuple[Mapping[str, object], Mapping[str, object]]: + if case.sync: + call = _PUBLIC_MESSAGES if native else _PYTHON_MESSAGES + try: + result = await asyncio.to_thread(lambda: call(**kwargs)) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return _error(error, "call"), {} + if isinstance(result, dict): + body = cast(Mapping[str, object], result) + return _response(body, case.private_keys), _hidden_params(body) + chunks, failure = await asyncio.to_thread(_consume_sync, cast(Iterator[object], result)) + else: + acall = _PUBLIC_AMESSAGES if native else _PYTHON_AMESSAGES + try: + result = await acall(**kwargs) + except Exception as error: # noqa: BLE001 # the raised error is part of the compared outcome + return _error(error, "call"), {} + if isinstance(result, dict): + body = cast(Mapping[str, object], result) + return _response(body, case.private_keys), _hidden_params(body) + chunks, failure = await _consume_async(cast(AsyncIterator[object], result)) + hidden: Final = _hidden_params(result) + if failure is not None: + return _error(failure, "stream", chunks), hidden + return { + "kind": "stream", + "chunk_types": sorted({type(chunk).__name__ for chunk in chunks}), + "text": _stream_text(chunks), + }, hidden + + +async def _terminal_events(recorder: RecordingLogger) -> tuple[str, ...]: + for _ in range(100): + await drain_logging() + if any(name in _TERMINAL_EVENTS for name in recorder.names): + break + await asyncio.sleep(0.05) + await asyncio.sleep(0.2) + await drain_logging() + return tuple(name for name in recorder.names if name in _TERMINAL_EVENTS) + + +def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping[str, object], Mapping[str, object]]: + terminal: Final = [event for event in recorder.events if event.name in _TERMINAL_EVENTS] + if not terminal: + return {"events": events}, {} + details: Final = _mapping(terminal[-1].kwargs) + standard: Final = _mapping(details.get("standard_logging_object")) + error_class: Final = _mapping(standard.get("error_information")).get("error_class") + litellm_params: Final = _mapping(details.get("litellm_params")) + metadata: Final = litellm_params.get("metadata") + metadata_keys: Final = _mapping(metadata).keys() + logged: Final[Mapping[str, object]] = { + "events": events, + **{name: standard.get(name) for name in _COMPARED_LOG_FIELDS}, + "error_class": error_class, + } + callback_kwargs: Final[Mapping[str, object]] = { + "logged_metadata": sorted(metadata_keys) if isinstance(metadata, dict) else metadata, + "logged_model_info": litellm_params.get("model_info"), + "logged_response_headers": bool(details.get("response_headers")), + "logged_api_key": details.get("api_key"), + "logged_call": { + "custom_llm_provider": details.get("custom_llm_provider"), + "model": details.get("model"), + "stream": details.get("stream"), + }, + } + return logged, callback_kwargs + + +async def _leg(case: Case, server: RecordingServer, probe: FallbackProbe | None) -> Leg: + with isolated_callback_registries(), _environment(case.env), _settings(case.settings): + recorder: Final = RecordingLogger() + litellm.callbacks.append(recorder) + for spec in case.upstream: + server.enqueue(replace(spec)) + request_count: Final = len(server.requests) + fallbacks_before: Final = 0 if probe is None else probe.calls + outcome, hidden = await _invoke(case, _kwargs(case, server), native=probe is not None) + events: Final = await _terminal_events(recorder) + server.responses.clear() + logged, callback_kwargs = _logged(recorder, events) + return Leg( + fell_back=probe is not None and probe.calls > fallbacks_before, + requests=tuple( + { + "method": sent.method, + "path": sent.path, + "body": sent.body if sent.body is not None else sent.raw_body.decode(errors="replace"), + "headers": { + name: value for name, value in sorted(sent.headers.items()) if name not in _TRANSPORT_HEADERS + }, + } + for sent in server.requests[request_count:] + ), + outcome=outcome, + hidden_params=hidden, + logged=logged, + callback_kwargs=callback_kwargs, + ) + + +@pytest.fixture +def parity_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.expected_requests = None + return recording_server + + +_LEGS: Final[dict[str, tuple[Leg, Leg]]] = {} + + +async def _legs(case: Case, server: RecordingServer, probe: FallbackProbe) -> tuple[Leg, Leg]: + if case.name not in _LEGS: + _LEGS[case.name] = (await _leg(case, server, probe=None), await _leg(case, server, probe=probe)) + _dump(case.name, *_LEGS[case.name]) + return _LEGS[case.name] + + +def _dump(name: str, python: Leg, native: Leg) -> None: + """Writes both legs to ``$MESSAGES_PARITY_DUMP/.json`` when that variable is set.""" + dump: Final = os.environ.get("MESSAGES_PARITY_DUMP") + if not dump: + return + os.makedirs(dump, exist_ok=True) + with open(os.path.join(dump, f"{name}.json"), "w") as out: + json.dump({"python": asdict(python), "native": asdict(native)}, out, indent=1, default=repr) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("case", "aspect"), tuple(param for case in CASES for param in case.params())) +async def test_native_messages_match_python( + case: Case, aspect: str, parity_server: RecordingServer, native_messages: FallbackProbe +) -> None: + python, native = await _legs(case, parity_server, native_messages) + match aspect: + case "dispatch": + assert not native.fell_back + case "request": + assert native.requests == python.requests + case "outcome": + assert native.outcome == python.outcome + case "hidden_params": + assert native.hidden_params == python.hidden_params + case "logged": + assert native.logged == python.logged + case _: + assert native.callback_kwargs.get(aspect) == python.callback_kwargs.get(aspect) diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py index d7b82767b97..7beeb20f109 100644 --- a/tests/test_litellm_rust/support/recording_server.py +++ b/tests/test_litellm_rust/support/recording_server.py @@ -25,6 +25,7 @@ class ResponseSpec: status: int = 200 headers: dict[str, str] = field(default_factory=dict) delay: float = 0 + chunk_delay: float = 0 events: tuple[tuple[str, object], ...] = () def payloads(self) -> tuple[bytes, ...]: @@ -89,7 +90,9 @@ def recording_service() -> Iterator[RecordingServer]: self.send_header(name, value) self.end_headers() try: - for payload in payloads: + for index, payload in enumerate(payloads): + if index and response.chunk_delay: + time.sleep(response.chunk_delay) self.wfile.write(payload) self.wfile.flush() except (BrokenPipeError, ConnectionResetError): From 34314a99f099811348c656819841469af506dca9 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 2 Nov 2026 21:02:52 -0800 Subject: [PATCH 08/34] fix(messages): make rust messages copy python messages behaviour (#49611) Closes the Rust vs Python /v1/messages differences the parity suite marks as known gaps, copying Python exactly. --- .../python_contract.json | 18 + .../callbacks-legacy-python/src/adapter.rs | 117 ++++++- .../callbacks-legacy-python/src/callbacks.rs | 51 ++- .../callbacks-legacy-python/src/python.rs | 6 + .../src/test_support.rs | 7 + litellm-rust/crates/host-http/tests/serve.rs | 14 +- .../crates/host-native/tests/driver.rs | 8 +- .../crates/host-python/tests/support/mod.rs | 2 +- litellm-rust/crates/host/src/interceptors.rs | 24 +- litellm-rust/crates/host/tests/observation.rs | 4 +- litellm-rust/crates/http/src/error.rs | 76 ++++- .../crates/inference-chat/src/handler.rs | 2 +- .../crates/inference-messages/Cargo.toml | 2 +- .../inference-messages/src/common_utils.rs | 74 ++++ .../crates/inference-messages/src/handler.rs | 191 ++++++++--- .../crates/inference-messages/src/lib.rs | 1 + .../tests/messages/request.rs | 14 +- .../tests/messages/response.rs | 25 +- .../tests/messages/stream.rs | 43 +++ .../crates/inference-ocr/src/handler.rs | 4 +- .../crates/inference-responses/src/handler.rs | 2 +- litellm-rust/crates/inference/src/context.rs | 8 +- litellm-rust/crates/inference/src/error.rs | 28 ++ .../src/formats/messages/request.rs | 7 +- .../src/formats/messages/response.rs | 36 +- .../crates/llms-types/src/recognized.rs | 10 + .../crates/llms/src/anthropic/common_utils.rs | 57 +++- .../llms/src/anthropic/messages/thinking.rs | 233 +++++++++++-- .../src/anthropic/messages/transformation.rs | 28 +- .../src/azure_ai/messages/transformation.rs | 2 +- .../llms/src/base_llm/messages/context.rs | 19 ++ litellm-rust/crates/llms/src/error.rs | 22 ++ .../src/vertex_ai/messages/transformation.rs | 2 +- .../crates/python-bridge/src/errors.rs | 59 +++- .../python-bridge/src/routes/messages/host.rs | 21 +- .../python-bridge/src/routes/messages/mod.rs | 13 +- .../host/callbacks_legacy_python.py | 45 ++- litellm/rust_bridge/host/routes/messages.py | 200 ++++++++++- .../sdk/messages/test_native_parity.py | 321 ++++++------------ .../cache/test_python_cache.py | 48 +-- .../messages/test_callbacks.py | 25 +- .../messages/test_streaming.py | 13 +- .../support/recording_server.py | 8 +- .../rust_bridge/host/routes/test_messages.py | 107 ++++++ .../host/test_callbacks_legacy_python.py | 54 +++ .../unit/rust_bridge/messages/test_secrets.py | 12 +- 46 files changed, 1595 insertions(+), 468 deletions(-) diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index f65516265c5..ab887bbc014 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -33,6 +33,17 @@ "api_key", "additional_args" ], + "record_response": [ + "logger", + "status", + "headers", + "body" + ], + "record_api_window": [ + "logger", + "start", + "end" + ], "defers_async_logging": [ "logger" ], @@ -91,6 +102,13 @@ "logger", "head" ], + "stamp_stream_head": [ + "head", + "logger", + "kwargs", + "start_time", + "end_time" + ], "stream_success": [ "logger", "url_route", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 9431b9d1326..1bb84184e4c 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -57,6 +57,8 @@ pub struct LegacyLogging { asynchronous: bool, internal: bool, cache_key: Option, + /// When the provider request went out, until its window is recorded (Messages only). + api_started: Option, } fn datetime(py: Python<'_>, epoch_seconds: f64) -> PyResult> { @@ -87,6 +89,60 @@ impl LegacyLogging { asynchronous, internal: false, cache_key: None, + api_started: None, + } + } + + /// The kwargs `Logging.update_from_kwargs` sees. Python's Messages handler takes `metadata` as + /// its own argument, so the logger never sees the caller's raw metadata, only the validated + /// request metadata merged in through `litellm_params`. Python updates the logger once, before + /// the call; the refresh once the result is ready leaves out `litellm_metadata` too, so the + /// logged metadata is not rebuilt from what the call wrote into it since. + fn logging_kwargs<'py>(&self, py: Python<'py>, refresh: bool) -> PyResult> { + let kwargs = self.call.kwargs().bind(py).copy()?; + if self.operation != LoggingOperation::Messages { + return Ok(kwargs); + } + let consumed: &[&str] = if refresh { + &["metadata", "litellm_metadata"] + } else { + &["metadata"] + }; + for name in consumed { + if kwargs.contains(name)? { + kwargs.del_item(name)?; + } + } + Ok(kwargs) + } + + /// Python's Messages handler hands the logger its request params, validated `metadata` + /// included, as `litellm_params`, beside the `aanthropic_messages` call flag. + fn logged_request_params( + &self, + context: &RequestContext, + ) -> Option> { + if self.operation != LoggingOperation::Messages { + return None; + } + let mut params = context + .optional_params + .as_object() + .cloned() + .unwrap_or_default(); + params.insert( + "aanthropic_messages".to_owned(), + Value::Bool(self.asynchronous), + ); + Some(params) + } + + /// Python times the provider exchange inside the async `AsyncHTTPHandler.post`; its sync + /// client is untimed. + fn close_api_window(&mut self, py: Python<'_>, end: f64) -> PyResult<()> { + match self.api_started.take() { + Some(start) => self.logger()?.record_api_window(py, start, end), + None => Ok(()), } } @@ -111,6 +167,16 @@ impl LegacyLogging { } } + /// The key `pre_call` logs. Python's Messages handler logs an empty string, never the key. + fn logged_api_key<'a>(&self, context: &'a RequestContext) -> Option<&'a str> { + match self.operation { + LoggingOperation::Messages => Some(""), + LoggingOperation::Completion | LoggingOperation::Responses | LoggingOperation::Ocr => { + context.api_key.as_ref().map(|api_key| api_key.expose()) + } + } + } + fn stream_billing(&self) -> Option { match self.operation { LoggingOperation::Messages => Some(PassThroughStream { @@ -377,7 +443,13 @@ impl LegacyLogging { context: &RequestContext, ) -> PyResult>> { let logger = self.logger()?; - logger.update_from_kwargs(py, self.call.kwargs(), &wire, context)?; + logger.update_from_kwargs( + py, + &self.logging_kwargs(py, false)?, + &wire, + context, + self.logged_request_params(context).as_ref(), + )?; let body = to_py(py, &wire.body)? .into_bound(py) .cast_into::()?; @@ -400,11 +472,14 @@ impl LegacyLogging { self.logger()?.pre_call( py, self.input_description(), - context.api_key.as_ref().map(|api_key| api_key.expose()), + self.logged_api_key(context), &body, &headers, &wire.url, )?; + if self.operation == LoggingOperation::Messages && self.asynchronous { + self.api_started = Some(epoch_seconds()); + } let headers = headers .iter() .map(|(name, value)| Ok((name.extract::()?, value.extract::()?))) @@ -455,7 +530,7 @@ impl LegacyLogging { py, ( &logger, - self.call.kwargs(), + self.logging_kwargs(py, true)?, &facts.provider.model, logger.getattr("optional_params")?, params, @@ -483,6 +558,16 @@ impl LegacyLogging { py: Python<'_>, raw: &RawResponse, ) -> PyResult> { + if self.operation == LoggingOperation::Messages { + // Python's Messages handler never calls `post_call` at receipt: it stores the raw + // response, and logging replays it through `AnthropicConfig.transform_response`. + self.close_api_window(py, epoch_seconds())?; + if let Some(head) = &raw.head { + self.logger()? + .record_response(py, head.status, &head.headers, &raw.body)?; + } + return Ok(HookStep::Ready(())); + } let api_key = self .request .as_ref() @@ -522,6 +607,9 @@ impl LegacyLogging { ) -> PyResult> { self.end = Some(datetime(py, timing.end_time)?); self.error = Some(error.clone_ref(py).into_value(py)); + if self.logger.is_some() { + self.close_api_window(py, timing.end_time)?; + } if self.stream.is_some() { return self.stream_failure(py); } @@ -543,6 +631,20 @@ impl LegacyLogging { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } + self.close_api_window(py, epoch_seconds())?; + if self.operation == LoggingOperation::Messages { + // Python's `@client` stamps the call metadata onto the stream it hands back. + Streaming::StampHead.call( + py, + ( + head.bind(py), + self.logger()?.object(py), + self.call.kwargs(), + &self.start, + datetime(py, epoch_seconds())?, + ), + )?; + } Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), @@ -1117,9 +1219,7 @@ assert logger.update['custom_llm_provider'] == 'edenai', logger.update let step = logging .before_provider_request(py, Box::new(wire), context) .unwrap(); - let raw = RawResponse { - body: "raw response".into(), - }; + let raw = RawResponse::new("raw response".into()); assert!(matches!( logging .on_event( @@ -1869,9 +1969,10 @@ assert logger.calls[1][1] is response py, &locals, c" -assert logger.names() == ['stream_opened', 'stream_success'], logger.calls +assert logger.names() == ['stamp_stream_head', 'stream_opened', 'stream_success'], logger.calls assert logger.calls[0][1] is head -chunks = logger.calls[1][1] +assert logger.calls[1][1] is head +chunks = logger.calls[2][1] assert len(chunks) == 2 assert chunks[0] is first assert chunks[1] is last diff --git a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs index f185f21a9c3..1951c9ebfb4 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/callbacks.rs @@ -5,6 +5,7 @@ use litellm_host::interceptors::{RequestContext, WireRequest}; use litellm_host_python::to_py; use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict}; +use serde_json::Value; use crate::logger::PythonLogger; use crate::python::{Logging, Wrapper}; @@ -15,9 +16,10 @@ pub trait LegacyCallbacks { fn update_from_kwargs( &self, py: Python<'_>, - kwargs: &Py, + kwargs: &Bound<'_, PyDict>, wire: &WireRequest, context: &RequestContext, + request_params: Option<&serde_json::Map>, ) -> PyResult<()>; /// `Logging.pre_call`. @@ -31,6 +33,20 @@ pub trait LegacyCallbacks { url: &str, ) -> PyResult<()>; + /// Records the provider's raw HTTP response on the logger, as Python's Messages handler + /// does with `httpx_response` and `response_headers`. + fn record_response( + &self, + py: Python<'_>, + status: u16, + headers: &[(String, String)], + body: &str, + ) -> PyResult<()>; + + /// Records one provider HTTP exchange's window, as Python's timed `AsyncHTTPHandler.post` + /// does with `llm_api_duration_ms` and the metadata's `llm_api_timing_windows`. + fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()>; + /// `Logging.post_call`. fn post_call( &self, @@ -83,12 +99,13 @@ impl LegacyCallbacks for PythonLogger { fn update_from_kwargs( &self, py: Python<'_>, - kwargs: &Py, + kwargs: &Bound<'_, PyDict>, wire: &WireRequest, context: &RequestContext, + request_params: Option<&serde_json::Map>, ) -> PyResult<()> { let secret_fields: Vec<&str> = context.secret_fields.iter().map(String::as_str).collect(); - let redacted_kwargs = redact(py, kwargs.bind(py), &secret_fields)?; + let redacted_kwargs = redact(py, kwargs, &secret_fields)?; let optional_params = redact( py, &to_py(py, &context.optional_params)? @@ -97,23 +114,23 @@ impl LegacyCallbacks for PythonLogger { &secret_fields, )?; let params = PyDict::new(py); - params.set_item( - "litellm_call_id", - kwargs.bind(py).get_item("litellm_call_id")?, - )?; + params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?; params.set_item("api_base", &wire.url)?; for name in ["logger_fn", "litellm_request_debug"] { - if let Some(value) = kwargs.bind(py).get_item(name)? { + if let Some(value) = kwargs.get_item(name)? { params.set_item(name, value)?; } } for name in custom_pricing_fields(py)? { - if let Some(value) = kwargs.bind(py).get_item(&name)? + if let Some(value) = kwargs.get_item(&name)? && !value.is_none() { params.set_item(name, value)?; } } + for (name, value) in request_params.into_iter().flatten() { + params.set_item(name, to_py(py, value)?)?; + } Logging::Update.call( py, ( @@ -145,6 +162,22 @@ impl LegacyCallbacks for PythonLogger { Ok(()) } + fn record_response( + &self, + py: Python<'_>, + status: u16, + headers: &[(String, String)], + body: &str, + ) -> PyResult<()> { + Logging::RecordResponse.call(py, (self.object(py), status, headers.to_vec(), body))?; + Ok(()) + } + + fn record_api_window(&self, py: Python<'_>, start: f64, end: f64) -> PyResult<()> { + Logging::RecordApiWindow.call(py, (self.object(py), start, end))?; + Ok(()) + } + fn post_call( &self, py: Python<'_>, diff --git a/litellm-rust/crates/callbacks-legacy-python/src/python.rs b/litellm-rust/crates/callbacks-legacy-python/src/python.rs index 6f1ee9dfd78..6d910d35f53 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/python.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/python.rs @@ -44,6 +44,10 @@ pub(crate) enum Logging { PreCall, #[strum(serialize = "post_call")] PostCall, + #[strum(serialize = "record_response")] + RecordResponse, + #[strum(serialize = "record_api_window")] + RecordApiWindow, #[strum(serialize = "defers_async_logging")] DefersAsync, #[strum(serialize = "defer_success")] @@ -77,6 +81,8 @@ pub(crate) enum DeploymentHooks { pub(crate) enum Streaming { #[strum(serialize = "stream_opened")] Opened, + #[strum(serialize = "stamp_stream_head")] + StampHead, #[strum(serialize = "stream_success")] Success, #[strum(serialize = "stream_failure")] diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 093ce60e510..8acfa7a898b 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -65,6 +65,12 @@ FAKES = { 'post_call': lambda logger, original_response, api_key, additional_args: logger.post_call( original_response, api_key, additional_args ), + 'record_response': lambda logger, status, headers, body: logger.model_call_details.update( + response_headers=dict(headers) + ), + 'record_api_window': lambda logger, start, end: logger.model_call_details.update( + llm_api_duration_ms=(end - start) * 1000 + ), 'defers_async_logging': lambda logger: bool(getattr(logger, '_defer_async_logging', False)), 'defer_success': lambda logger, pending: setattr(logger, '_native_pending_logging', pending), 'sync_success_for_async_call': lambda logger, response, start, end: logger.handle_sync_success_callbacks_for_async_calls( @@ -85,6 +91,7 @@ FAKES = { ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), 'stream_opened': lambda logger, head: logger.record('stream_opened', head), + 'stamp_stream_head': lambda head, logger, kwargs, start, end: logger.record('stamp_stream_head', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/host-http/tests/serve.rs b/litellm-rust/crates/host-http/tests/serve.rs index 2ba39ad4cd2..fd1a8940b33 100644 --- a/litellm-rust/crates/host-http/tests/serve.rs +++ b/litellm-rust/crates/host-http/tests/serve.rs @@ -215,9 +215,7 @@ async fn projection_custom_operations_and_hooks_feed_the_http_response(intercept ) .await?; route_hooks - .after_provider_response(RawResponse { - body: wire.url.clone(), - }) + .after_provider_response(RawResponse::new(wire.url.clone())) .await?; Ok(CallOutput::Complete(Bytes::from(wire.url))) }, @@ -501,9 +499,7 @@ async fn hook_rejection_stops_execution_and_is_reported_once( move |_, _, route_hooks, _observations| async move { if event { route_hooks - .after_provider_response(RawResponse { - body: "response".into(), - }) + .after_provider_response(RawResponse::new("response".into())) .await?; } else { route_hooks @@ -637,9 +633,7 @@ async fn unary_calls_use_into_response_after_hooks_and_before_success(intercepto ) .await?; route_hooks - .after_provider_response(RawResponse { - body: wire.url.clone(), - }) + .after_provider_response(RawResponse::new(wire.url.clone())) .await?; Ok(CallOutput::Complete(json!({"url": wire.url}))) }, @@ -689,7 +683,7 @@ async fn unary_failure_preserves_the_error_without_converting( None, |_, _, route_hooks, _observations| async move { route_hooks - .after_provider_response(RawResponse { body: "raw".into() }) + .after_provider_response(RawResponse::new("raw".into())) .await?; Err(TestError::Provider) }, diff --git a/litellm-rust/crates/host-native/tests/driver.rs b/litellm-rust/crates/host-native/tests/driver.rs index c41c1e5a43b..44c30234328 100644 --- a/litellm-rust/crates/host-native/tests/driver.rs +++ b/litellm-rust/crates/host-native/tests/driver.rs @@ -170,9 +170,7 @@ fn dispatching_call() -> TestMachine { ) .await?; route_hooks - .after_provider_response(RawResponse { - body: wire.url.clone(), - }) + .after_provider_response(RawResponse::new(wire.url.clone())) .await?; Ok(CallOutput::Complete(wire.url)) }, @@ -660,9 +658,7 @@ async fn response_interception_waits_and_can_reject_after_observation(#[case] re "input", Some(sender), |_, _, interceptors, observers| async move { - let raw = RawResponse { - body: "provider response".into(), - }; + let raw = RawResponse::new("provider response".into()); observers.unwrap().emit(CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, )); diff --git a/litellm-rust/crates/host-python/tests/support/mod.rs b/litellm-rust/crates/host-python/tests/support/mod.rs index 003fce74309..15e864c0718 100644 --- a/litellm-rust/crates/host-python/tests/support/mod.rs +++ b/litellm-rust/crates/host-python/tests/support/mod.rs @@ -467,7 +467,7 @@ pub fn success_machine() -> MachineFactory { .before_provider_request(wire(), context()) .await?; host.interceptors - .after_provider_response(RawResponse { body: "raw".into() }) + .after_provider_response(RawResponse::new("raw".into())) .await?; Ok(format!("{projected}|{signed}|{}", wire.url)) }) diff --git a/litellm-rust/crates/host/src/interceptors.rs b/litellm-rust/crates/host/src/interceptors.rs index 39af7c410e0..9398ea4bad6 100644 --- a/litellm-rust/crates/host/src/interceptors.rs +++ b/litellm-rust/crates/host/src/interceptors.rs @@ -28,6 +28,28 @@ pub struct RequestContext { #[derive(Clone, Debug, PartialEq, Eq)] pub struct RawResponse { pub body: String, + /// The provider's HTTP status and headers, when the route forwards them to logging. + pub head: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResponseHead { + pub status: u16, + pub headers: Vec<(String, String)>, +} + +impl RawResponse { + pub fn new(body: String) -> Self { + Self { body, head: None } + } + + #[must_use] + pub fn with_head(self, head: ResponseHead) -> Self { + Self { + head: Some(head), + ..self + } + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -186,7 +208,7 @@ mod tests { .await?; Interceptors::after_provider_response( &channel.interceptors, - RawResponse { body: "raw".into() }, + RawResponse::new("raw".into()), ) .await?; Ok((sent, ())) diff --git a/litellm-rust/crates/host/tests/observation.rs b/litellm-rust/crates/host/tests/observation.rs index 22872047d81..fba85b63bea 100644 --- a/litellm-rust/crates/host/tests/observation.rs +++ b/litellm-rust/crates/host/tests/observation.rs @@ -74,9 +74,7 @@ fn snapshots_do_not_retain_runtime_objects(capacity: NonZeroUsize, #[case] faile #[rstest] fn provider_snapshots_own_the_response_body(capacity: NonZeroUsize) { - let mut raw = RawResponse { - body: "provider response".into(), - }; + let mut raw = RawResponse::new("provider response".into()); let event: CallEvent<(), (), &RawResponse> = CallEvent::Execution(ExecutionEvent::ProviderResponseReceived { raw: &raw }); let expected = raw.clone(); diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs index 29c8b988c0b..7078d4aba2a 100644 --- a/litellm-rust/crates/http/src/error.rs +++ b/litellm-rust/crates/http/src/error.rs @@ -58,8 +58,20 @@ pub(crate) enum Repr { Network(#[source] reqwest::Error), #[error("upstream request timed out")] Timeout(#[source] reqwest::Error), + /// No bytes arrived within the read timeout, which the caller enforces per read; `body` says + /// the response head had already arrived. + #[error("upstream request timed out")] + ReadTimeout { + #[source] + elapsed: Elapsed, + body: bool, + }, #[error("upstream request failed with status {status}: {body}")] - Status { status: u16, body: UpstreamBody }, + Status { + status: u16, + body: UpstreamBody, + headers: Vec<(String, String)>, + }, #[error("invalid WebSocket URL")] WebSocketUrl(#[source] url::ParseError), @@ -136,13 +148,59 @@ impl Repr { impl Error { pub fn upstream(status: u16, body: String) -> Self { + Self::upstream_with_headers(status, body, Vec::new()) + } + + /// An upstream failure that keeps the provider's response headers (`retry-after`, + /// `request-id`) for the caller. + pub fn upstream_with_headers( + status: u16, + body: String, + headers: Vec<(String, String)>, + ) -> Self { Repr::Status { status, body: UpstreamBody(body), + headers, } .into() } + /// The provider's response headers on an upstream failure; empty for every other error. + pub fn upstream_headers(&self) -> &[(String, String)] { + match &*self.0 { + Repr::Status { headers, .. } => headers, + _ => &[], + } + } + + /// A timeout reached before the connection was established, as opposed to while reading. + pub fn is_connect_timeout(&self) -> bool { + matches!(&*self.0, Repr::Timeout(error) if error.is_connect()) + } + + /// A read the caller timed itself (httpx's per-read timeout) ran out. + pub fn read_timeout(elapsed: Elapsed) -> Self { + Repr::ReadTimeout { + elapsed, + body: false, + } + .into() + } + + /// A read of the response body the caller timed itself ran out, after the head arrived. + pub fn body_read_timeout(elapsed: Elapsed) -> Self { + Repr::ReadTimeout { + elapsed, + body: true, + } + .into() + } + + pub fn is_body_read_timeout(&self) -> bool { + matches!(&*self.0, Repr::ReadTimeout { body: true, .. }) + } + pub fn computed_header(name: impl Into) -> Self { Repr::ComputedHeader(name.into()).into() } @@ -169,6 +227,7 @@ impl Error { match &*self.0 { Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::Handshake(_) | Repr::HandshakeTimeout(_) @@ -227,6 +286,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::Handshake(_) | Repr::HandshakeTimeout(_) @@ -259,6 +319,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -295,6 +356,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -333,6 +395,7 @@ impl Error { | Repr::Signature(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) @@ -356,7 +419,10 @@ impl Error { pub fn is_timeout(&self) -> bool { match &*self.0 { - Repr::Timeout(_) | Repr::HandshakeTimeout(_) | Repr::DownloadTimeout(_) => true, + Repr::Timeout(_) + | Repr::ReadTimeout { .. } + | Repr::HandshakeTimeout(_) + | Repr::DownloadTimeout(_) => true, Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) @@ -392,7 +458,7 @@ impl Error { pub fn upstream_response(&self) -> Option<(u16, &str)> { match &*self.0 { - Repr::Status { status, body } => Some((*status, &body.0)), + Repr::Status { status, body, .. } => Some((*status, &body.0)), Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) @@ -408,6 +474,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) | Repr::WebSocketSchemeRewrite(_) @@ -431,7 +498,7 @@ impl Error { pub fn status(&self) -> Option { match &*self.0 { Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status), - Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408), + Repr::Timeout(_) | Repr::ReadTimeout { .. } | Repr::DownloadTimeout(_) => Some(408), Repr::Read { .. } | Repr::InvalidPem { .. } | Repr::Client(_) @@ -482,6 +549,7 @@ impl Error { | Repr::Connect(_) | Repr::Network(_) | Repr::Timeout(_) + | Repr::ReadTimeout { .. } | Repr::Status { .. } | Repr::WebSocketUrl(_) | Repr::WebSocketScheme(_) diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index f948e8fffaf..cdcc3e911b9 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -108,7 +108,7 @@ pub(super) async fn execute( if !status.is_success() { return Err(litellm_http::Error::upstream(status.as_u16(), text).into()); } - let raw = RawResponse { body: text.clone() }; + let raw = RawResponse::new(text.clone()); if let Some(observers) = observers { observers.emit(litellm_host::lifecycle::CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference-messages/Cargo.toml b/litellm-rust/crates/inference-messages/Cargo.toml index ac52cb65a0d..0c5be52550d 100644 --- a/litellm-rust/crates/inference-messages/Cargo.toml +++ b/litellm-rust/crates/inference-messages/Cargo.toml @@ -24,6 +24,7 @@ litellm-tracing.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } +tokio.workspace = true tracing.workspace = true [dev-dependencies] @@ -34,5 +35,4 @@ litellm-host-native.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-inference-testing.workspace = true rstest.workspace = true -tokio.workspace = true wiremock.workspace = true diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index a57eca1c6a9..e5ece2837d8 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -35,6 +35,14 @@ impl MessagesProvider { pub(crate) fn config(self) -> &'static dyn BaseMessagesConfig { self.config } + + /// Python's `MESSAGES_SERVICE_TIER_PROVIDERS`: everyone else has `service_tier` dropped. + pub(crate) fn accepts_service_tier(self) -> bool { + matches!( + self.provider, + LlmProviders::Anthropic | LlmProviders::AzureAi | LlmProviders::BedrockMantle + ) + } } /// Python's `get_provider_anthropic_messages_config`: Vertex AI serves only its Claude @@ -119,3 +127,69 @@ mod tests { ); } } + +/// Python's `_merge_messages_header_sources`: later sources win per header name, except +/// `anthropic-beta` (any casing), whose values from every source are unioned, deduplicated and +/// sorted into one lowercase `anthropic-beta` entry. +pub fn merge_messages_header_sources( + sources: impl IntoIterator>, +) -> Map { + let mut merged = Map::new(); + let mut betas = std::collections::BTreeSet::new(); + for (name, value) in sources.into_iter().flatten() { + if !name.eq_ignore_ascii_case("anthropic-beta") { + merged.insert(name, value); + continue; + } + let values: Vec<&str> = match &value { + Value::String(text) => vec![text.as_str()], + Value::Array(entries) => entries.iter().filter_map(Value::as_str).collect(), + _ => Vec::new(), + }; + betas.extend( + values + .iter() + .flat_map(|value| value.split(',')) + .map(str::trim) + .filter(|beta| !beta.is_empty()) + .map(str::to_owned), + ); + } + if !betas.is_empty() { + merged.insert( + "anthropic-beta".to_owned(), + Value::String(betas.into_iter().collect::>().join(",")), + ); + } + merged +} + +#[cfg(test)] +mod merge_tests { + use serde_json::{Map, Value, json}; + + use super::merge_messages_header_sources; + + fn map(value: Value) -> Map { + value.as_object().cloned().unwrap() + } + + #[test] + fn later_sources_win_except_anthropic_beta_which_is_unioned_and_sorted() { + let merged = merge_messages_header_sources([ + map(json!({"x-a": "1", "anthropic-beta": "b-2, a-1"})), + map(json!({"x-a": "2", "Anthropic-Beta": ["c-3", 7, "a-1"]})), + map(json!({"anthropic-beta": 5})), + ]); + assert_eq!( + Value::Object(merged), + json!({"x-a": "2", "anthropic-beta": "a-1,b-2,c-3"}) + ); + } + + #[test] + fn no_beta_values_leave_no_beta_header() { + let merged = merge_messages_header_sources([map(json!({"anthropic-beta": ""}))]); + assert!(merged.is_empty()); + } +} diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 2e647688a1a..e0a7aa5adb8 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -10,9 +10,9 @@ use litellm_core_utils::{ get_provider_specific_headers::get_provider_specific_headers, settings::Lookup, }; use litellm_host::interceptors::{ - Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest, + Cost, ExecutionFacts, Interceptors, ProviderIdentity, RawResponse, RequestContext, + ResponseHead, ResultSource, WireRequest, }; -use litellm_http::request::with_default_headers; use litellm_llms::base_llm::{ auth::{Authenticated, ValidatedEnvironment, resolve_auth}, messages::{ @@ -21,14 +21,16 @@ use litellm_llms::base_llm::{ transformation::BaseMessagesConfig, }, }; -use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse}; +use litellm_llms_types::formats::messages::{ + MessagesOptionalParams, MessagesRequest, MessagesResponse, +}; use litellm_secrets::source::Secrets; use litellm_tracing::ByteChunk; use serde_json::Value; use super::{ Error, MessagesCall, MessagesCallResponse, MessagesRoute, - common_utils::{MessagesProvider, string_headers}, + common_utils::{MessagesProvider, merge_messages_header_sources, string_headers}, constants::MESSAGES_TIMEOUT_SECS, route::Messages, types::invalid_request, @@ -126,14 +128,18 @@ async fn provider_call( MessagesRequest { model, ..body }, shaping.settings.reasoning_auto_summary, )?; + let transform_context = + MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params); + let filtered = requested_optional_params(sanitized, provider, &transform_context)?; let trimmed = - without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; + without_additional_drop_params(filtered, &shaping.settings.additional_drop_params)?; let scoped = get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str()); - let forwarded = string_headers(Some( - extra_headers.into_iter().flatten().chain(scoped).collect(), - ))?; + let forwarded = string_headers(Some(merge_messages_header_sources([ + extra_headers.unwrap_or_default(), + scoped, + ])))?; let validated = config.validate_environment( forwarded, api_key.as_deref(), @@ -144,16 +150,13 @@ async fn provider_call( )?; let body = config.request_body( - config.transform_anthropic_messages_request( - trimmed, - &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params), - )?, + config.transform_anthropic_messages_request(trimmed, &transform_context)?, &validated.headers, shaping.settings.drop_params, )?; let environment = ValidatedEnvironment { headers: config.request_headers( - with_default_headers(validated.headers, config.default_headers()), + with_python_default_headers(validated.headers, config.default_headers()), &body, ), auth: validated.auth, @@ -184,7 +187,7 @@ async fn provider_call( .before_provider_request( WireRequest { url, - headers: authenticated.headers, + headers: joined_header_values(authenticated.headers), body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?), }, request_context, @@ -213,6 +216,64 @@ async fn provider_call( }) } +/// Python sends its header dict through `httpx.Headers`, which folds names that differ only in +/// case into one lowercase header whose values are joined with `", "`. +fn joined_header_values(headers: Vec<(String, String)>) -> Vec<(String, String)> { + let mut joined: Vec<(String, String)> = Vec::with_capacity(headers.len()); + let mut positions: std::collections::HashMap = + std::collections::HashMap::with_capacity(headers.len()); + for (name, value) in headers { + match positions.entry(name.to_ascii_lowercase()) { + std::collections::hash_map::Entry::Occupied(position) => { + let present = &mut joined[*position.get()].1; + present.push_str(", "); + present.push_str(&value); + } + std::collections::hash_map::Entry::Vacant(position) => { + position.insert(joined.len()); + joined.push((name, value)); + } + } + } + joined +} + +/// Python's `finalize_messages_headers` adds a default only when the exact lowercase name is +/// absent, so a caller's `Anthropic-Version` is sent alongside the default, not instead of it. +fn with_python_default_headers( + headers: Vec<(String, String)>, + defaults: &[(&str, &str)], +) -> Vec<(String, String)> { + let missing: Vec<(String, String)> = defaults + .iter() + .filter(|(name, _)| !headers.iter().any(|(present, _)| present == name)) + .map(|(name, value)| ((*name).to_string(), (*value).to_string())) + .collect(); + headers.into_iter().chain(missing).collect() +} + +/// Python's `get_requested_anthropic_messages_optional_param`, which gates speed, service tier +/// and sampling params before `additional_drop_params` and the environment check run. +fn requested_optional_params( + request: MessagesRequest, + provider: MessagesProvider, + context: &MessagesTransformContext, +) -> Result { + let request = litellm_llms::anthropic::messages::transformation::drop_unsupported_params( + request, context, + )?; + if provider.accepts_service_tier() { + return Ok(request); + } + Ok(MessagesRequest { + params: MessagesOptionalParams { + service_tier: None, + ..request.params + }, + ..request + }) +} + fn without_additional_drop_params( request: MessagesRequest, paths: &[String], @@ -245,6 +306,7 @@ async fn call_provider( } = request; let provider_name = provider.as_str(); log_request_body(provider_name, stream, &wire.body); + let timeout = timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS)); let response = send( http, Authenticated { @@ -254,10 +316,11 @@ async fn call_provider( &wire.url, &wire.body, timeout, + stream, ) .await?; if !response.status().is_success() { - return Err(provider_error(response).await); + return Err(provider_error(response, timeout).await); } let config = provider.config(); if stream { @@ -265,11 +328,24 @@ async fn call_provider( response, config.stream_decoder(), provider_name, + timeout, )); } + let head = ResponseHead { + status: response.status().as_u16(), + headers: litellm_http::request::header_pairs(response.headers()), + }; + let status = head.status; let text = response.text().await.map_err(litellm_http::Error::from)?; log_response_body(&text); - context.response_received(&text).await?; + context + .raw_response_received(RawResponse::new(text.clone()).with_head(head)) + .await?; + if serde_json::from_str::(&text).is_err() { + // Python raises `AnthropicError(status_code=, message=)` for a + // body it cannot parse, and the public mapping classifies it by that status. + return Err(litellm_http::Error::upstream(status, text).into()); + } decode_response(config, &identity.model, &text) .map(|message| MessagesCallResponse::Complete(Box::new(message))) } @@ -281,30 +357,53 @@ fn serialize_failure(err: serde_json::Error) -> Error { )) } +/// Python's httpx timeout bounds each phase and each read, never the whole exchange. A complete +/// response keeps the client's overall deadline; a stream is bounded until its head arrives and +/// then per read (see [`timed_chunks`]), so a long stream that keeps sending is not cut off. async fn send( http: &litellm_http::Client, authenticated: Authenticated, url: &str, body: &Value, - timeout: Option, + timeout: Duration, + stream: bool, ) -> Result { - let request = outbound_request( - authenticated, - url, - body, - Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))), - )?; - Ok(litellm_inference::outbound::send(request, http).await?) + let request = outbound_request(authenticated, url, body, (!stream).then_some(timeout))?; + if !stream { + return Ok(litellm_inference::outbound::send(request, http).await?); + } + tokio::time::timeout(timeout, litellm_inference::outbound::send(request, http)) + .await + .map_err(litellm_http::Error::read_timeout)? + .map_err(Error::from) } -async fn provider_error(response: reqwest::Response) -> Error { +/// The stream's body, each read bounded by `timeout` as httpx bounds each read. +fn timed_chunks( + response: reqwest::Response, + timeout: Duration, +) -> impl futures_util::Stream> + Send + 'static { + futures_util::stream::try_unfold(response, move |mut response| async move { + let chunk = tokio::time::timeout(timeout, response.chunk()) + .await + .map_err(litellm_http::Error::body_read_timeout)? + .map_err(litellm_http::Error::from)?; + Ok(chunk.map(|chunk| (chunk, response))) + }) +} + +/// The provider's error, its body read under `timeout`: a stream has no overall deadline once +/// its head arrives, so an error body that stalls would otherwise hold the call forever. +async fn provider_error(response: reqwest::Response, timeout: Duration) -> Error { let status = response.status().as_u16(); - match response.text().await { - Ok(text) => { + let headers = litellm_http::request::header_pairs(response.headers()); + match tokio::time::timeout(timeout, response.text()).await { + Ok(Ok(text)) => { log_error_body(status, &text); - litellm_http::Error::upstream(status, text).into() + litellm_http::Error::upstream_with_headers(status, text, headers).into() } - Err(error) => litellm_http::Error::from(error).into(), + Ok(Err(error)) => litellm_http::Error::from(error).into(), + Err(elapsed) => litellm_http::Error::read_timeout(elapsed).into(), } } @@ -329,22 +428,22 @@ fn streaming_response( response: reqwest::Response, decoder: Option, provider: &'static str, + timeout: Duration, ) -> MessagesCallResponse { let headers = response .headers() .iter() .filter_map(|(name, value)| Some((name.to_string(), value.to_str().ok()?.to_string()))) .collect(); + let chunks = timed_chunks(response, timeout) + .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)); let chunks = match decoder { - None => futures_util::stream::try_unfold(response, move |mut response| async move { - let chunk = response.chunk().await.map_err(litellm_http::Error::from)?; - Ok(chunk.map(|chunk| { - log_chunk(provider, "provider_response", &chunk); - (chunk, response) - })) - }) - .boxed(), - Some(decode) => decoded_chunks(response, decode, provider), + None => chunks.map_err(Error::from).boxed(), + Some(decode) => decoded_chunks( + chunks.map_err(std::io::Error::other).boxed(), + decode, + provider, + ), }; MessagesCallResponse::Stream { head: super::route::MessagesStreamHead { headers }, @@ -353,15 +452,10 @@ fn streaming_response( } fn decoded_chunks( - response: reqwest::Response, + bytes: ByteStream, decode: StreamDecoder, provider: &'static str, ) -> BoxStream<'static, Result> { - let bytes: ByteStream = response - .bytes_stream() - .inspect_ok(move |chunk| log_chunk(provider, "provider_response", chunk)) - .map_err(|error| std::io::Error::other(litellm_http::Error::from(error))) - .boxed(); futures_util::stream::try_unfold(decode(bytes), move |mut events| async move { let Some(event) = events.try_next().await? else { return Ok(None); @@ -964,9 +1058,12 @@ mod tests { .send() .await .unwrap(); - let MessagesCallResponse::Stream { mut chunks, .. } = - streaming_response(response, Some(streaming::event_stream), "test") - else { + let MessagesCallResponse::Stream { mut chunks, .. } = streaming_response( + response, + Some(streaming::event_stream), + "test", + Duration::from_secs(5), + ) else { panic!("a streaming response returns chunks"); }; diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index bfefb9c63b0..745442d0314 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -14,6 +14,7 @@ use std::sync::Arc; use crate::common_utils::messages_provider; +pub use common_utils::merge_messages_header_sources; pub use litellm_inference::Error; pub use types::{ MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, diff --git a/litellm-rust/crates/inference-messages/tests/messages/request.rs b/litellm-rust/crates/inference-messages/tests/messages/request.rs index e53bdb8d252..216aa96fbe6 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/request.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/request.rs @@ -440,7 +440,12 @@ async fn an_oauth_key_sends_the_browser_access_header_and_the_oauth_beta(call: M #[case::deepseek("deepseek")] #[case::minimax("minimax")] #[tokio::test] -async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[case] provider: &str) { +/// Python adds a default unless that exact lowercase name is present, then folds the casings +/// into one header, so a capitalised caller header is sent beside the default. +async fn capitalised_caller_protocol_headers_are_joined_with_the_defaults( + call: MessagesCall, + #[case] provider: &str, +) { let upstream = upstream([message_response()]).await; run_message(MessagesCall { @@ -457,10 +462,13 @@ async fn caller_protocol_headers_win_over_the_defaults(call: MessagesCall, #[cas .await; let request = only_request(&upstream).await; - assert_eq!(request.header_values("anthropic-version"), ["2024-01-01"]); + assert_eq!( + request.header_values("anthropic-version"), + ["2024-01-01, 2023-06-01"] + ); assert_eq!( request.header_values("content-type"), - ["application/json; charset=utf-8"] + ["application/json; charset=utf-8, application/json"] ); } diff --git a/litellm-rust/crates/inference-messages/tests/messages/response.rs b/litellm-rust/crates/inference-messages/tests/messages/response.rs index 7aff1286858..76813fe0a34 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/response.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/response.rs @@ -225,14 +225,25 @@ async fn an_upstream_error_keeps_its_status_and_body(call: MessagesCall, #[case] } #[rstest] -#[case::not_json(ResponseTemplate::new(200).set_body_string("not json"))] -#[case::not_a_message(json_response(json!({"unexpected": true})))] #[tokio::test] -async fn an_unreadable_success_body_is_an_invalid_response( - call: MessagesCall, - #[case] response: ResponseTemplate, -) { - let upstream = upstream([response]).await; +async fn an_unparseable_success_body_is_reported_as_the_upstream_answer(call: MessagesCall) { + let upstream = upstream([ResponseTemplate::new(200).set_body_string("not json")]).await; + + let error = run(MessagesCall { + api_key: Some("sk".into()), + api_base: Some(upstream.uri()), + ..call + }) + .await + .expect_err("an unparseable body fails"); + + assert_eq!(error.upstream_response(), Some((200, "not json"))); +} + +#[rstest] +#[tokio::test] +async fn an_unreadable_success_body_is_an_invalid_response(call: MessagesCall) { + let upstream = upstream([json_response(json!({"unexpected": true}))]).await; let error = run(MessagesCall { api_key: Some("sk".into()), diff --git a/litellm-rust/crates/inference-messages/tests/messages/stream.rs b/litellm-rust/crates/inference-messages/tests/messages/stream.rs index 4a8a061f764..99ba08495aa 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/stream.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/stream.rs @@ -432,6 +432,49 @@ async fn dropping_the_sdk_stream_closes_the_unfinished_upstream( .unwrap(); } +/// Answers with an error head whose promised body never arrives. +async fn stalling_error_upstream() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let connection = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0; 4096]; + let _ = socket.read(&mut request).await; + socket + .write_all(b"HTTP/1.1 429 Too Many Requests\r\ncontent-type: application/json\r\ncontent-length: 100\r\n\r\n") + .await + .unwrap(); + let _ = socket.read_to_end(&mut Vec::new()).await; + }); + (base, connection) +} + +#[rstest] +#[tokio::test] +async fn the_timeout_covers_a_stalled_stream_error_body(call: MessagesCall) { + let (base, connection) = stalling_error_upstream().await; + let host = RecordingStreamHost::new( + MessagesCall { + timeout: Some(Duration::from_millis(300)), + ..streaming(call, base) + }, + usize::MAX, + ); + + let error = tokio::time::timeout(Duration::from_secs(5), stream_through(&host)) + .await + .expect("the stalled error body gives up within the timeout") + .expect_err("a stalled error body fails the call"); + + assert!(error.dispatched(), "{error:?}"); + assert_eq!(error.upstream_response(), None); + assert!(host.seen.into_inner().unwrap().is_empty()); + tokio::time::timeout(Duration::from_secs(5), connection) + .await + .expect("timing out closes the upstream connection") + .unwrap(); +} + #[rstest] #[tokio::test] async fn the_sdk_yields_a_body_error_once_after_delivered_chunks(call: MessagesCall) { diff --git a/litellm-rust/crates/inference-ocr/src/handler.rs b/litellm-rust/crates/inference-ocr/src/handler.rs index c126a192153..be23af60c69 100644 --- a/litellm-rust/crates/inference-ocr/src/handler.rs +++ b/litellm-rust/crates/inference-ocr/src/handler.rs @@ -91,9 +91,7 @@ impl> CallHooks for OcrCallHooks<'_, H> { } fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> { - let raw = RawResponse { - body: String::from_utf8_lossy(body).into_owned(), - }; + let raw = RawResponse::new(String::from_utf8_lossy(body).into_owned()); if let Some(observers) = self.observers { observers.emit(litellm_host::lifecycle::CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs index 19092a63a14..f81a51ad192 100644 --- a/litellm-rust/crates/inference-responses/src/handler.rs +++ b/litellm-rust/crates/inference-responses/src/handler.rs @@ -114,7 +114,7 @@ pub(super) async fn execute( }); } let body = response.text().await.map_err(litellm_http::Error::from)?; - let raw = RawResponse { body: body.clone() }; + let raw = RawResponse::new(body.clone()); if let Some(observers) = observers { observers.emit(litellm_host::lifecycle::CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs index 963aa7b90c5..ef2635d24a1 100644 --- a/litellm-rust/crates/inference/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -42,9 +42,11 @@ impl<'a, I: Interceptors> CallContext<'a, I> { } pub async fn response_received(&self, body: &str) -> Result<(), Error> { - let raw = RawResponse { - body: body.to_owned(), - }; + self.raw_response_received(RawResponse::new(body.to_owned())) + .await + } + + pub async fn raw_response_received(&self, raw: RawResponse) -> Result<(), Error> { if let Some(observers) = &self.observers { observers.emit(CallEvent::Execution( ExecutionEvent::ProviderResponseReceived { raw: raw.clone() }, diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index a8e6fb4e1b3..841e0c53799 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -146,6 +146,34 @@ impl Error { } } + /// The provider's response headers on an upstream failure; empty for every other error. + pub fn upstream_headers(&self) -> &[(String, String)] { + match self { + Self::Llm(error) => error.upstream_headers(), + Self::Http(error) => error.upstream_headers(), + _ => &[], + } + } + + /// A read of a streamed body timed out after the response head arrived. + pub fn body_read_timeout(&self) -> bool { + match self { + Self::Llm(error) => error.is_body_read_timeout(), + Self::Http(error) => error.is_body_read_timeout(), + _ => false, + } + } + + /// `Some(connect)` when the call timed out, where `connect` says the connection was never + /// established; `None` for every other error. + pub fn timeout(&self) -> Option { + match self { + Self::Llm(error) => error.is_timeout(), + Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()), + _ => None, + } + } + pub fn upstream_response(&self) -> Option<(u16, &str)> { match self { Self::Llm(error) => error.upstream_response(), diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index 9dbfa068025..d30d21e74e7 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -264,7 +264,12 @@ pub enum ThinkingDisplay { #[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct EnabledThinking { - #[serde(default, skip_serializing_if = "Option::is_none")] + /// An explicit `null` stays `Unrecognized(Null)`: Python rejects it, but defaults a missing key to 0. + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::recognized::present" + )] pub budget_tokens: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub display: Option>, diff --git a/litellm-rust/crates/llms-types/src/formats/messages/response.rs b/litellm-rust/crates/llms-types/src/formats/messages/response.rs index 9b0c5d31b3a..276a30a6cb4 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/response.rs @@ -10,14 +10,29 @@ pub struct MessagesResponse { pub content: Vec, pub stop_reason: Option, pub stop_sequence: Option, - #[serde(skip_serializing_if = "Option::is_none")] + /// `Some(Value::Null)` when the provider sent an explicit `null`, which Python keeps. + #[serde( + default, + deserialize_with = "present_value", + skip_serializing_if = "Option::is_none" + )] pub usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde( + default, + deserialize_with = "present_value", + skip_serializing_if = "Option::is_none" + )] pub container: Option, #[serde(flatten)] pub extra: Map, } +fn present_value<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + >::deserialize(deserializer).map(Some) +} + #[cfg(test)] mod tests { use super::*; @@ -73,4 +88,21 @@ mod tests { assert_eq!(body.get("usage").cloned(), usage); assert_eq!(body.get("container").cloned(), container); } + + #[test] + fn explicit_null_usage_and_container_survive_a_round_trip() { + let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m", + "content": [], "stop_reason": null, "stop_sequence": null, + "usage": null, "container": null}); + let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses"); + assert_eq!(serde_json::to_value(parsed).expect("serializable"), body); + } + + #[test] + fn absent_usage_and_container_stay_absent() { + let body = json!({"id": "msg_1", "type": "message", "role": "assistant", "model": "m", + "content": [], "stop_reason": null, "stop_sequence": null}); + let parsed: MessagesResponse = serde_json::from_value(body.clone()).expect("parses"); + assert_eq!(serde_json::to_value(parsed).expect("serializable"), body); + } } diff --git a/litellm-rust/crates/llms-types/src/recognized.rs b/litellm-rust/crates/llms-types/src/recognized.rs index 58e8dd7a2e7..0545389afbc 100644 --- a/litellm-rust/crates/llms-types/src/recognized.rs +++ b/litellm-rust/crates/llms-types/src/recognized.rs @@ -7,6 +7,16 @@ pub enum Recognized { Unrecognized(Value), } +/// `deserialize_with` for an optional `Recognized` field whose explicit `null` must stay +/// distinguishable from an absent key: `null` becomes `Some(Unrecognized(Null))`. +pub fn present<'de, D, T>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + Recognized: serde::Deserialize<'de>, +{ + as serde::Deserialize<'de>>::deserialize(deserializer).map(Some) +} + impl Recognized { pub fn known(&self) -> Option<&T> { match self { diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index aec5d9ebc27..13653f438f7 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -154,11 +154,42 @@ pub fn complete_anthropic_url( ) -> String { let api_base = resolve_anthropic_api_base(api_base, env_lookup); - let api_base = api_base.trim_end_matches('/'); - if api_base.ends_with(MESSAGES_PATH) { + ensure_api_route(&api_base, MESSAGES_PATH) +} + +/// Python's `urlsplit(url).path`, for the URLs an `api_base` holds. +fn url_path(url: &str) -> &str { + let before_fragment = url.split('#').next().unwrap_or_default(); + let before_query = before_fragment.split('?').next().unwrap_or_default(); + match before_query.split_once("://") { + Some((_, rest)) => rest.find('/').map_or("", |slash| &rest[slash..]), + None => before_query, + } +} + +/// Python's `_strip_trailing_slashes_keeping_the_scheme`. +fn strip_trailing_slashes(api_base: &str) -> String { + let stripped = api_base.trim_end_matches('/'); + if stripped.ends_with(':') && !stripped.contains('/') && !stripped.contains('?') { return api_base.to_string(); } - format!("{api_base}{MESSAGES_PATH}") + match stripped.split_once('?') { + Some((before, query)) if url_path(stripped).ends_with('/') => { + format!("{}?{query}", before.trim_end_matches('/')) + } + _ => stripped.to_string(), + } +} + +/// Python's `ensure_api_route`: an `api_base` whose path already ends in the route is kept as +/// it is, query string included; anything else gets the route appended to the whole string. +fn ensure_api_route(api_base: &str, route: &str) -> String { + let base = strip_trailing_slashes(api_base); + let route = route.trim_start_matches('/'); + if url_path(&base).ends_with(&format!("/{route}")) { + return base; + } + format!("{}/{route}", strip_trailing_slashes(&base)) } pub fn existing_betas(headers: &[(String, String)]) -> BetaSet { @@ -1922,4 +1953,24 @@ mod tests { expected ); } + + #[rstest] + #[case::bare_host("https://api.example.com", "https://api.example.com/v1/messages")] + #[case::trailing_slash("https://api.example.com/", "https://api.example.com/v1/messages")] + #[case::route_present( + "https://api.example.com/v1/messages/", + "https://api.example.com/v1/messages" + )] + #[case::route_and_query( + "https://api.example.com/v1/messages?beta=true", + "https://api.example.com/v1/messages?beta=true" + )] + #[case::query_without_route( + "https://api.example.com/?x=1", + "https://api.example.com?x=1/v1/messages" + )] + #[case::scheme_only("https://", "https:///v1/messages")] + fn the_messages_url_follows_python_ensure_api_route(#[case] api_base: &str, #[case] url: &str) { + assert_eq!(complete_anthropic_url(Some(api_base), &|_| None), url); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index df1ae13fd28..ae3488a21f9 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -2,8 +2,8 @@ use litellm_llms_types::{ formats::{ chat_completions::ReasoningEffort, messages::{ - EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, ThinkingConfig, - ThinkingDisplay, + AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig, + ThinkingConfig, ThinkingDisplay, }, }, recognized::Recognized, @@ -36,16 +36,17 @@ fn budget_for_effort(budgets: &ThinkingBudgets, effort: ReasoningEffort) -> Opti fn effort_for_budget( budgets: &ThinkingBudgets, - budget_tokens: u64, + budget_tokens: i64, capabilities: &MessagesModelCapabilities, ) -> EffortLevel { - if budget_tokens >= budgets.xhigh && capabilities.effort_tiers.xhigh { + let reaches = |tier: u64| i128::from(budget_tokens) >= i128::from(tier); + if reaches(budgets.xhigh) && capabilities.effort_tiers.xhigh { return EffortLevel::Xhigh; } - if budget_tokens >= budgets.high { + if reaches(budgets.high) { return EffortLevel::High; } - if budget_tokens >= budgets.medium { + if reaches(budgets.medium) { return EffortLevel::Medium; } EffortLevel::Low @@ -231,32 +232,129 @@ fn drop_disabled_thinking(request: MessagesRequest, context: &ThinkingContext) - } } +/// `_numeric_budget_tokens`: Python reads any int, float (truncated) or int-parseable string +/// as a budget, and nothing else. +fn numeric_budget_tokens(budget: &Recognized) -> Option { + let value = match budget { + Recognized::Known(budget) => return Some(i64::try_from(*budget).unwrap_or(i64::MAX)), + Recognized::Unrecognized(value) => value, + }; + match value { + Value::Number(number) => number + .as_i64() + .or_else(|| number.as_f64().map(|float| float.trunc() as i64)), + Value::String(text) => python_int(text), + Value::Null | Value::Bool(_) | Value::Array(_) | Value::Object(_) => None, + } +} + +/// Python's `int(str)`: surrounding whitespace, a sign, and single underscores between digits. +fn python_int(text: &str) -> Option { + let trimmed = text.trim(); + let digits = trimmed.trim_start_matches(['+', '-']); + let well_formed = trimmed.len() - digits.len() <= 1 + && !digits.is_empty() + && !digits.starts_with('_') + && !digits.ends_with('_') + && !digits.contains("__") + && digits.chars().all(|ch| ch.is_ascii_digit() || ch == '_'); + if !well_formed { + return None; + } + let cleaned: String = trimmed.chars().filter(|ch| *ch != '_').collect(); + cleaned.parse::().ok().or_else(|| { + Some(if cleaned.starts_with('-') { + i64::MIN + } else { + i64::MAX + }) + }) +} + +fn python_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "NoneType", + Value::Bool(_) => "bool", + Value::Number(number) if number.is_f64() => "float", + Value::Number(_) => "int", + Value::String(_) => "str", + Value::Array(_) => "list", + Value::Object(_) => "dict", + } +} + +fn kept_display( + display: Option<&Recognized>, + context: &ThinkingContext, +) -> Option> { + match display? { + Recognized::Known(ThinkingDisplay::Summarized | ThinkingDisplay::Omitted) => { + display.cloned() + } + Recognized::Known(ThinkingDisplay::Updates) if context.keeps_display_updates => { + display.cloned() + } + Recognized::Known(ThinkingDisplay::Updates) | Recognized::Unrecognized(_) => None, + } +} + +/// Keys Python's `carried_thinking_keys` keeps on the rebuilt adaptive config: the adaptive +/// keys always, and any other key unless `drop_params` is set. +fn carried_thinking_keys( + extra: &serde_json::Map, + context: &ThinkingContext, +) -> serde_json::Map { + extra + .iter() + .filter(|(key, _)| key.as_str() == "block_binding" || !context.drop_params) + .map(|(key, value)| (key.clone(), value.clone())) + .collect() +} + +/// `AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently`. fn translate_legacy_thinking_for_adaptive_model( request: MessagesRequest, context: &ThinkingContext, -) -> MessagesRequest { +) -> Result { let capabilities = &context.capabilities; if !capabilities.supports_adaptive_thinking || capabilities.supports_legacy_thinking { - return request; + return Ok(request); } let Some(ThinkingConfig::Enabled(enabled)) = known_thinking(&request) else { - return request; + return Ok(request); }; - let budget = enabled - .budget_tokens - .as_ref() - .and_then(Recognized::known) - .copied() - .unwrap_or(0); - let level = effort_for_budget(&context.budgets, budget, capabilities); - MessagesRequest { + let budget = match &enabled.budget_tokens { + None => Some(0), + Some(budget) => numeric_budget_tokens(budget), + }; + if budget.is_none() && !context.drop_params { + let actual = match &enabled.budget_tokens { + Some(Recognized::Unrecognized(value)) => python_type_name(value), + Some(Recognized::Known(_)) | None => "int", + }; + return Err(Error::InvalidRequest( + crate::ErrorDetail::NonNumericThinkingBudget { actual }, + )); + } + let thinking = ThinkingConfig::Adaptive(AdaptiveThinking { + display: kept_display(enabled.display.as_ref(), context), + extra: carried_thinking_keys(&enabled.extra, context), + }); + let output_config = match budget { + Some(budget) => with_default_effort( + request.params.output_config, + effort_for_budget(&context.budgets, budget, capabilities), + ), + None => request.params.output_config, + }; + Ok(MessagesRequest { params: MessagesOptionalParams { - thinking: Some(Recognized::Known(ThinkingConfig::adaptive(None))), - output_config: with_default_effort(request.params.output_config, level), + thinking: Some(Recognized::Known(thinking)), + output_config, ..request.params }, ..request - } + }) } fn translate_adaptive_effort_for_non_adaptive_model( @@ -337,7 +435,7 @@ pub fn translate_thinking( ) -> Result { let request = translate_reasoning_effort(request, context)?; let request = drop_disabled_thinking(request, context); - let request = translate_legacy_thinking_for_adaptive_model(request, context); + let request = translate_legacy_thinking_for_adaptive_model(request, context)?; let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?; Ok(drop_incompatible_temperature_for_thinking(request, context)) } @@ -363,6 +461,7 @@ mod tests { ThinkingContext { capabilities, budgets: ThinkingBudgets::default(), + ..ThinkingContext::default() } } @@ -1239,10 +1338,102 @@ mod tests { let context = ThinkingContext { capabilities, budgets: overridden_budgets(overrides), + ..ThinkingContext::default() }; assert_eq!( translate_thinking(request(input), &context), Ok(request(expected)) ); } + + use serde_json::json; + + fn legacy_context(drop_params: bool, keeps_display_updates: bool) -> ThinkingContext { + ThinkingContext { + drop_params, + keeps_display_updates, + ..context(opus_4_7()) + } + } + + #[rstest] + #[case::float_budget(json!(8192.0), "xhigh")] + #[case::float_below_a_tier(json!(1023.9), "low")] + #[case::string_budget(json!(" 16_000 "), "xhigh")] + #[case::negative_budget(json!(-5), "low")] + fn legacy_budgets_read_like_python_numbers(#[case] budget: Value, #[case] effort: &str) { + let translated = translate_thinking( + request( + json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": budget}}), + ), + &legacy_context(false, false), + ); + assert_eq!( + translated, + Ok(request(json!({ + "max_tokens": 64000, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": effort}, + }))) + ); + } + + #[rstest] + #[case::text(json!("lots"), "str")] + #[case::list(json!([1]), "list")] + #[case::null(Value::Null, "NoneType")] + #[case::boolean(json!(true), "bool")] + fn a_non_numeric_legacy_budget_is_rejected_unless_dropped( + #[case] budget: Value, + #[case] actual: &'static str, + ) { + let fields = + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": budget}}); + assert_eq!( + translate_thinking(request(fields.clone()), &legacy_context(false, false)), + Err(Error::InvalidRequest( + crate::ErrorDetail::NonNumericThinkingBudget { actual } + )) + ); + assert_eq!( + translate_thinking(request(fields), &legacy_context(true, false)), + Ok(request( + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}) + )) + ); + } + + #[rstest] + #[case::summarized("summarized", false, Some("summarized"))] + #[case::updates_kept("updates", true, Some("updates"))] + #[case::updates_dropped("updates", false, None)] + #[case::unknown("verbose", true, None)] + fn legacy_display_survives_only_where_python_keeps_it( + #[case] display: &str, + #[case] keeps_display_updates: bool, + #[case] kept: Option<&str>, + ) { + let translated = translate_thinking( + request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "display": display}})), + &legacy_context(false, keeps_display_updates), + ) + .unwrap(); + let thinking = serde_json::to_value(translated.params.thinking).unwrap(); + assert_eq!(thinking.get("display").and_then(Value::as_str), kept); + } + + #[rstest] + #[case::kept(false, json!({"type": "adaptive", "block_binding": "strict", "note": "x"}))] + #[case::dropped(true, json!({"type": "adaptive", "block_binding": "strict"}))] + fn legacy_extra_keys_follow_drop_params(#[case] drop_params: bool, #[case] expected: Value) { + let translated = translate_thinking( + request(json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "x"}})), + &legacy_context(drop_params, false), + ) + .unwrap(); + assert_eq!( + serde_json::to_value(translated.params.thinking).unwrap(), + expected + ); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index fb6d7259bf9..18789e5b2be 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -56,7 +56,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig { request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { - transform_messages_request(request, context) + transform_messages_request(request, &context.keeping_display_updates()) } fn secret_names(&self) -> &'static [&'static str] { @@ -206,6 +206,8 @@ fn feature_betas(request: &MessagesRequest) -> BetaSet { .then_some(AnthropicBeta::PerTurnControl20260701), uses_thinking_display_updates(params) .then_some(AnthropicBeta::ThinkingDisplayUpdates20260818), + uses_thinking_block_binding(params) + .then_some(AnthropicBeta::ThinkingBindingControls20260801), messages_carry_tool_changes(&request.messages) .then_some(AnthropicBeta::MidConversationToolChanges20260701), has_advisor_tool(tools).then_some(AnthropicBeta::AdvisorTool20260301), @@ -267,6 +269,25 @@ fn uses_thinking_display_updates(params: &MessagesOptionalParams) -> bool { display.and_then(Recognized::known) == Some(&ThinkingDisplay::Updates) } +/// `AnthropicModelInfo.thinking_betas`: any thinking object carrying `block_binding`. +fn uses_thinking_block_binding(params: &MessagesOptionalParams) -> bool { + match params.thinking.as_ref() { + Some(Recognized::Known(ThinkingConfig::Enabled(thinking))) => { + thinking.extra.contains_key("block_binding") + } + Some(Recognized::Known(ThinkingConfig::Adaptive(thinking))) => { + thinking.extra.contains_key("block_binding") + } + Some(Recognized::Known(ThinkingConfig::Disabled(thinking))) => { + thinking.extra.contains_key("block_binding") + } + Some(Recognized::Unrecognized(Value::Object(thinking))) => { + thinking.contains_key("block_binding") + } + Some(Recognized::Unrecognized(_)) | None => false, + } +} + fn is_tool_reference_change(block: &ContentBlock) -> bool { matches!( &block.block_type, @@ -297,7 +318,8 @@ fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error }) } -fn drop_unsupported_params( +/// The speed and sampling gates of Python's `get_requested_anthropic_messages_optional_param`. +pub fn drop_unsupported_params( request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { @@ -838,6 +860,8 @@ mod tests { low: 2000, ..ThinkingBudgets::default() }, + drop_params: true, + keeps_display_updates: false, }, drop_params: true, } diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index a2ba2cee0d3..d13cfa81a94 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -68,7 +68,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig { .capabilities .supports_mid_conversation_system, )), - context, + &context.keeping_display_updates(), ) } diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index e59a63813b0..b514e975aa4 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -110,6 +110,11 @@ impl ThinkingBudgets { pub struct ThinkingContext { pub capabilities: MessagesModelCapabilities, pub budgets: ThinkingBudgets, + /// `drop_params` as Python's thinking translation reads it. + pub drop_params: bool, + /// The provider accepts `display: "updates"` on thinking translated from the legacy shape + /// (Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`). + pub keeps_display_updates: bool, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct MessagesTransformContext { @@ -130,8 +135,22 @@ impl MessagesTransformContext { thinking: ThinkingContext { capabilities, budgets: ThinkingBudgets::from_lookup(env), + drop_params, + keeps_display_updates: false, }, drop_params, } } + + /// The same context for a provider in Python's `LEGACY_THINKING_DISPLAY_UPDATES_PROVIDERS`. + #[must_use] + pub fn keeping_display_updates(&self) -> Self { + Self { + thinking: ThinkingContext { + keeps_display_updates: true, + ..self.thinking + }, + ..*self + } + } } diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index b3d1421083a..2844fbf3e01 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -92,6 +92,24 @@ impl Error { } } + pub fn upstream_headers(&self) -> &[(String, String)] { + match self { + Self::Http(error) => error.upstream_headers(), + _ => &[], + } + } + + pub fn is_body_read_timeout(&self) -> bool { + matches!(self, Self::Http(error) if error.is_body_read_timeout()) + } + + pub fn is_timeout(&self) -> Option { + match self { + Self::Http(error) if error.is_timeout() => Some(error.is_connect_timeout()), + _ => None, + } + } + pub fn upstream_response(&self) -> Option<(u16, &str)> { match self { Self::Http(error) => error.upstream_response(), @@ -158,6 +176,10 @@ pub enum ErrorDetail { value: String, hint: String, }, + #[error( + "thinking.budget_tokens must be a number, got {actual}. To drop it, set `litellm.drop_params=True`, or `drop_params: true` under `litellm_settings` on the proxy" + )] + NonNumericThinkingBudget { actual: &'static str }, #[error( "Missing API Base - A call is being made to LLM Provider but no api base is set either in the environment variables ({{LLM_PROVIDER}}_API_KEY) or via params" )] diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index e5e3a056fab..038617c2e10 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -77,7 +77,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig { request: MessagesRequest, context: &MessagesTransformContext, ) -> Result { - let request = transform_messages_request(request, context)?; + let request = transform_messages_request(request, &context.keeping_display_updates())?; let request = strip_cache_control_scope(normalize_system_role_messages( request, context diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index b42c08b7e78..426cd861729 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -23,6 +23,17 @@ pyo3::create_exception!( ); const MESSAGES_REQUEST_ERROR: &str = "messages_request_error"; +/// The rejection is a parameter the model does not support, which Python raises as +/// `UnsupportedParamsError`. +const MESSAGES_UNSUPPORTED_PARAM: &str = "messages_unsupported_param"; +/// The phase label (`Connection` or `Read`) of a timed-out call, for Python's timeout text, or +/// `Body` for a streamed body read that timed out after the head arrived. +const MESSAGES_TIMEOUT: &str = "messages_timeout"; +/// No credential reached the provider, which Python raises as `AuthenticationError`. +const MESSAGES_MISSING_CREDENTIAL: &str = "messages_missing_credential"; +/// Python refuses the request only once its logger knows the resolved model and provider (the +/// provider config's request transform), so the failure logs them. +const MESSAGES_LATE_REJECTION: &str = "messages_late_rejection"; pub(crate) fn upstream_error( py: Python<'_>, @@ -78,17 +89,57 @@ pub(crate) fn route_failure( return Ok(raised); } if let Some((status, body)) = error.upstream_response() { - return public(upstream_error(py, status, body, &HeaderMap::new())?); + let headers = error + .upstream_headers() + .iter() + .filter_map(|(name, value)| { + Some(( + reqwest::header::HeaderName::from_bytes(name.as_bytes()).ok()?, + reqwest::header::HeaderValue::from_str(value).ok()?, + )) + }) + .collect::(); + return public(upstream_error(py, status, body, &headers)?); } let native = match (marker, error.rejected_request()) { (RequestMarker::Messages, Some(rejection)) => { let native = PyValueError::new_err(Report::new(rejection).to_string()); - native.value(py).setattr(MESSAGES_REQUEST_ERROR, true)?; + let value = native.value(py); + value.setattr(MESSAGES_REQUEST_ERROR, true)?; + if matches!( + rejection.downcast_ref::(), + Some( + litellm_llms::ErrorDetail::UnsupportedParameter { .. } + | litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. } + ) + ) { + value.setattr(MESSAGES_UNSUPPORTED_PARAM, true)?; + } + if matches!( + rejection.downcast_ref::(), + Some(litellm_llms::ErrorDetail::NonNumericThinkingBudget { .. }) + ) { + value.setattr(MESSAGES_LATE_REJECTION, true)?; + } native } - (RequestMarker::Messages, None) | (RequestMarker::Unmarked, _) => { - by_fault(error.is_request(), &error) + (RequestMarker::Messages, None) => { + let native = by_fault(error.is_request(), &error); + let value = native.value(py); + if let Some(connect) = error.timeout() { + let phase = match (connect, error.body_read_timeout()) { + (true, _) => "Connection", + (false, true) => "Body", + (false, false) => "Read", + }; + value.setattr(MESSAGES_TIMEOUT, phase)?; + } + if error.is_missing_credential() { + value.setattr(MESSAGES_MISSING_CREDENTIAL, true)?; + } + native } + (RequestMarker::Unmarked, _) => by_fault(error.is_request(), &error), }; public(native) } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 27ce8c943f0..af614f17360 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -34,11 +34,9 @@ fn merge_headers( forwarded: Option>, extra_headers: Option>, ) -> Option> { - let merged: Map = forwarded - .into_iter() - .flatten() - .chain(extra_headers.into_iter().flatten()) - .collect(); + let merged = litellm_inference_messages::merge_messages_header_sources( + forwarded.into_iter().chain(extra_headers), + ); (!merged.is_empty()).then_some(merged) } @@ -107,10 +105,19 @@ impl MessagesPythonHost { .map(|value| python_timeout_seconds(py, value.unbind())) .transpose()? .flatten(); - let custom_llm_provider = string("custom_llm_provider")?; - let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; let api_key = string("api_key")?; let api_base = string("api_base")?; + // Python's handler resolves a bare model name (`claude-sonnet-4-5`) through + // `get_llm_provider`'s model lists, which the native prefix lookup does not carry. + let custom_llm_provider = match string("custom_llm_provider")? { + Some(provider) => Some(provider), + None => py + .import(ROUTE_HOST_MODULE)? + .getattr("inferred_provider")? + .call1((&model, api_base.as_deref()))? + .extract()?, + }; + let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; let extra_headers = self.merged_headers(py, arguments)?; let provider_specific_header = self.provider_specific_header(py, arguments)?; let litellm_params = project_litellm_params(argument, |name| module_global(py, name))?; diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index c788760b268..a9ae01bd7ee 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -50,9 +50,18 @@ fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyR ) } +/// Python's sync Messages handler raises for every native provider before it sends anything +/// or opens a logging object (`BaseLLMHTTPHandler.anthropic_messages_handler`), and Rust copies +/// that until both sides serve sync calls. +const SYNC_NOT_IMPLEMENTED: &str = "anthropic_messages_handler is not implemented for sync calls"; + #[pyfunction] -pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { - run_messages(py, call, false) +#[pyo3(signature = (call))] +pub(crate) fn messages(call: NativeCall<'_>) -> PyResult> { + drop(call); + Err(pyo3::exceptions::PyValueError::new_err( + SYNC_NOT_IMPLEMENTED, + )) } #[pyfunction] diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index e155680bd30..92590531cdc 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -12,7 +12,7 @@ import datetime import traceback import uuid import weakref -from collections.abc import Awaitable, Coroutine, Mapping +from collections.abc import Awaitable, Coroutine, Mapping, Sequence from dataclasses import dataclass from typing import ( TYPE_CHECKING, @@ -22,6 +22,7 @@ from typing import ( cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations ) +import httpx from pydantic import TypeAdapter if TYPE_CHECKING: @@ -243,6 +244,27 @@ def post_call( logger.post_call(original_response=original_response, api_key=api_key, additional_args=additional_args) +def record_response(logger: Logging, status: int, headers: Sequence[tuple[str, str]], body: str) -> None: + """Stores the provider response the way Python's Messages handler does, for logging to replay.""" + response: Final = httpx.Response(status_code=status, headers=list(headers), content=body.encode()) + details: Final = logger.model_call_details # the call's own logger: recording on it is the point + details["httpx_response"] = response + details["response_headers"] = dict(response.headers) + + +def record_api_window(logger: Logging, start: float, end: float) -> None: + """Records the provider exchange's window the way Python's timed `AsyncHTTPHandler.post` does.""" + from litellm.litellm_core_utils.logging_utils import ( + _set_duration_in_model_call_details, # pyright: ignore[reportPrivateUsage] # the timed post's own recorder + ) + + _set_duration_in_model_call_details( + logging_obj=logger, + start_time=datetime.datetime.fromtimestamp(start), # noqa: DTZ006 # the timed post records naive local times + end_time=datetime.datetime.fromtimestamp(end), # noqa: DTZ006 # the timed post records naive local times + ) + + def defers_async_logging(logger: LoggingSurface) -> bool: return bool(getattr(logger, "defer_async_logging", False)) @@ -359,6 +381,27 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t _STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str]) +class _HiddenParamsCarrier: + """Stands in for the stream `@client` stamps, so the head becomes its `_hidden_params`.""" + + def __init__(self, hidden_params: dict[str, object]) -> None: # mutable-ok: the stamps land in this dict + self._hidden_params: object = hidden_params + + +def stamp_stream_head( + head: dict[str, object], # mutable-ok: the stream's hidden params, stamped in place + logger: Logging, + kwargs: dict[str, object], # mutable-ok: `finalize` takes the call kwargs as a dict + start_time: datetime.datetime, + end_time: datetime.datetime, +) -> None: + carrier: Final = _HiddenParamsCarrier(head) + finalize(carrier, logger, kwargs, start_time, end_time) + stamped: Final = carrier._hidden_params # pyright: ignore[reportPrivateUsage] # the carrier's own field + if stamped is not head and isinstance(stamped, dict): + head.update(cast("dict[str, object]", stamped)) # cast-ok: update_response_metadata writes a str-keyed dict + + def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: logger.stream = True logger.model_call_details["stream"] = True diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index 017ff65e2a7..c8b53097f46 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -1,21 +1,35 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +import datetime +from collections.abc import Callable, Mapping, Sequence from dataclasses import asdict, dataclass -from typing import Final, cast # noqa: TID251 # narrows the normalized native payload to the public TypedDict +from typing import Final, TypeAlias, cast # noqa: TID251 # narrows native payloads to public TypedDicts import httpx from pydantic import TypeAdapter, ValidationError import litellm +from litellm.litellm_core_utils import exception_mapping_utils from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout +from litellm.llms.anthropic.common_utils import AnthropicError +from litellm.llms.anthropic.messages.native import handler +from litellm.llms.anthropic.messages.native.handler import ( + _native_messages_config as native_messages_config, # pyright: ignore[reportPrivateUsage] # the handler's own config lookup +) +from litellm.llms.anthropic.messages.native.handler import ( + _with_deployment_output_effort as _deployment_output_effort, # pyright: ignore[reportPrivateUsage] # the handler's own effort rounding +) from litellm.llms.anthropic.messages.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures +from litellm.rust_bridge.host import callbacks_legacy_python from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse _DROP_PATHS: Final = TypeAdapter(list[object]) +_METADATA: Final = TypeAdapter(dict[str, object]) _BODY_FIELDS: Final = frozenset( { "max_tokens", @@ -40,6 +54,7 @@ _BODY_FIELDS: Final = frozenset( "cache_control", "reasoning_effort", "safeguards", + "service_tier", } ) @@ -59,8 +74,40 @@ def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: ) +def inferred_provider(model: str, api_base: str | None) -> str | None: + """The provider Python's handler resolves for a call that names none, or None when the model + carries no provider `get_llm_provider` knows.""" + try: + _, provider, _, _ = litellm.get_llm_provider(model=model, api_base=api_base) + except litellm.BadRequestError: + return None + return provider + + def body_params(request: Mapping[str, object]) -> Mapping[str, object]: - return known_params(_BODY_FIELDS, request) + params: Final = known_params(_BODY_FIELDS, request) + if deployment_reasoning_effort_levels(request.get("model_info")) is None: + return params + return _with_deployment_effort(params, request) + + +def _with_deployment_effort(params: Mapping[str, object], request: Mapping[str, object]) -> Mapping[str, object]: + """Python's handler rounds the requested effort to the levels the deployment declares in + `model_info.reasoning_effort_levels`, the one place a deployment, not the model, decides.""" + explicit_provider: Final = optional_str(request.get("custom_llm_provider")) + try: + model, provider, _, _ = litellm.get_llm_provider( + model=str(request["model"]), custom_llm_provider=explicit_provider + ) + except litellm.BadRequestError: + return params + return _deployment_output_effort( + params, + model=model, + custom_llm_provider=provider, + kwargs=request, + drop_params=litellm.drop_params is True or request.get("drop_params") is True, + ) def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, object]: @@ -71,18 +118,147 @@ def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, obj return anthropic_messages_stream_hidden_params(httpx.Headers(list(headers))) -def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: - if getattr(error, "messages_request_error", False): - return litellm.BadRequestError( - message=str(error), - model=str(request["model"]).removeprefix(f"{request_provider}/"), - llm_provider=request_provider, - ) - return failures.map_native_failure( - error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) +_UPSTREAM_ARGS: Final = TypeAdapter(tuple[int, str]) +_UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) +_ERROR_CLASS: TypeAlias = Callable[..., Exception] +_validate_metadata: Final = cast( # cast-ok: the handler's untyped metadata validator + Callable[[dict[str, object]], object], handler.validate_anthropic_api_metadata +) +_exception_type: Final = cast( # cast-ok: the legacy public exception mapper is untyped + Callable[..., Exception], exception_mapping_utils.exception_type +) +_BODY_READ_TIMEOUT: Final = "Timeout on reading data from socket" # aiohttp's SocketTimeoutError text +_MISSING_KEY: Final = ( + "Missing Anthropic API Key - A call is being made to anthropic but no key is set " + "either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` " + "or `ANTHROPIC_AUTH_TOKEN` in your environment vars" +) + + +def _rejection(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: + """The error Python's handler raises for a request it refuses before sending it.""" + metadata: Final = request.get("metadata") + if isinstance(metadata, dict): + try: + _validate_metadata(_METADATA.validate_python(metadata)) + except ValidationError as invalid: + return invalid + if getattr(error, "messages_unsupported_param", False): + return litellm.UnsupportedParamsError(message=str(error)) + return litellm.BadRequestError( + message=str(error), + model=str(request["model"]).removeprefix(f"{request_provider}/"), + llm_provider=request_provider, ) +def _timeout_text(phase: str, request: Mapping[str, object]) -> str: + """`litellm.Timeout`'s text as Python's HTTP client raises it for the same timeout.""" + seconds: Final = request.get("timeout") + logging_obj: Final = request.get("litellm_logging_obj") + start_time: Final = getattr(logging_obj, "start_time", None) + time_taken: Final = ( + round( + (datetime.datetime.now() - start_time).total_seconds(), # noqa: DTZ005 # the logger's start_time is naive local time + 3, + ) + if isinstance(start_time, datetime.datetime) + else None + ) + return str( + litellm.Timeout( + message=f"{phase} timed out. Timeout passed={seconds}, time taken={time_taken} seconds", + model="default-model-name", + llm_provider="litellm-httpx-handler", + ) + ) + + +def _upstream_failure( + error: Exception, request: Mapping[str, object] +) -> tuple[int, str, Sequence[tuple[str, str]]] | None: + """The status, text and headers Python's HTTP client reports for the same failure.""" + timeout_phase: Final = getattr(error, "messages_timeout", None) + if isinstance(timeout_phase, str): + return 408, _timeout_text(timeout_phase, request), () + try: + status, text = _UPSTREAM_ARGS.validate_python(error.args) + return status, text, _UPSTREAM_HEADERS.validate_python(getattr(error, "headers", None)) + except ValidationError: + return None + + +def _provider_error(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception | None: + """The provider exception Python's `_handle_error` builds for the same failure.""" + failure: Final = _upstream_failure(error, request) + if failure is None: + return None + status, text, headers = failure + api_base: Final = request.get("api_base") + config: Final = native_messages_config( + str(request["model"]).removeprefix(f"{request_provider}/"), + request_provider, + request.get("model_info"), + api_base if isinstance(api_base, str) else None, + ) + return ( + cast(_ERROR_CLASS, config.get_error_class)( # cast-ok: the config's untyped headers parameter + error_message=text, status_code=status, headers=dict(headers) + ) + if config is not None + else AnthropicError(status_code=status, message=text, headers=httpx.Headers(headers)) + ) + + +def _log_resolved_call(request: Mapping[str, object], request_provider: str) -> None: + """Tells the logger the resolved call, as Python's handler does before its provider config + transforms the request.""" + logger: Final = request.get("litellm_logging_obj") + if not isinstance(logger, LiteLLMLoggingObj): + return + callbacks_legacy_python.update_logging( + logger, + kwargs={name: value for name, value in request.items() if name != "metadata"}, + model=str(request["model"]).removeprefix(f"{request_provider}/"), + optional_params=dict(body_params(request)), + litellm_params={"model_info": request.get("model_info"), **body_params(request)}, + custom_llm_provider=request_provider, + ) + + +def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: + """Raises what Python's Messages handler raises for the same failure.""" + if getattr(error, "messages_timeout", None) == "Body": + # Python reads a stream's body outside its HTTP client's error mapping, so the transport's + # own timeout reaches the caller unmapped. + return httpx.ReadTimeout(_BODY_READ_TIMEOUT) + if getattr(error, "messages_late_rejection", False): + _log_resolved_call(request, request_provider) + if getattr(error, "messages_request_error", False): + return _rejection(error, request, request_provider) + if getattr(error, "messages_missing_credential", False): + return litellm.AuthenticationError( + message=_MISSING_KEY, + llm_provider=request_provider, + model=str(request["model"]).removeprefix(f"{request_provider}/"), + ) + provider_error: Final = _provider_error(error, request, request_provider) + if provider_error is None: + return failures.map_native_failure( + error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) + ) + try: + return _exception_type( + model=str(request["model"]), + custom_llm_provider=request_provider, + original_exception=provider_error, + extra_kwargs=dict(request), + ) + except Exception as mapped: # noqa: BLE001 # exception_type raises the mapped public error + mapped.__context__ = error + return mapped + + def _drop_params(kwargs: Mapping[str, object]) -> bool: return bool(litellm.drop_params) or normalize_drop_params(kwargs.get("drop_params")) is True diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py index cdfba773cf6..41008f0d57f 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -20,6 +20,7 @@ R request, S response, C callbacks, F covered by a fallback detector today) """ import asyncio +import copy import json import os import re @@ -335,38 +336,12 @@ def _kw(**kwargs: object) -> Mapping[str, object]: return MappingProxyType(kwargs) -A2_BARE_MODEL: Final = "A2: native resolves the provider from the model prefix only, so a bare Claude model name fails" -A10_NO_MAX_TOKENS: Final = "A10: native admission declines a call without an int max_tokens and Python serves it" -R3_LEGACY_THINKING: Final = "R3: native maps a float legacy thinking budget to effort low instead of Python's tier" -R4_BINDING_BETA: Final = "R4: native omits the thinking-binding-controls beta for block_binding" -R5_DEPLOYMENT_EFFORT: Final = "R5: native ignores the deployment's reasoning_effort_levels from model_info" -R7_DROP_ORDER: Final = "R7: native applies additional_drop_params before Python's unsupported-param check" -R8_VERSION_HEADER: Final = "R8: Python sends anthropic-version twice when the caller capitalises it; native once" -R9_QUERY_PATH: Final = "R9: native appends /v1/messages again to an api_base that ends in /v1/messages?query" -R11_PARAM_ERROR: Final = "R11: native raises BadRequestError where Python raises UnsupportedParamsError" -R12_METADATA_ERROR: Final = ( - "R12: Python raises a pydantic ValidationError for a non-string metadata.user_id; native a BadRequestError" -) -S1_S2_ERROR_HEADERS: Final = ( - "S1/S2: native drops upstream error headers and wraps the error body in a (status, body) tuple" -) -S2_ERROR_MESSAGE: Final = ( - "S2: native wraps the upstream error body in a (status, body) tuple and strips the provider from model" -) -S2_S12_ERROR_STATUS: Final = "S2/S12: native keeps the upstream status where Python maps it (413 to 400, 529 to 500)" -S3_TIMEOUT: Final = "S3: native raises APIConnectionError (500) for a timeout where Python raises Timeout (408)" -S5_STREAM_DEADLINE: Final = "S5: native applies the timeout to the whole stream instead of to each read" +A10_NO_MAX_TOKENS: Final = "A10: native admission hands a call without an int max_tokens to Python, whose signature raises the TypeError; only dispatch differs" S6_PRIVATE_KEYS: Final = "S6: native writes _hidden_params into the response dict" -S8_TYPED_RESPONSE: Final = "S8: native re-serializes the response and drops null usage and container" -S9_SYNC: Final = "S9: Python's sync handler raises ValueError for native providers; native serves the call" -S13_INVALID_JSON: Final = "S13: native reports an unparseable success body as an internal error, not AnthropicException" -S14_MISSING_KEY: Final = ( - "S14: native raises APIConnectionError for a missing API key where Python raises AuthenticationError" +S15_COMPLETE_DEADLINE: Final = ( + "S15: by design, native bounds a complete response by one overall deadline where Python bounds each read, " + "so a body trickling in slower than the timeout succeeds in Python and times out natively" ) -C4_METADATA: Final = "C4: native callbacks get the raw Anthropic metadata, not Python's merged litellm metadata" -C6_RESPONSE_HEADERS: Final = "C6: native success logs carry no response_headers" -C7_API_KEY: Final = "C7: native callbacks see the caller's api_key where Python logs an empty string" -C8_STREAM_HIDDEN: Final = "C8: native streams lack the call metadata @client stamps into _hidden_params" F_MOCK: Final = "F: mock_response is Python-only (detector mock_response)" F_THINKING_RETRY: Final = ( "F: retry without thinking after a rejected signature is Python-only (detector thinking_signature_retry)" @@ -377,176 +352,10 @@ F_STREAM_COST: Final = "F: cost injection into streamed usage is Python-only (de KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType( { - "plain_text": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "system_string": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "system_blocks_with_cache_control": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "multi_turn_tool_result": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "tools_with_tool_choice": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "tool_choice_none": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "tool_input_examples": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "sampling_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "stop_sequences": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "metadata_user_id": _gaps( - logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "metadata_extra_keys_are_filtered": _gaps( - logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "metadata_user_id_not_a_string": _gaps(outcome=R12_METADATA_ERROR, logged=R12_METADATA_ERROR), - "service_tier": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "thinking_enabled": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "thinking_disabled": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "legacy_thinking_on_adaptive_model_keeps_display": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "legacy_thinking_float_budget_on_adaptive_model": _gaps( - request=R3_LEGACY_THINKING, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "adaptive_thinking_with_block_binding": _gaps( - request=R4_BINDING_BETA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "reasoning_effort": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "reasoning_effort_on_adaptive_model": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "output_config_effort": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "output_format_json_schema": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "speed_on_model_without_speed": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR), - "speed_on_model_without_speed_drop_params": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "speed_on_fast_model": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "temperature_on_adaptive_model": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR), - "top_k_on_adaptive_model": _gaps(outcome=R11_PARAM_ERROR, logged=R11_PARAM_ERROR), - "top_k_on_adaptive_model_drop_params": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "additional_drop_params_hides_rejected_temperature": _gaps( - request=R7_DROP_ORDER, - outcome=R7_DROP_ORDER, - hidden_params=R7_DROP_ORDER, - logged=R7_DROP_ORDER, - logged_metadata=R7_DROP_ORDER, - logged_api_key=R7_DROP_ORDER, - logged_call=R7_DROP_ORDER, - ), - "additional_drop_params_nested_path": _gaps( - logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "context_management_edits": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "container_skills": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "remote_mcp_servers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "inference_geo": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "top_level_cache_control": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "null_safeguards_are_stripped": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "empty_text_blocks_are_stripped": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "tool_use_ids_are_sanitized": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "provider_specific_fields_are_stripped": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "image_block": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "document_block": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "server_web_search_tool": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "extra_headers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "anthropic_beta_from_headers_and_extra_headers": _gaps( - logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "caller_anthropic_version_capitalised": _gaps( - request=R8_VERSION_HEADER, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "provider_specific_header": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "explicit_custom_llm_provider": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "bare_model_name": _gaps(**dict.fromkeys(ASPECTS[1:], A2_BARE_MODEL) | {"logged_model_info": None}), - "api_base_with_messages_path": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "api_base_with_messages_path_and_query": _gaps( - request=R9_QUERY_PATH, - logged=R9_QUERY_PATH, - logged_response_headers=C6_RESPONSE_HEADERS, - logged_api_key=C7_API_KEY, - ), - "api_key_from_environment": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "auth_token_from_environment": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "reasoning_auto_summary": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "global_drop_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "request_timeout_setting": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "litellm_metadata_and_metadata": _gaps( - logged_metadata=C4_METADATA, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "model_info_with_id": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "deployment_effort_levels": _gaps( - request=R5_DEPLOYMENT_EFFORT, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "response_with_tool_use": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_with_thinking": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_with_cache_usage": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_with_null_usage_and_container": _gaps(outcome=S8_TYPED_RESPONSE), - "response_without_stop_sequence": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_with_unknown_field": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_key_order": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_upstream_headers": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "response_private_keys": _gaps( - outcome=S6_PRIVATE_KEYS, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "upstream_400": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_401": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_403": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_404": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_413": _gaps(outcome=S2_S12_ERROR_STATUS, logged_api_key=C7_API_KEY), - "upstream_429_with_retry_after": _gaps(outcome=S1_S2_ERROR_HEADERS, logged_api_key=C7_API_KEY), - "upstream_500": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_529_overloaded": _gaps(outcome=S2_S12_ERROR_STATUS, logged_api_key=C7_API_KEY), - "upstream_400_context_window": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_non_json_error": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "upstream_invalid_json_success": _gaps( - outcome=S13_INVALID_JSON, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "upstream_timeout": _gaps(outcome=S3_TIMEOUT, logged=S3_TIMEOUT, logged_api_key=C7_API_KEY), + "response_private_keys": _gaps(outcome=S6_PRIVATE_KEYS), "missing_max_tokens": _gaps(dispatch=A10_NO_MAX_TOKENS), - "missing_api_key": _gaps(outcome=S14_MISSING_KEY, logged=S14_MISSING_KEY), - "stream_text": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), - "stream_tool_use": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), - "stream_thinking": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), - "stream_error_event_mid_stream": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), - "stream_longer_than_timeout": _gaps( - outcome=S5_STREAM_DEADLINE, - hidden_params=S5_STREAM_DEADLINE, - logged=S5_STREAM_DEADLINE, - logged_metadata=S5_STREAM_DEADLINE, - logged_api_key=S5_STREAM_DEADLINE, - ), - "stream_upstream_429": _gaps(outcome=S1_S2_ERROR_HEADERS, logged_api_key=C7_API_KEY), - "stream_false_explicit": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "sync_text": _gaps( - request=S9_SYNC, - outcome=S9_SYNC, - hidden_params=S9_SYNC, - logged=S9_SYNC, - logged_metadata=S9_SYNC, - logged_response_headers=S9_SYNC, - logged_api_key=S9_SYNC, - logged_call=S9_SYNC, - ), - "sync_stream": _gaps( - request=S9_SYNC, - outcome=S9_SYNC, - hidden_params=S9_SYNC, - logged=S9_SYNC, - logged_metadata=S9_SYNC, - logged_response_headers=S9_SYNC, - logged_api_key=S9_SYNC, - logged_call=S9_SYNC, - ), - "sync_upstream_429": _gaps( - request=S9_SYNC, - outcome=S9_SYNC, - logged=S9_SYNC, - logged_response_headers=S9_SYNC, - logged_api_key=S9_SYNC, - logged_call=S9_SYNC, - ), "mock_response": _gaps( + logged_response_headers=F_MOCK, request=F_MOCK, outcome=F_MOCK, hidden_params=F_MOCK, @@ -556,32 +365,25 @@ KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType( logged_call=F_MOCK, ), "mock_response_rate_limit": _gaps( - request=F_MOCK, outcome=F_MOCK, logged=F_MOCK, logged_api_key=F_MOCK, logged_call=F_MOCK + logged_response_headers=F_MOCK, + request=F_MOCK, + outcome=F_MOCK, + logged=F_MOCK, + logged_api_key=F_MOCK, + logged_call=F_MOCK, ), - "thinking_replay": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), "thinking_signature_rejected_then_retried": _gaps( request=F_THINKING_RETRY, outcome=F_THINKING_RETRY, hidden_params=F_THINKING_RETRY, logged=F_THINKING_RETRY, - logged_metadata=C4_METADATA, - logged_response_headers=C6_RESPONSE_HEADERS, - logged_api_key=C7_API_KEY, + logged_metadata=F_THINKING_RETRY, + logged_response_headers=F_THINKING_RETRY, ), - "cache_control_injection_points": _gaps( - request=F_CACHE_INJECTION, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "tool_custom_field": _gaps( - request=F_TOOL_CUSTOM, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "strict_tools_rejected_then_retried": _gaps(outcome=S2_ERROR_MESSAGE, logged_api_key=C7_API_KEY), - "modify_params": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "stream_cost_injection": _gaps(outcome=F_STREAM_COST, hidden_params=F_STREAM_COST, logged_api_key=C7_API_KEY), - "enable_prompt_caching": _gaps( - request=F_CACHE_INJECTION, logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY - ), - "request_timeout_kwarg": _gaps(logged_response_headers=C6_RESPONSE_HEADERS, logged_api_key=C7_API_KEY), - "stream_timeout_kwarg": _gaps(hidden_params=C8_STREAM_HIDDEN, logged_api_key=C7_API_KEY), + "cache_control_injection_points": _gaps(request=F_CACHE_INJECTION), + "tool_custom_field": _gaps(request=F_TOOL_CUSTOM), + "stream_cost_injection": _gaps(outcome=F_STREAM_COST), + "enable_prompt_caching": _gaps(request=F_CACHE_INJECTION), } ) @@ -606,14 +408,55 @@ REQUEST_CASES: Final = ( Case("metadata_user_id", _kw(metadata={"user_id": "user-123"})), Case("metadata_extra_keys_are_filtered", _kw(metadata={"user_id": "user-123", "session": "s-1"})), Case("metadata_user_id_not_a_string", _kw(metadata={"user_id": 123})), - Case("service_tier", _kw(service_tier="auto"), gaps=_gaps(request="R1: native drops service_tier")), + Case("service_tier", _kw(service_tier="auto")), Case("thinking_enabled", _kw(thinking={"type": "enabled", "budget_tokens": 2048}, max_tokens=4096)), Case("thinking_disabled", _kw(thinking={"type": "disabled"})), Case( "legacy_thinking_on_adaptive_model_keeps_display", _kw(thinking={"type": "enabled", "budget_tokens": 2048, "display": "summarized"}, max_tokens=4096), model=OPUS_ADAPTIVE, - gaps=_gaps(request="R3: native replaces legacy thinking with a bare adaptive block and drops display"), + ), + Case( + "legacy_thinking_display_updates_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": 2048, "display": "updates"}, max_tokens=4096), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_string_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": " 4_096 "}, max_tokens=8192), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_non_numeric_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": "lots"}, max_tokens=8192), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_null_budget_on_adaptive_model", + _kw(thinking={"type": "enabled", "budget_tokens": None}, max_tokens=8192), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_non_numeric_budget_dropped", + _kw(thinking={"type": "enabled", "budget_tokens": [1]}, max_tokens=8192, drop_params=True), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_carries_extra_keys", + _kw( + thinking={"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "kept"}, + max_tokens=4096, + ), + model=OPUS_ADAPTIVE, + ), + Case( + "legacy_thinking_drops_extra_keys_with_drop_params", + _kw( + thinking={"type": "enabled", "budget_tokens": 2048, "block_binding": "strict", "note": "dropped"}, + max_tokens=4096, + drop_params=True, + ), + model=OPUS_ADAPTIVE, ), Case( "legacy_thinking_float_budget_on_adaptive_model", @@ -731,7 +574,6 @@ REQUEST_CASES: Final = ( Case( "anthropic_beta_from_headers_and_extra_headers", _kw(headers={"anthropic-beta": "client-beta-1"}, extra_headers={"anthropic-beta": "deploy-beta-2"}), - gaps=_gaps(request="R2: native keeps the last anthropic-beta source instead of the union"), ), Case("caller_anthropic_version_capitalised", _kw(extra_headers={"Anthropic-Version": "2023-06-01"})), Case( @@ -774,6 +616,24 @@ REQUEST_CASES: Final = ( ), model=OPUS_ADAPTIVE, ), + Case( + "deployment_effort_levels_from_legacy_thinking", + _kw( + thinking={"type": "enabled", "budget_tokens": 30000}, + max_tokens=40000, + model_info={"id": "deployment-1", "reasoning_effort_levels": ["low", "medium"]}, + ), + model=OPUS_ADAPTIVE, + ), + Case( + "deployment_effort_levels_below_request_with_drop_params", + _kw( + output_config={"effort": "high"}, + drop_params=True, + model_info={"id": "deployment-1", "reasoning_effort_levels": ["low"]}, + ), + model=OPUS_ADAPTIVE, + ), ) RESPONSE_CASES: Final = ( @@ -820,6 +680,18 @@ FAILURE_CASES: Final = ( Case("upstream_non_json_error", upstream=(ResponseSpec(body=b"bad gateway", status=502),)), Case("upstream_invalid_json_success", upstream=(ResponseSpec(body=b"not json", status=200),)), Case("upstream_timeout", _kw(timeout=0.5), upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), delay=1.5),)), + Case( + "response_trickles_slower_than_timeout", + _kw(timeout=1), + upstream=(ResponseSpec(body=dict(TEXT_RESPONSE), chunk_delay=0.5, body_chunks=4),), + gaps=_gaps( + outcome=S15_COMPLETE_DEADLINE, + hidden_params=S15_COMPLETE_DEADLINE, + logged=S15_COMPLETE_DEADLINE, + logged_metadata=S15_COMPLETE_DEADLINE, + logged_response_headers=S15_COMPLETE_DEADLINE, + ), + ), Case("missing_max_tokens", _kw(max_tokens=None), upstream=()), Case("missing_api_key", _kw(api_key=None), upstream=()), ) @@ -838,6 +710,11 @@ STREAM_CASES: Final = ( _kw(stream=True, timeout=1), upstream=(_stream(SLOW_STREAM, chunk_delay=0.25),), ), + Case( + "stream_read_stalls_past_timeout", + _kw(stream=True, timeout=0.5), + upstream=(_stream(SLOW_STREAM, chunk_delay=1.0),), + ), Case( "stream_upstream_429", _kw(stream=True), @@ -967,7 +844,9 @@ def _settings(settings: Mapping[str, object]) -> Generator[None]: def _kwargs(case: Case, server: RecordingServer) -> dict[str, object]: - supplied: Final = dict(case.kwargs) + # Deep: Python writes into the caller's own `litellm_metadata`, which must not leak into the + # other leg. + supplied: Final = copy.deepcopy(dict(case.kwargs)) suffix: Final = supplied.pop("api_base_suffix", "") kwargs: Final[dict[str, object]] = { "model": case.model, diff --git a/tests/test_litellm_rust/cache/test_python_cache.py b/tests/test_litellm_rust/cache/test_python_cache.py index 722cdbcbaca..beb818c7162 100644 --- a/tests/test_litellm_rust/cache/test_python_cache.py +++ b/tests/test_litellm_rust/cache/test_python_cache.py @@ -18,10 +18,7 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( ) from litellm.proxy.hooks.parallel_request_limiter_v3 import PROXY_MaxParallelRequestsHandler_v3 from litellm.proxy.utils import InternalUsageCache -from litellm.rust_bridge import runtime -from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule -from litellm.rust_bridge.configuration import Rollout -from litellm.rust_bridge.public_call import NativeCall +from litellm.rust_bridge.catalog import Route from litellm.types.caching import CachingSupportedCallTypes from tests.test_litellm_rust.cache.support import cache_key, collect, invoke, payload from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_EVENTS, MESSAGES_MODEL, MESSAGES_RESPONSE @@ -430,49 +427,6 @@ async def test_python_cache_operations_stay_in_the_rust_callers_task( assert len(recording_server.requests) == 1 -def test_sync_rust_messages_calls_python_cache(recording_server: RecordingServer) -> None: - from litellm.rust_bridge.messages.entrypoints import NATIVE_MESSAGES - - litellm.cache = Cache() - recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) - arguments: Final = { - "model": MESSAGES_MODEL, - "messages": list(MESSAGES), - "max_tokens": 32, - "api_key": "test-key", - "api_base": recording_server.base_url, - } - request: Final = NativeCall( - args=(), - kwargs=arguments, - base={ - "model": MESSAGES_MODEL, - "messages": list(MESSAGES), - "max_tokens": 32, - "stream": None, - "api_key": "test-key", - "api_base": recording_server.base_url, - "custom_llm_provider": "anthropic", - **arguments, - }, - ) - - def call() -> object: - return runtime.run( - RouteContext(Route.MESSAGES), - binding=NATIVE_MESSAGES, - native=lambda hook: hook(request), - python=runtime.NO_PYTHON, - rules=(RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),), - ) - - first: Final = call() - second: Final = call() - assert payload(first) == payload(second) - assert cache_key(second) - assert len(recording_server.requests) == 1 - - @pytest.mark.asyncio @pytest.mark.parametrize("namespace_source", ("cache", "metadata")) async def test_rust_messages_legacy_cache_honors_request_namespaces( diff --git a/tests/test_litellm_rust/messages/test_callbacks.py b/tests/test_litellm_rust/messages/test_callbacks.py index cd5478dfc3a..a3c03898a12 100644 --- a/tests/test_litellm_rust/messages/test_callbacks.py +++ b/tests/test_litellm_rust/messages/test_callbacks.py @@ -14,7 +14,6 @@ from litellm.types.integrations.argilla import ArgillaItem from litellm.types.utils import ModelResponse, StandardLoggingPayload from tests.test_litellm_rust.messages.data import BAD_REQUEST, MESSAGES_RESPONSE, RATE_LIMITED, call from tests.test_litellm_rust.support.callback_recorder import HookRecorder, RecordingLogger, request_body -from tests.test_litellm_rust.support.gaps import gap from tests.test_litellm_rust.support.isolation import rebound from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, relaxed from tests.test_litellm_rust.support.results import response_id, with_id @@ -22,7 +21,6 @@ from tests.test_litellm_rust.support.routing import assert_served_natively pytestmark = pytest.mark.requires_rust_extension -SYNC_HANDLERS_ON_ASYNC_CALLS: Final = gap("native async calls also run the sync success and failure handlers") _OBJECT: Final = TypeAdapter(dict[str, object]) _HEADERS: Final = TypeAdapter(dict[str, str]) @@ -211,7 +209,7 @@ async def test_pre_call_deployment_rejection_logs_failure_without_provider_io(se failures: Final = await recorder.wait_for_async("async_log_failure_event") assert raised.value is rejection - assert server.requests == [] + assert not server.requests assert len(failures) == 1 assert _OBJECT.validate_python(failures[0].kwargs)["exception"] is raised.value assert not {"log_pre_api_call", "log_success_event", "async_log_success_event"} & set(recorder.names) @@ -341,8 +339,8 @@ async def test_a_failing_logger_does_not_fail_the_call(server: RecordingServer) pytest.param("input", False, False, 1, id="global-input"), pytest.param("success", True, False, 1, id="global-async-success"), pytest.param("success", True, True, 1, id="request-async-success"), - pytest.param("success", False, False, 0, id="global-sync-success", marks=SYNC_HANDLERS_ON_ASYNC_CALLS), - pytest.param("success", False, True, 0, id="request-sync-success", marks=SYNC_HANDLERS_ON_ASYNC_CALLS), + pytest.param("success", False, False, 0, id="global-sync-success"), + pytest.param("success", False, True, 0, id="request-sync-success"), pytest.param("failure", True, False, 1, id="global-async-failure"), pytest.param("failure", False, False, 1, id="global-sync-failure"), pytest.param("failure", False, True, 1, id="request-sync-failure"), @@ -384,12 +382,11 @@ async def test_pre_call_rule_rejects_before_provider_io(server: RecordingServer) checked.append(text) return False - with rebound(litellm, "pre_call_rules", [reject]): - with pytest.raises(litellm.APIResponseValidationError): - await call(server) + with rebound(litellm, "pre_call_rules", [reject]), pytest.raises(litellm.APIResponseValidationError): + await call(server) assert checked == ["Hello"] - assert server.requests == [] + assert not server.requests @pytest.mark.asyncio @@ -419,11 +416,11 @@ async def test_dataset_hook_returning_none_drops_the_record_from_the_success_sin @pytest.mark.asyncio -async def test_sync_call_returns_the_provider_message(server: RecordingServer) -> None: +async def test_sync_call_raises_like_python_without_logging(server: RecordingServer) -> None: recorder: Final = RecordingLogger() - response: Final = await call(server, asynchronous=False, callbacks=[recorder]) + with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): + await call(server, asynchronous=False, callbacks=[recorder]) - assert_served_natively(server, response) - assert response_id(response) == MESSAGES_RESPONSE["id"] - assert len(await recorder.wait_for_async("log_success_event")) == 1 + assert not server.requests + assert not recorder.names diff --git a/tests/test_litellm_rust/messages/test_streaming.py b/tests/test_litellm_rust/messages/test_streaming.py index c700c093d1a..e57631f48b9 100644 --- a/tests/test_litellm_rust/messages/test_streaming.py +++ b/tests/test_litellm_rust/messages/test_streaming.py @@ -6,7 +6,6 @@ import pytest from tests.test_litellm_rust.messages.data import MESSAGES_EVENTS, MESSAGES_RESPONSE, call from tests.test_litellm_rust.support.callback_recorder import RecordingLogger, drain_logging from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec, relaxed -from tests.test_litellm_rust.support.results import consume from tests.test_litellm_rust.support.routing import assert_served_natively pytestmark = pytest.mark.requires_rust_extension @@ -46,16 +45,14 @@ async def test_stream_relays_provider_events_and_logs_success_once_after_the_las @pytest.mark.asyncio -async def test_sync_stream_relays_provider_events_and_logs_success_once(server: RecordingServer) -> None: - server.enqueue(STREAM) +async def test_sync_stream_raises_like_python_before_sending(server: RecordingServer) -> None: recorder: Final = RecordingLogger() - stream: Final = await call(server, asynchronous=False, stream=True, callbacks=[recorder]) - chunks: Final = await consume(stream) + with pytest.raises(ValueError, match="anthropic_messages_handler is not implemented for sync calls"): + await call(server, asynchronous=False, stream=True, callbacks=[recorder]) - assert_served_natively(server) - assert b"".join(chunks) == b"".join(STREAM.payloads()) - assert len(await recorder.wait_for_async("async_log_success_event")) == 1 + assert not server.requests + assert not recorder.names @pytest.mark.asyncio diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py index 7beeb20f109..5743658abdf 100644 --- a/tests/test_litellm_rust/support/recording_server.py +++ b/tests/test_litellm_rust/support/recording_server.py @@ -27,12 +27,14 @@ class ResponseSpec: delay: float = 0 chunk_delay: float = 0 events: tuple[tuple[str, object], ...] = () + # Splits a non-stream body into this many writes, `chunk_delay` apart, so it trickles in. + body_chunks: int = 1 def payloads(self) -> tuple[bytes, ...]: - if isinstance(self.body, bytes): - return (self.body,) if not self.events: - return (json.dumps(self.body).encode(),) + body: Final = self.body if isinstance(self.body, bytes) else json.dumps(self.body).encode() + size: Final = max(-(-len(body) // max(self.body_chunks, 1)), 1) + return tuple(body[start : start + size] for start in range(0, len(body), size)) or (body,) return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events) diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index abf0e055e73..9d8b09dd1c4 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -138,3 +138,110 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" assert additional["request-id"] == additional["llm_provider-request-id"] + + +_ADAPTIVE_OPUS: Final = "anthropic/claude-opus-4-7" + + +@pytest.mark.parametrize( + ("model", "expected"), + [("claude-sonnet-4-5", "anthropic"), ("gpt-4o", "openai"), ("not-a-known-model", None)], +) +def test_inferred_provider_is_the_one_the_python_handler_resolves(model: str, expected: str | None) -> None: + assert messages_host.inferred_provider(model, None) == expected + + +@pytest.mark.parametrize( + ("request_effort", "levels", "drop_params", "expected"), + [ + ("max", ["low", "high"], False, "high"), + ("high", ["low"], True, "low"), + ], +) +def test_body_params_round_effort_to_the_levels_the_deployment_declares( + request_effort: str, levels: list[str], drop_params: bool, expected: str +) -> None: + request: Final[dict[str, object]] = { + "model": _ADAPTIVE_OPUS, + "messages": [], + "max_tokens": 8, + "output_config": {"effort": request_effort}, + "drop_params": drop_params, + "model_info": {"id": "deployment-1", "reasoning_effort_levels": levels}, + } + + assert messages_host.body_params(request)["output_config"] == {"effort": expected} + + +def test_body_params_keep_the_requested_effort_without_deployment_levels() -> None: + request: Final[dict[str, object]] = { + "model": _ADAPTIVE_OPUS, + "messages": [], + "max_tokens": 8, + "output_config": {"effort": "max"}, + } + + assert messages_host.body_params(request)["output_config"] == {"effort": "max"} + + +def _marked(phase: str | None = None, *args: object) -> ValueError: + error: Final = ValueError(*args) + if phase is not None: + error.messages_timeout = phase # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + return error + + +def test_a_stalled_stream_body_reaches_the_caller_as_the_transport_timeout() -> None: + import httpx + + mapped: Final = messages_host.map_failure(_marked("Body"), {"model": _ADAPTIVE_OPUS}, "anthropic") + + assert isinstance(mapped, httpx.ReadTimeout) + assert str(mapped) == "Timeout on reading data from socket" + + +def test_a_native_read_timeout_maps_to_the_public_timeout_with_python_text() -> None: + mapped: Final = messages_host.map_failure(_marked("Read"), {"model": _ADAPTIVE_OPUS, "timeout": 1.0}, "anthropic") + + assert isinstance(mapped, litellm.Timeout) + assert "Read timed out. Timeout passed=1.0" in str(mapped) + + +def test_an_upstream_status_maps_through_the_provider_error_class() -> None: + upstream: Final = ValueError(429, "slow down") + upstream.headers = [("retry-after", "3")] # pyright: ignore[reportAttributeAccessIssue] # set by the native host + + mapped: Final = messages_host.map_failure(upstream, {"model": _ADAPTIVE_OPUS}, "anthropic") + + assert isinstance(mapped, litellm.RateLimitError) + assert "slow down" in str(mapped) + assert mapped.__context__ is upstream + + +def test_a_late_rejection_tells_the_logger_the_resolved_call_first() -> None: + import datetime + + from litellm.litellm_core_utils.litellm_logging import Logging + + logger: Final = Logging( + model=_ADAPTIVE_OPUS, + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="late", + function_id="late", + ) + rejected: Final = ValueError("budget_tokens must be a number") + rejected.messages_late_rejection = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + rejected.messages_request_error = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + + mapped: Final = messages_host.map_failure( + rejected, + {"model": _ADAPTIVE_OPUS, "messages": [], "max_tokens": 8, "litellm_logging_obj": logger}, + "anthropic", + ) + + assert isinstance(mapped, litellm.BadRequestError) + assert logger.model_call_details["custom_llm_provider"] == "anthropic" + assert logger.model_call_details["optional_params"] == {"max_tokens": 8} diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index a40f8a77105..1ba0188e76d 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -5,6 +5,7 @@ from pathlib import Path from types import MappingProxyType from typing import Final +import httpx import pytest from pydantic import TypeAdapter @@ -244,3 +245,56 @@ def test_the_rust_contract_matches_the_shim_signatures() -> None: contract: Final = TypeAdapter(dict[str, list[str]]).validate_json(CONTRACT_PATH.read_text()) assert contract == {name: list(inspect.signature(getattr(legacy, name)).parameters) for name in contract} + + +def _messages_logger() -> Logging: + return Logging( + model="claude-sonnet-4-5", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.datetime.now(), + litellm_call_id="messages", + function_id="messages", + ) + + +def test_native_provider_response_is_recorded_for_logging_replay() -> None: + logger: Final = _messages_logger() + + legacy.record_response(logger, 200, (("request-id", "req_native"),), '{"id": "msg_native"}') + + recorded: Final[object] = logger.model_call_details["httpx_response"] + assert isinstance(recorded, httpx.Response) + assert recorded.status_code == 200 + assert recorded.content == b'{"id": "msg_native"}' + assert logger.model_call_details["response_headers"]["request-id"] == "req_native" + + +def test_native_provider_exchange_window_is_recorded_like_the_timed_post() -> None: + logger: Final = _messages_logger() + logger.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed"}} + + legacy.record_api_window(logger, 1_700_000_000.0, 1_700_000_000.25) + + assert logger.model_call_details["llm_api_duration_ms"] == 250.0 + metadata: Final = logger.model_call_details["litellm_params"]["litellm_metadata"] + assert metadata["llm_api_timing_windows"] == ((1_700_000_000.0, 1_700_000_000.25),) + + +def test_stream_head_is_stamped_in_place_with_the_call_hidden_params() -> None: + logger: Final = _messages_logger() + head: Final[dict[str, object]] = {"additional_headers": {"request-id": "req_native"}} + started: Final = datetime.datetime.now() + + legacy.stamp_stream_head( + head, + logger, + {"model": "anthropic/claude-sonnet-4-5", "litellm_call_id": "messages"}, + started, + started + datetime.timedelta(milliseconds=5), + ) + + assert head["additional_headers"] == {"request-id": "req_native", "llm_provider-request-id": "req_native"} + assert head["litellm_call_id"] == "messages" + assert "_response_ms" in head diff --git a/tests/unit/rust_bridge/messages/test_secrets.py b/tests/unit/rust_bridge/messages/test_secrets.py index 64559d63193..5ee89fa7794 100644 --- a/tests/unit/rust_bridge/messages/test_secrets.py +++ b/tests/unit/rust_bridge/messages/test_secrets.py @@ -11,7 +11,7 @@ import litellm from litellm.integrations.custom_secret_manager import CustomSecretManager from litellm.llms.anthropic.messages.native.handler import anthropic_messages from litellm.rust_bridge.host import settings -from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES +from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES from litellm.rust_bridge.public_call import NativeCall from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from tests.test_litellm_rust.messages.data import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE @@ -72,21 +72,13 @@ async def _python_messages() -> object: return await anthropic_messages(**_public_kwargs()) -async def _rust_messages() -> object: - route: Final = NATIVE_MESSAGES.load() - assert route is not None - return route(_native_request()) - - async def _rust_amessages() -> object: route: Final = NATIVE_AMESSAGES.load() assert route is not None return await route(_native_request()) -@pytest.fixture( - params=(_python_messages, _rust_messages, _rust_amessages), ids=("python-async", "rust-sync", "rust-async") -) +@pytest.fixture(params=(_python_messages, _rust_amessages), ids=("python-async", "rust-async")) def messages(request: pytest.FixtureRequest) -> Messages: return cast(Messages, request.param) From fcef9cd0a3aa69580a87293f724d8afb11de8858 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Mon, 2 Nov 2026 21:28:46 -0800 Subject: [PATCH 09/34] feat(videos): serve custom_provider_map BaseVideoConfig providers on /v1/videos (#49785) * feat(videos): serve custom_provider_map BaseVideoConfig providers on /v1/videos A provider registered in custom_provider_map with a BaseVideoConfig handler now resolves on every video call type (create, status, content, list, remix, edit, extension, characters), the same way CustomLLM handlers already resolve on /v1/images/generations. The video entry points and the HTTP handler stop casting the provider name to LlmProviders, which raised before any lookup ran, and a provider without a video config gets a 400 instead of a 500. * fix(videos): type the custom provider lookups so the type gate holds Pick the CustomLLM for a custom provider through ProviderConfigManager.get_custom_llm_handler, which narrows the now wider custom_provider_map handler union, read custom_provider_map instead of the private _custom_providers list, and annotate the untyped custom_llm_provider parameters on the video entry points * test(videos): cover a CustomLLM shadowing a built-in video provider name and type the new tests Mark custom_handler ReadOnly on CustomLLMItem, type the new tests, and assert that a CustomLLM registered under openai still leaves openai video on the built-in config --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/images/main.py | 11 +- litellm/llms/custom_httpx/llm_http_handler.py | 20 +- litellm/main.py | 12 +- litellm/types/llms/custom_llm.py | 5 +- litellm/utils.py | 36 +++- litellm/videos/main.py | 79 ++++---- .../_support/custom_video_provider.py | 181 ++++++++++++++++++ .../test_custom_video_provider_wire.py | 113 +++++++++++ tests/unit/test_utils.py | 80 +++++--- tests/unit/videos/test_main.py | 74 +++---- 10 files changed, 488 insertions(+), 123 deletions(-) create mode 100644 tests/integration/_support/custom_video_provider.py create mode 100644 tests/integration/providers/test_custom_video_provider_wire.py diff --git a/litellm/images/main.py b/litellm/images/main.py index 165543aadba..f37b252fa45 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -28,7 +28,6 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.llms.custom_llm import CustomLLM from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params #################### Initialize provider clients #################### @@ -608,10 +607,7 @@ def image_generation( ) elif custom_llm_provider in litellm._custom_providers: # Assume custom LLM provider # Get the Custom Handler - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -915,10 +911,7 @@ def image_edit( # Check for custom provider if custom_llm_provider in litellm._custom_providers: - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 879e4e02e3e..d247c5169bd 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -8105,7 +8105,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8301,7 +8301,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8482,7 +8482,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8646,7 +8646,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8787,7 +8787,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -8960,7 +8960,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9147,7 +9147,7 @@ class BaseLLMHTTPHandler: ): if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9280,7 +9280,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9362,7 +9362,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: @@ -9547,7 +9547,7 @@ class BaseLLMHTTPHandler: """ if client is None or not isinstance(client, AsyncHTTPHandler): async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders(custom_llm_provider), + llm_provider=custom_llm_provider, params={"ssl_verify": litellm_params.get("ssl_verify", None)}, ) else: diff --git a/litellm/main.py b/litellm/main.py index 38dd9f90be9..b5d1398ecad 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -266,7 +266,7 @@ from .llms.codestral.completion.handler import CodestralTextCompletion from .llms.cohere.embed import handler as cohere_embed from .llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from .llms.custom_llm import CustomLLM, custom_chat_llm_router +from .llms.custom_llm import custom_chat_llm_router from .llms.databricks.common_utils import DatabricksBase from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha @@ -5337,10 +5337,7 @@ def _complete_custom_providers( stream: Final = ctx.stream timeout: Final = ctx.timeout - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) @@ -7911,10 +7908,7 @@ def embedding( litellm_params={}, ) elif custom_llm_provider in litellm._custom_providers: - custom_handler: CustomLLM | None = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + custom_handler: Final = ProviderConfigManager.get_custom_llm_handler(custom_llm_provider) if custom_handler is None: raise LiteLLMUnknownProvider(model=model, custom_llm_provider=custom_llm_provider) diff --git a/litellm/types/llms/custom_llm.py b/litellm/types/llms/custom_llm.py index 6b583b0a83d..bb12449617d 100644 --- a/litellm/types/llms/custom_llm.py +++ b/litellm/types/llms/custom_llm.py @@ -1,11 +1,12 @@ from typing import TYPE_CHECKING -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict if TYPE_CHECKING: + from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_llm import CustomLLM class CustomLLMItem(TypedDict): provider: str - custom_handler: "CustomLLM" + custom_handler: ReadOnly["CustomLLM | BaseVideoConfig"] diff --git a/litellm/utils.py b/litellm/utils.py index ddfc38ca677..85e669843fe 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -467,6 +467,7 @@ if TYPE_CHECKING: ) from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.llms.custom_llm import CustomLLM from litellm.types.llms.anthropic import ( ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, @@ -10958,8 +10959,13 @@ class ProviderConfigManager: @staticmethod def get_provider_video_config( model: str | None, - provider: LlmProviders, + provider: LlmProviders | str, ) -> BaseVideoConfig | None: + if not isinstance(provider, LlmProviders): + custom_video_config: Final = ProviderConfigManager.get_custom_provider_video_config(provider) + if custom_video_config is not None or provider not in LlmProvidersSet: + return custom_video_config + return ProviderConfigManager.get_provider_video_config(model=model, provider=LlmProviders(provider)) if LlmProviders.OPENAI == provider: from litellm.llms.openai.videos.transformation import OpenAIVideoConfig @@ -11010,6 +11016,34 @@ class ProviderConfigManager: return VolcEngineVideoConfig() return None + @staticmethod + def get_custom_llm_handler(provider: str | None) -> CustomLLM | None: + """The `CustomLLM` registered for `provider` in `litellm.custom_provider_map`, if its handler is one.""" + from litellm.llms.custom_llm import CustomLLM + + return next( + ( + item["custom_handler"] + for item in litellm.custom_provider_map + if item["provider"] == provider and isinstance(item["custom_handler"], CustomLLM) + ), + None, + ) + + @staticmethod + def get_custom_provider_video_config(provider: str) -> BaseVideoConfig | None: + """The `BaseVideoConfig` registered for `provider` in `litellm.custom_provider_map`, if its handler is one.""" + from litellm.llms.base_llm.videos.transformation import BaseVideoConfig + + return next( + ( + item["custom_handler"] + for item in litellm.custom_provider_map + if item["provider"] == provider and isinstance(item["custom_handler"], BaseVideoConfig) + ), + None, + ) + @staticmethod def get_provider_container_config( provider: LlmProviders, diff --git a/litellm/videos/main.py b/litellm/videos/main.py index 85e651b687c..66eb10e30db 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -26,6 +26,17 @@ from litellm.types.videos.utils import decode_video_id_with_provider from litellm.utils import ProviderConfigManager, client from litellm.videos.utils import VideoGenerationRequestUtils + +def _unsupported_video_call( + call_name: str, custom_llm_provider: str, model: str | None = None +) -> litellm.BadRequestError: + return litellm.BadRequestError( + message=f"{call_name} is not supported for {custom_llm_provider}", + model=model or "", + llm_provider=custom_llm_provider, + ) + + #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() @@ -40,7 +51,7 @@ async def avideo_generation( size: str | None = None, user: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -170,7 +181,7 @@ def video_generation( size: str | None = None, user: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -208,11 +219,11 @@ def video_generation( # get provider config video_generation_provider_config: BaseVideoConfig | None = ProviderConfigManager.get_provider_video_config( model=model, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_generation_provider_config is None: - raise ValueError(f"video generation is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video generation", custom_llm_provider, model) local_vars.update(kwargs) # Get VideoGenerationOptionalRequestParams with only valid parameters @@ -329,11 +340,11 @@ def video_content( # get provider config video_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_provider_config is None: - raise ValueError(f"video support download is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video support download", custom_llm_provider) local_vars.update(kwargs) # For video content download, we don't need complex optional parameter handling @@ -456,7 +467,7 @@ async def avideo_remix( video_id: str, prompt: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -559,7 +570,7 @@ def video_remix( video_id: str, prompt: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -598,11 +609,11 @@ def video_remix( # get provider config video_remix_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_remix_provider_config is None: - raise ValueError(f"video remix is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video remix", custom_llm_provider) local_vars.update(kwargs) # For video remix, we need the video_id and prompt @@ -660,7 +671,7 @@ async def avideo_list( order: str | None = None, api_key: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -774,7 +785,7 @@ def video_list( limit: int | None = None, order: str | None = None, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -810,11 +821,11 @@ def video_list( # get provider config video_list_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_list_provider_config is None: - raise ValueError(f"video list is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video list", custom_llm_provider) local_vars.update(kwargs) # For video list, we need the query parameters @@ -871,7 +882,7 @@ def video_list( async def avideo_status( video_id: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -970,7 +981,7 @@ def video_status( def video_status( video_id: str, timeout=600, # default to 10 minutes - custom_llm_provider=None, + custom_llm_provider: str | None = None, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: dict[str, object] | None = None, @@ -1031,11 +1042,11 @@ def video_status( # get provider config video_status_provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if video_status_provider_config is None: - raise ValueError(f"video status is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video status", custom_llm_provider) local_vars.update(kwargs) # For video status, we need the video_id @@ -1088,7 +1099,7 @@ async def avideo_create_character( name: str, video: FileTypes, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1143,7 +1154,7 @@ def video_create_character( name: str, video: FileTypes, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1172,11 +1183,11 @@ def video_create_character( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video create character is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video create character", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = {"name": name} @@ -1218,7 +1229,7 @@ def video_create_character( async def avideo_get_character( character_id: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1268,7 +1279,7 @@ async def avideo_get_character( def video_get_character( character_id: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1297,11 +1308,11 @@ def video_get_character( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video get character is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video get character", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = {"character_id": character_id} @@ -1343,7 +1354,7 @@ async def avideo_edit( video_id: str, prompt: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1398,7 +1409,7 @@ def video_edit( video_id: str, prompt: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1430,11 +1441,11 @@ def video_edit( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video edit is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video edit", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = {"video_id": video_id, "prompt": prompt} @@ -1480,7 +1491,7 @@ async def avideo_extension( prompt: str, seconds: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1534,7 +1545,7 @@ def video_extension( prompt: str, seconds: str, timeout=600, - custom_llm_provider=None, + custom_llm_provider: str | None = None, extra_headers: dict[str, object] | None = None, extra_query: dict[str, object] | None = None, extra_body: dict[str, object] | None = None, @@ -1564,11 +1575,11 @@ def video_extension( provider_config: Final[BaseVideoConfig | None] = ProviderConfigManager.get_provider_video_config( model=None, - provider=litellm.LlmProviders(custom_llm_provider), + provider=custom_llm_provider, ) if provider_config is None: - raise ValueError(f"video extension is not supported for {custom_llm_provider}") + raise _unsupported_video_call("video extension", custom_llm_provider) local_vars.update(kwargs) request_params: Final[dict] = { diff --git a/tests/integration/_support/custom_video_provider.py b/tests/integration/_support/custom_video_provider.py new file mode 100644 index 00000000000..3513d1e3186 --- /dev/null +++ b/tests/integration/_support/custom_video_provider.py @@ -0,0 +1,181 @@ +"""A customer-owned video provider for the proxy's `custom_provider_map`: a job queue with its own wire shape.""" + +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +from httpx._types import RequestFiles + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_llm import CustomLLM +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id + +_STATUS_BY_STATE: Final = MappingProxyType({"queued": "queued", "running": "in_progress", "done": "completed"}) + + +def _video_object(job: Mapping[str, object], model: str | None, custom_llm_provider: str | None) -> VideoObject: + job_id: Final = str(job["job_id"]) + seconds: Final = job.get("seconds") + video: Final = VideoObject( + id=encode_video_id_with_provider(job_id, custom_llm_provider, model) if custom_llm_provider else job_id, + object="video", + status=_STATUS_BY_STATE[str(job["state"])], + created_at=0, + model=model, + seconds=str(seconds) if seconds is not None else None, + ) + return video + + +class QueueVideoConfig(BaseVideoConfig): + def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseVideoConfig contract + return ["seconds"] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict: # mutable-ok: BaseVideoConfig contract + return {key: value for key, value in video_create_optional_params.items() if key == "seconds"} + + def validate_environment( + self, + headers: dict, # mutable-ok: BaseVideoConfig contract + model: str, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> dict: # mutable-ok: BaseVideoConfig contract + key: Final = (litellm_params.api_key if litellm_params is not None else None) or api_key or "" + return {**headers, "X-Api-Key": key} + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: BaseVideoConfig contract + ) -> str: + if api_base is None: + raise ValueError("api_base is required") + return f"{api_base.rstrip('/')}/jobs" + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: dict, # mutable-ok: BaseVideoConfig contract + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + ) -> tuple[dict, RequestFiles, str]: # mutable-ok: BaseVideoConfig contract + seconds: Final = video_create_optional_request_params.get("seconds") + return ({"model": model, "prompt": prompt, "seconds": seconds}, [], api_base) + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + request_data: dict | None = None, + ) -> VideoObject: + return _video_object(raw_response.json(), model, custom_llm_provider) + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + return (f"{api_base}/{extract_original_video_id(video_id)}", {}) + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + client: object = None, + ) -> VideoObject: + return _video_object(raw_response.json(), None, custom_llm_provider) + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + variant: str | None = None, + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + return (f"{api_base}/{extract_original_video_id(video_id)}/file", {}) + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + client: object = None, + ) -> bytes: + return raw_response.content + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, object] | None = None, + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + return (api_base, {}) + + def transform_video_list_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseVideoConfig contract + return raw_response.json() + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + extra_body: dict[str, object] | None = None, + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + raise NotImplementedError("the queue backend has no remix") + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("the queue backend has no remix") + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict]: # mutable-ok: BaseVideoConfig contract + raise NotImplementedError("the queue backend has no delete") + + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> VideoObject: + raise NotImplementedError("the queue backend has no delete") + + +VIDEO_CONFIG: Final = QueueVideoConfig() +IMAGE_ONLY_HANDLER: Final = CustomLLM() diff --git a/tests/integration/providers/test_custom_video_provider_wire.py b/tests/integration/providers/test_custom_video_provider_wire.py new file mode 100644 index 00000000000..a40ecae3ed4 --- /dev/null +++ b/tests/integration/providers/test_custom_video_provider_wire.py @@ -0,0 +1,113 @@ +"""A provider registered in `custom_provider_map` with a `BaseVideoConfig` serves /v1/videos create, status and content.""" + +import json +import shutil +from contextlib import ExitStack +from pathlib import Path +from typing import Final + +from integration._support.client import Gateway +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server + +_SUPPORT_MODULE: Final = Path(__file__).resolve().parents[1] / "_support" / "custom_video_provider.py" +_PROMPT: Final = "a cat walking across a bridge" +_MP4: Final = b"\x00\x00\x00\x18ftypmp42" + b"queue-video-bytes" + + +def _queue_backend(request: Request) -> Reply: + if (request.method, request.target) == ("POST", "/jobs"): + return Reply(body=json.dumps({"job_id": "job-7", "state": "queued"}).encode()) + if (request.method, request.target) == ("GET", "/jobs/job-7"): + return Reply(body=json.dumps({"job_id": "job-7", "state": "done", "seconds": 8}).encode()) + if (request.method, request.target) == ("GET", "/jobs/job-7/file"): + return Reply(body=_MP4, content_type="video/mp4") + return Reply(status=404, body=json.dumps({"error": f"no route {request.method} {request.target}"}).encode()) + + +def _write_config(path: Path, wire: Wire) -> Path: + shutil.copy(_SUPPORT_MODULE, path.parent / "custom_video_provider.py") + path.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "queue-video", + "litellm_params": { + "model": "queue-video/seedance-1", + "api_base": wire.url, + "api_key": "queue-secret", + }, + }, + { + "model_name": "image-only-video", + "litellm_params": { + "model": "image-only/seedance-1", + "api_base": wire.url, + "api_key": "unused", + }, + }, + ], + "litellm_settings": { + "custom_provider_map": [ + {"provider": "queue-video", "custom_handler": "custom_video_provider.VIDEO_CONFIG"}, + {"provider": "image-only", "custom_handler": "custom_video_provider.IMAGE_ONLY_HANDLER"}, + ] + }, + } + ), + encoding="utf-8", + ) + return path + + +def _lower_headers(request: Request) -> dict[str, str]: + return {name.lower(): value for name, value in request.headers.items()} + + +def test_custom_provider_map_base_video_config_serves_create_status_and_content( + gateway: Gateway, tmp_path: Path +) -> None: + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_queue_backend)) + config: Final = _write_config(tmp_path / "custom-video.yaml", wire) + owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config)) + + created: Final = owned.gateway.request( + "POST", "/v1/videos", {"model": "queue-video", "prompt": _PROMPT, "seconds": "8"} + ) + assert created.status_code == 200, created.text + created_body: Final = created.json() + assert created_body["status"] == "queued", created_body + video_id: Final = created_body["id"] + assert isinstance(video_id, str) and video_id.startswith("video_"), created_body + + submits: Final = wire.drain() + assert [(r.method, r.target) for r in submits] == [("POST", "/jobs")], submits + assert json.loads(submits[0].body) == {"model": "seedance-1", "prompt": _PROMPT, "seconds": "8"} + assert _lower_headers(submits[0])["x-api-key"] == "queue-secret" + + status: Final = owned.gateway.request("GET", f"/v1/videos/{video_id}") + assert status.status_code == 200, status.text + assert status.json()["status"] == "completed", status.text + assert status.json()["id"] == video_id, status.text + + content: Final = owned.gateway.request("GET", f"/v1/videos/{video_id}/content") + assert content.status_code == 200, content.text + assert content.content == _MP4 + + polls: Final = wire.drain() + assert [(r.method, r.target) for r in polls] == [("GET", "/jobs/job-7"), ("GET", "/jobs/job-7/file")], polls + assert all(_lower_headers(r)["x-api-key"] == "queue-secret" for r in polls) + + +def test_custom_provider_without_a_video_config_is_refused_with_400(gateway: Gateway, tmp_path: Path) -> None: + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_queue_backend)) + config: Final = _write_config(tmp_path / "custom-video.yaml", wire) + owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config)) + + refused: Final = owned.gateway.request("POST", "/v1/videos", {"model": "image-only-video", "prompt": _PROMPT}) + assert refused.status_code == 400, refused.text + assert "video generation is not supported for image-only" in refused.text + assert wire.drain() == () diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ed2f023dd11..010c2ee6f92 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -967,21 +967,11 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost": {"type": "number"}, "cache_creation_input_token_cost_above_1hr": {"type": "number"}, "cache_creation_input_token_cost_above_12hr": {"type": "number"}, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": { - "type": "number" - }, - "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": { - "type": "number" - }, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_flex": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_priority": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_batches": {"type": "number"}, + "cache_creation_input_token_cost_above_12hr_above_272k_tokens_ultrafast": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_flex": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_priority": {"type": "number"}, "cache_creation_input_token_cost_above_12hr_batches": {"type": "number"}, @@ -1671,6 +1661,38 @@ def test_get_provider_rerank_config(): assert isinstance(config, HostedVLLMRerankConfig) +def test_get_provider_video_config_resolves_a_custom_provider_map_base_video_config( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A `BaseVideoConfig` registered in `custom_provider_map` answers for its provider name, a `CustomLLM` + registered there does not, and a built-in name still resolves to its built-in config even when a `CustomLLM` + shadows it in the map.""" + from litellm.llms.custom_llm import CustomLLM + from litellm.llms.openai.videos.transformation import OpenAIVideoConfig + + class QueueVideoConfig(OpenAIVideoConfig): + pass + + video_config: Final = QueueVideoConfig() + monkeypatch.setattr( + litellm, + "custom_provider_map", + [ + {"provider": "queue-video", "custom_handler": video_config}, + {"provider": "image-only", "custom_handler": CustomLLM()}, + {"provider": "openai", "custom_handler": CustomLLM()}, + ], + ) + monkeypatch.setattr(litellm, "_custom_providers", ["queue-video", "image-only", "openai"]) + + assert ProviderConfigManager.get_provider_video_config(model="seedance-1", provider="queue-video") is video_config + assert ProviderConfigManager.get_provider_video_config(model="seedance-1", provider="image-only") is None + assert isinstance( + ProviderConfigManager.get_provider_video_config(model="sora-2", provider="openai"), OpenAIVideoConfig + ) + assert ProviderConfigManager.get_provider_video_config(model="seedance-1", provider="not-registered") is None + + def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts(): """Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS dropped response_format before the speech_to_completion bridge could honor it.""" @@ -7572,9 +7594,10 @@ async def test_oidc_google_api_key_fails_closed_when_the_metadata_server_fails_a side_effect=[metadata_failure(), httpx.Response(200, text=recovered_token)] ) chat: Final = upstream.post(_VLLM_CHAT_URL).respond(200, json=_VLLM_REPLY) - with caplog.at_level(logging.WARNING, logger="LiteLLM"), pytest.raises( - (litellm.AuthenticationError, litellm.ServiceUnavailableError) - ) as excinfo: + with ( + caplog.at_level(logging.WARNING, logger="LiteLLM"), + pytest.raises((litellm.AuthenticationError, litellm.ServiceUnavailableError)) as excinfo, + ): await _ping_vllm_through(call_path, "vllm-sa") sent_while_failing: Final = chat.call_count await _ping_vllm_through(call_path, "vllm-sa") @@ -7823,9 +7846,7 @@ async def test_router_resolves_an_oidc_api_key_only_on_a_proxy_level_deployment( try: with respx.mock(assert_all_called=True) as upstream: upstream.post(_MISTRAL_CHAT_URL).respond(200, json=_MISTRAL_REPLY) - await router.acompletion( - model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"} - ) + await router.acompletion(model="devstral", messages=_PING, metadata={"user_api_key_team_id": "team-1"}) sent: Final = upstream.calls.last.request.headers["Authorization"] finally: router.discard() @@ -7893,7 +7914,9 @@ async def test_an_inline_oidc_api_key_reaches_the_endpoint_its_deployment_config with respx.mock(assert_all_called=False) as upstream: upstream.route().respond(200, json=_BEDROCK_REPLY) await router.acompletion( - model="claude", messages=_PING, aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com" + model="claude", + messages=_PING, + aws_bedrock_runtime_endpoint="https://BEDROCK-runtime.us-east-1.amazonaws.com", ) sent: Final = tuple( (call.request.url.host, call.request.headers["Authorization"]) for call in upstream.calls @@ -8102,7 +8125,9 @@ async def test_a_team_row_naming_an_oidc_credential_reaches_only_the_credentials ( lambda reference: LiteLLM_Params( **litellm.Router( - model_list=[{"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}}] + model_list=[ + {"model_name": "openai/*", "litellm_params": {"model": "openai/*", "api_key": reference}} + ] ).get_model_list(model_name="openai/*")[0]["litellm_params"] ), lambda reference: "Bearer listing-token", @@ -9108,7 +9133,9 @@ def test_basic_trimming_no_max_tokens_specified(): trimmed_messages = trim_messages(messages, model="gpt-4") print("trimmed messages for gpt-4") print(trimmed_messages) - assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"]["max_input_tokens"] + assert (get_token_count(messages=trimmed_messages, model="gpt-4")) <= litellm.model_cost["gpt-4"][ + "max_input_tokens" + ] def test_multiple_messages_trimming(): @@ -11427,7 +11454,9 @@ def test_declared_mid_conversation_system_support_prefers_the_deployment_over_th ) -def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises(monkeypatch: pytest.MonkeyPatch) -> None: +def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setitem(litellm.model_cost, "hosted_vllm/unreadable-probe", "not a model info entry") litellm.get_model_info.cache_clear() assert litellm.utils.declared_mid_conversation_system_support("unreadable-probe", "hosted_vllm", None) is None @@ -11448,7 +11477,6 @@ def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_ assert optional_params["extra_body"] == {"kept": True, "cache_control_injection_points": [point]} - @pytest.mark.parametrize( ("provider", "expected_warnings"), (("streamlake", 1), ("veniceai", 1), ("my-own-llm", 0)), diff --git a/tests/unit/videos/test_main.py b/tests/unit/videos/test_main.py index 22e1e5c05eb..76e9b062476 100644 --- a/tests/unit/videos/test_main.py +++ b/tests/unit/videos/test_main.py @@ -32,7 +32,7 @@ reflect production. from contextlib import ExitStack from dataclasses import dataclass -from typing import Any, Dict +from typing import Any, Dict, Final from unittest.mock import MagicMock, patch import pytest @@ -205,9 +205,7 @@ def test_video_edit__dispatch_and_provider_from_id(seams): def test_video_extension__dispatch_and_provider_from_id(seams): - result = videos_main.video_extension( - video_id=AZURE_VIDEO_ID, prompt="continue", seconds="5" - ) + result = videos_main.video_extension(video_id=AZURE_VIDEO_ID, prompt="continue", seconds="5") seams.assert_only("video_extension_handler") assert result is seams.handler.video_extension_handler.return_value @@ -285,9 +283,7 @@ def test_generation__mock_response_short_circuits(seams): def test_list__mock_response_short_circuits(seams): - resp = videos_main.video_list( - mock_response=[{"id": "v1", "object": "video", "status": "completed"}] - ) + resp = videos_main.video_list(mock_response=[{"id": "v1", "object": "video", "status": "completed"}]) assert isinstance(resp, list) assert resp[0].id == "v1" @@ -318,7 +314,7 @@ def test_get_character__mock_response_short_circuits(seams): def test_unsupported_provider_raises_without_dispatch(seams): seams.get_config.return_value = None - with pytest.raises(litellm.APIConnectionError): + with pytest.raises(litellm.BadRequestError, match="video status is not supported for azure"): videos_main.video_status(video_id=AZURE_VIDEO_ID) seams.handler.video_status_handler.assert_not_called() @@ -333,9 +329,7 @@ def test_unsupported_provider_raises_without_dispatch(seams): async def test_avideo_generation__delegates_with_async_flag(): sentinel = VideoObject(id="v-async", object="video", status="queued") with ( - patch.object( - videos_main, "video_generation", MagicMock(return_value=sentinel) - ) as sync, + patch.object(videos_main, "video_generation", MagicMock(return_value=sentinel)) as sync, patch.object( litellm, "get_llm_provider", @@ -352,9 +346,7 @@ async def test_avideo_generation__delegates_with_async_flag(): @pytest.mark.asyncio async def test_avideo_status__delegates_untouched(): sentinel = VideoObject(id="v-async", object="video", status="queued") - with patch.object( - videos_main, "video_status", MagicMock(return_value=sentinel) - ) as sync: + with patch.object(videos_main, "video_status", MagicMock(return_value=sentinel)) as sync: result = await videos_main.avideo_status(video_id="video_plain") assert result is sentinel @@ -367,9 +359,7 @@ async def test_avideo_content__pre_decodes_provider_before_delegating(): """avideo_content resolves the provider from the encoded id itself before handing off, so the sync worker receives the decoded provider, not None.""" sentinel = b"mp4-bytes" - with patch.object( - videos_main, "video_content", MagicMock(return_value=sentinel) - ) as sync: + with patch.object(videos_main, "video_content", MagicMock(return_value=sentinel)) as sync: result = await videos_main.avideo_content(video_id=AZURE_VIDEO_ID) assert result is sentinel @@ -393,9 +383,7 @@ DB_YAML_CREDS = { CREDENTIAL_OPERATIONS = [ ( "video_generation_handler", - lambda: videos_main.video_generation( - prompt="p", model="sora-2", **DB_YAML_CREDS - ), + lambda: videos_main.video_generation(prompt="p", model="sora-2", **DB_YAML_CREDS), ), ( "video_status_handler", @@ -407,21 +395,15 @@ CREDENTIAL_OPERATIONS = [ ), ( "video_remix_handler", - lambda: videos_main.video_remix( - video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS - ), + lambda: videos_main.video_remix(video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS), ), ( "video_edit_handler", - lambda: videos_main.video_edit( - video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS - ), + lambda: videos_main.video_edit(video_id=AZURE_VIDEO_ID, prompt="p", **DB_YAML_CREDS), ), ( "video_extension_handler", - lambda: videos_main.video_extension( - video_id=AZURE_VIDEO_ID, prompt="p", seconds="5", **DB_YAML_CREDS - ), + lambda: videos_main.video_extension(video_id=AZURE_VIDEO_ID, prompt="p", seconds="5", **DB_YAML_CREDS), ), ( "video_list_handler", @@ -429,9 +411,7 @@ CREDENTIAL_OPERATIONS = [ ), ( "video_create_character_handler", - lambda: videos_main.video_create_character( - name="hero", video=MagicMock(name="vid"), **DB_YAML_CREDS - ), + lambda: videos_main.video_create_character(name="hero", video=MagicMock(name="vid"), **DB_YAML_CREDS), ), ( "video_get_character_handler", @@ -453,3 +433,33 @@ def test_db_yaml_credentials_reach_every_handler(seams, handler_name, invoke): assert litellm_params.get("api_base") == DB_YAML_CREDS["api_base"] assert litellm_params.get("api_version") == DB_YAML_CREDS["api_version"] assert litellm_params.get("vertex_project") == DB_YAML_CREDS["vertex_project"] + + +def test_video_generation__custom_provider_map_base_video_config_is_dispatched(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.openai.videos.transformation import OpenAIVideoConfig + + class QueueVideoConfig(OpenAIVideoConfig): + pass + + video_config: Final = QueueVideoConfig() + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "queue-video", "custom_handler": video_config}]) + monkeypatch.setattr(litellm, "_custom_providers", ["queue-video"]) + handler: Final = MagicMock(spec=BaseLLMHTTPHandler) + with patch.object(videos_main, "base_llm_http_handler", handler): + videos_main.video_generation(model="queue-video/seedance-1", prompt="a cat", api_base="http://backend.test") + kw: Final = handler.video_generation_handler.call_args.kwargs + assert kw["video_generation_provider_config"] is video_config + assert kw["custom_llm_provider"] == "queue-video" + assert kw["model"] == "seedance-1" + + +def test_video_generation__custom_provider_without_a_video_config_is_a_400(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.custom_llm import CustomLLM + + monkeypatch.setattr(litellm, "custom_provider_map", [{"provider": "image-only", "custom_handler": CustomLLM()}]) + monkeypatch.setattr(litellm, "_custom_providers", ["image-only"]) + handler: Final = MagicMock(spec=BaseLLMHTTPHandler) + with patch.object(videos_main, "base_llm_http_handler", handler): + with pytest.raises(litellm.BadRequestError, match="video generation is not supported for image-only"): + videos_main.video_generation(model="image-only/seedance-1", prompt="a cat") + handler.video_generation_handler.assert_not_called() From 3da6029977d9e90953a250399911117a603592be Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:10:00 +0000 Subject: [PATCH 10/34] test(bedrock): assert native structured output when the model rejects forced tools (#49799) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_vertex_anthropic_structured_output_wire.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py index 4e31122440d..db81ff70263 100644 --- a/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py +++ b/tests/integration/providers/test_vertex_anthropic_structured_output_wire.py @@ -1488,7 +1488,7 @@ def test_vertex_chat_json_schema_on_a_model_rejecting_forced_tools_maps_reasonin assert _reasoning_fields(structured_sent) == _reasoning_fields(plain_sent), (structured_sent.body, plain_sent.body) -def test_bedrock_invoke_json_schema_sends_the_json_tool_unforced_when_the_model_rejects_forced_tools( +def test_bedrock_invoke_json_schema_sends_native_structured_output_when_the_model_rejects_forced_tools( gateway: Gateway, ) -> None: marker, marker_hex = _marker("plain") @@ -1499,7 +1499,8 @@ def test_bedrock_invoke_json_schema_sends_the_json_tool_unforced_when_the_model_ assert _chat_content(response) == _payload(marker_hex), response.text sent: Final = _one_delivery(wire, marker_hex) assert sent.target == f"/model/{backend}/invoke", sent.target - assert _json_tool_schema(sent)["properties"] == _SCHEMA["properties"], sent.body + assert object_value(_folded_format(sent)["schema"])["properties"] == _SCHEMA["properties"], sent.body + assert "tools" not in sent.body, sent.body assert "tool_choice" not in sent.body, sent.body From 408773bb69c1fc975ef9c936bfbdc8334e2bfe4e Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:10:08 +0000 Subject: [PATCH 11/34] test(spend): assert Anthropic envelope for Messages budget refusals (#49800) Co-authored-by: mateo <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../spend/test_partial_stream_live_budget.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/spend/test_partial_stream_live_budget.py b/tests/integration/spend/test_partial_stream_live_budget.py index 89fb66c9bbf..cba51873e2d 100644 --- a/tests/integration/spend/test_partial_stream_live_budget.py +++ b/tests/integration/spend/test_partial_stream_live_budget.py @@ -346,5 +346,15 @@ def test_a_broken_messages_stream_counts_toward_the_budget(gateway: Gateway) -> _broken_messages_stream(gateway, model, key) after: Final = _messages(gateway, model, key, streamed=False) - assert_budget_refusal(after) + assert after.status_code == 422, after.text + body: Final = JSON_OBJECT.validate_json(after.content) + error: Final = JSON_OBJECT.validate_python(body["error"]) + assert body == { + "type": "error", + "error": { + "type": "invalid_request_error", + "message": error["message"], + "litellm_error_type": "budget_exceeded", + }, + }, after.text assert _rows(gateway, key, 2) == (metered, StatusRow(status="failure", spend=0.0)) From b777dd6f14b84090ee537fe745de778f51dffd7c Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:12:42 +0000 Subject: [PATCH 12/34] fix(router)!: scope context fallback precedence to the requested group (#49779) * fix(router): scope context fallback precedence to the requested group * test(router): keep existing fallback test formatting * fix(router): preserve context errors after ordered backup failures --- litellm/router_python_core.py | 52 +++--- tests/unit/test_router/core/test_fallbacks.py | 19 +-- .../unit/test_router/test_router_fallbacks.py | 153 ++++++++++++++++++ 3 files changed, 188 insertions(+), 36 deletions(-) diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 2235abc9897..ee67d82abfa 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -9152,11 +9152,17 @@ class PythonCore(DeprecatedRouterMembers): if include_fallback_errors: input_kwargs["include_fallback_errors"] = True - # ORDER-BASED FALLBACKS: prepend higher order levels to the fallback list - # Skip for error types that have their own dedicated fallback handlers - _skip_order_fallback: Final = isinstance( - e, - (litellm_exceptions.ContextWindowExceededError, litellm_exceptions.ContentPolicyViolationError), + context_window_fallback_model_group: Final = ( + self._get_fallback_model_group_for_lookup_groups( + fallbacks=context_window_fallbacks, + lookup_groups=lookup_groups, + ) + if isinstance(e, litellm_exceptions.ContextWindowExceededError) and context_window_fallbacks is not None + else None + ) + _skip_order_fallback: Final = ( + isinstance(e, litellm_exceptions.ContentPolicyViolationError) + or context_window_fallback_model_group is not None ) _request_team_id: Final[str | None] = (kwargs.get("metadata", {}) or {}).get("user_api_key_team_id") # Use wildcard-aware lookup so order-based fallback also works for model @@ -9202,14 +9208,23 @@ class PythonCore(DeprecatedRouterMembers): "original_model_group": original_model_group, } ) - response = await run_async_fallback( - *args, - **input_kwargs, - ) - return response + try: + return await run_async_fallback( + *args, + **input_kwargs, + ) + except Exception: + if isinstance(original_exception, litellm_exceptions.ContextWindowExceededError): + raise original_exception + raise # Weighted intra-group failover (simple-shuffle only); see _maybe_run_weighted_failover. - if self.enable_weighted_failover and not _skip_order_fallback and original_model_group is not None: + if ( + self.enable_weighted_failover + and not _skip_order_fallback + and not isinstance(e, litellm_exceptions.ContextWindowExceededError) + and original_model_group is not None + ): response = await self._maybe_run_weighted_failover( exception=e, original_model_group=original_model_group, @@ -9227,7 +9242,7 @@ class PythonCore(DeprecatedRouterMembers): # check if client-side fallbacks are used (e.g. fallbacks = ["gpt-3.5-turbo", "claude-3-haiku"] or fallbacks=[{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hey, how's it going?"}]}] is_non_standard_fallback_format: Final = check_non_standard_fallback_format(fallbacks=fallbacks) - if is_non_standard_fallback_format: + if is_non_standard_fallback_format and context_window_fallback_model_group is None: input_kwargs.update( { "fallback_model_group": fallbacks, @@ -9243,16 +9258,7 @@ class PythonCore(DeprecatedRouterMembers): return response if isinstance(e, litellm_exceptions.ContextWindowExceededError): - if context_window_fallbacks is not None: - context_window_fallback_model_group: Final[list[str] | None] = ( - self._get_fallback_model_group_for_lookup_groups( - fallbacks=context_window_fallbacks, - lookup_groups=lookup_groups, - ) - ) - if context_window_fallback_model_group is None: - raise original_exception - + if context_window_fallback_model_group is not None: input_kwargs.update( { "fallback_model_group": context_window_fallback_model_group, @@ -9266,7 +9272,7 @@ class PythonCore(DeprecatedRouterMembers): ) return response - else: + elif context_window_fallbacks is None: error_message = f"model={model_group}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}. fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks" verbose_router_logger.info( msg=f"Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \ diff --git a/tests/unit/test_router/core/test_fallbacks.py b/tests/unit/test_router/core/test_fallbacks.py index e4e5c3120c8..bc6c70543c8 100644 --- a/tests/unit/test_router/core/test_fallbacks.py +++ b/tests/unit/test_router/core/test_fallbacks.py @@ -1,6 +1,7 @@ """Cross-group fallbacks of a router core: where the chain comes from, how it is walked, and what is raised.""" from collections.abc import Callable, Mapping +from typing import Final import pytest @@ -355,21 +356,13 @@ async def test_an_error_specific_chain_wins_over_the_general_chain( @pytest.mark.asyncio -@pytest.mark.parametrize( - ("error", "field", "raised"), - [ - (context_window, "context_window_fallbacks", litellm.ContextWindowExceededError), - (content_policy, "content_policy_fallbacks", litellm.ContentPolicyViolationError), - ], - ids=["context-window", "content-policy"], -) -async def test_an_error_specific_chain_without_the_group_raises_without_trying_the_general_chain( - make_core: MakeCore, host: ScriptedHost, error: Callable[[], Exception], field: str, raised: type[Exception] +async def test_an_unmatched_content_policy_chain_does_not_try_the_general_chain( + make_core: MakeCore, host: ScriptedHost ) -> None: - core = build(make_core, fallbacks=[{"a": ["b"]}], **{field: [{"other": ["c"]}]}) - host.answer("openai/a", error()) + core: Final = build(make_core, fallbacks=[{"a": ["b"]}], content_policy_fallbacks=[{"other": ["c"]}]) + host.answer("openai/a", content_policy()) - with pytest.raises(raised): + with pytest.raises(litellm.ContentPolicyViolationError): await complete(core) assert host.deployments_called("acompletion") == ["a"] diff --git a/tests/unit/test_router/test_router_fallbacks.py b/tests/unit/test_router/test_router_fallbacks.py index bccd6290f44..b25d5c35892 100644 --- a/tests/unit/test_router/test_router_fallbacks.py +++ b/tests/unit/test_router/test_router_fallbacks.py @@ -1207,3 +1207,156 @@ def test_context_stream_fallback_preserves_other_errors_for_existing_continuatio message="provider unavailable", model="primary", llm_provider="openai", generated_content="already visible" ) assert _context_window_stream_fallback_error(error, "primary", "openai", True) is error + + +def _context_fallback_precedence_router(context_key: str | None, ordered: bool) -> Router: + deployments: Final = ( + ("primary", "primary", 1), + ("primary" if ordered else "unused", "ordered", 2), + ("generic", "generic", 1), + ("context", "context", 1), + ) + return Router( + model_list=[ + { + "model_name": group, + "litellm_params": { + "model": f"openai/{model}", + "api_key": "test-key", + "api_base": "https://fallback-migration.local/v1", + "order": order, + }, + } + for group, model, order in deployments + ], + fallbacks=[{"primary": ["generic"]}], + context_window_fallbacks=None if context_key is None else [{context_key: ["context"]}], + num_retries=0, + disable_cooldowns=True, + ) + + +@pytest.mark.parametrize("sync_mode", (False, True)) +@pytest.mark.parametrize( + ("context_key", "ordered", "flat_fallbacks", "expected_model"), + ( + (None, True, False, "ordered"), + ("unrelated", True, False, "ordered"), + ("unrelated", False, False, "generic"), + ("primary", True, False, "context"), + ("*", True, False, "context"), + ("primary", True, True, "context"), + ("primary", False, True, "context"), + ), +) +@pytest.mark.asyncio +async def test_context_fallback_precedence_is_specific_to_the_requested_group( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + context_key: str | None, + ordered: bool, + flat_fallbacks: bool, + expected_model: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _context_fallback_precedence_router(context_key, ordered) + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=( + httpx.Response( + 400, + json={ + "error": { + "message": "This model's maximum context length is 10 tokens", + "type": "invalid_request_error", + "code": "context_length_exceeded", + } + }, + ), + _fallback_chat_response(expected_model, f"served by {expected_model}"), + ) + ) + + response: Final = ( + router.completion( + model="primary", + messages=[{"role": "user", "content": "long prompt"}], + fallbacks=["generic"] if flat_fallbacks else [{"primary": ["generic"]}], + ) + if sync_mode + else await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "long prompt"}], + fallbacks=["generic"] if flat_fallbacks else [{"primary": ["generic"]}], + ) + ) + + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", expected_model) + assert response.choices[0].message.content == f"served by {expected_model}" + + +@pytest.mark.parametrize("matching_context", (False, True)) +@pytest.mark.asyncio +async def test_context_fallback_exhaustion_preserves_the_selected_chain( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + matching_context: bool, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _context_fallback_precedence_router("primary" if matching_context else "unrelated", True) + refusal: Final = httpx.Response( + 400, + json={"error": {"message": "This model's maximum context length is 10 tokens"}}, + ) + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=(refusal, refusal, _fallback_chat_response("generic", "generic answer")) + ) + + if matching_context: + with pytest.raises(litellm.ContextWindowExceededError): + await router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}]) + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "context") + return + + response: Final = await router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}]) + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "ordered", "generic") + assert response.choices[0].message.content == "generic answer" + + +@pytest.mark.parametrize("sync_mode", (False, True)) +@pytest.mark.parametrize("context_error", (False, True)) +@pytest.mark.parametrize("backup_status", (429, 503)) +@pytest.mark.asyncio +async def test_ordered_fallback_failures_preserve_the_original_context_error( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + sync_mode: bool, + context_error: bool, + backup_status: int, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _context_fallback_precedence_router("unrelated", True) + original_message: Final = ( + "This model's maximum context length is 10 tokens" if context_error else "primary unavailable" + ) + backup_error: Final = litellm.RateLimitError if backup_status == 429 else litellm.ServiceUnavailableError + expected_error: Final = litellm.ContextWindowExceededError if context_error else backup_error + route: Final = respx_mock.post("https://fallback-migration.local/v1/chat/completions").mock( + side_effect=( + httpx.Response(400 if context_error else 503, json={"error": {"message": original_message}}), + httpx.Response(backup_status, json={"error": {"message": "ordered deployment unavailable"}}), + httpx.Response(backup_status, json={"error": {"message": "generic deployment unavailable"}}), + ) + ) + + request: Final = ( + asyncio.to_thread(router.completion, model="primary", messages=[{"role": "user", "content": "long prompt"}]) + if sync_mode + else router.acompletion(model="primary", messages=[{"role": "user", "content": "long prompt"}]) + ) + with pytest.raises(expected_error) as raised: + await request + + assert tuple(json.loads(call.request.content)["model"] for call in route.calls) == ("primary", "ordered", "generic") + assert raised.value.status_code == (400 if context_error else backup_status) + assert (original_message if context_error else "generic deployment unavailable") in str(raised.value) From 4e007c6e8116ba59368e828abc4dc7565ce5519e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 06:13:56 +0000 Subject: [PATCH 13/34] docs: preserve author review when updating PR summaries (#49803) --- .github/PULL_REQUEST_TEMPLATE/general.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE/general.md b/.github/PULL_REQUEST_TEMPLATE/general.md index aafd968ee0f..4f1a18e925e 100644 --- a/.github/PULL_REQUEST_TEMPLATE/general.md +++ b/.github/PULL_REQUEST_TEMPLATE/general.md @@ -6,12 +6,13 @@ ## Handwritten 1-3 sentence description - From 4573fa1f312a06e3daed57fd096205e5b50386f4 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 22:29:37 -0800 Subject: [PATCH 14/34] fix(embeddings): honor the caller's timeout on Cohere sync and HuggingFace embedding requests (#48593) * fix(embeddings): honor the caller's timeout on Cohere sync and HuggingFace embedding requests Bedrock Cohere embeddings ran the sync request with the client default instead of the timeout passed to litellm.embedding, and HuggingFace embeddings dropped it on both the sync and async paths. Pass the timeout through to the HTTP request on each path. Fixes #47301 Fixes #47299 * fix(huggingface): pass the caller's timeout to the embedding model lookup too Without input_type the handler first fetches the model's pipeline tag from huggingface.co, and that GET still used the default timeout. Thread the caller's timeout through both lookup helpers and the transform step so every request the embedding makes honors it. Refs #47299 * fix(huggingface): keep the unreachable async lookup path untouched The async model lookup helper and the call_type="async" transform branch are never reached, since the async embedding path transforms through the sync lookup. Leave them as they were so the change only touches requests that actually run. --- litellm/llms/cohere/embed/handler.py | 4 +- litellm/llms/huggingface/embedding/handler.py | 22 ++++++--- litellm/main.py | 1 + .../embed/test_cohere_embedding_handler.py | 35 ++++++++++++++ .../test_huggingface_embedding_handler.py | 46 +++++++++++++++---- 5 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py diff --git a/litellm/llms/cohere/embed/handler.py b/litellm/llms/cohere/embed/handler.py index 2f60d94c08a..34fde444880 100644 --- a/litellm/llms/cohere/embed/handler.py +++ b/litellm/llms/cohere/embed/handler.py @@ -82,7 +82,7 @@ async def async_embedding( ) try: - response: Final = await client.post(api_base, headers=headers, data=json.dumps(data)) + response: Final = await client.post(api_base, headers=headers, data=json.dumps(data), timeout=timeout) except httpx.HTTPStatusError as e: ## LOGGING logging_obj.post_call( @@ -166,7 +166,7 @@ def embedding( if client is None or not isinstance(client, HTTPHandler): client = HTTPHandler(concurrent_limit=1) - response: Final = client.post(embed_url, headers=headers, data=json.dumps(data)) + response: Final = client.post(embed_url, headers=headers, data=json.dumps(data), timeout=timeout) return CohereEmbeddingConfig().transform_response( response=response, diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index a1d1a3f6a19..5d997babc6e 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -38,7 +38,9 @@ class _SupportsTokenEncode(Protocol): def encode(self, text: str) -> list[int]: ... -def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: str) -> object: +def get_hf_task_embedding_for_model( + model: str, task_type: str | None, api_base: str, timeout: float | httpx.Timeout | None = None +) -> object: if task_type is not None: if task_type in get_args(hf_tasks_embeddings): return task_type @@ -46,7 +48,7 @@ def get_hf_task_embedding_for_model(model: str, task_type: str | None, api_base: raise Exception(f"Invalid task_type={task_type}. Expected one of={hf_tasks_embeddings}") http_client: Final = HTTPHandler(concurrent_limit=1) - model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}") + model_info: Final = http_client.get(url=f"{api_base}/api/models/{model}", timeout=timeout) model_info_dict: Final = decoded_json_object(_DECODED_JSON.validate_python(model_info.json())) @@ -148,6 +150,7 @@ class HuggingFaceEmbedding(BaseLLM): call_type: Literal["sync", "async"], optional_params: dict, embed_url: str, + timeout: float | httpx.Timeout | None = None, ) -> dict: data: dict = {} @@ -165,7 +168,9 @@ class HuggingFaceEmbedding(BaseLLM): task_type: Final = optional_params.pop("input_type", None) if call_type == "sync": - hf_task: Final = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL) + hf_task: Final = get_hf_task_embedding_for_model( + model=model, task_type=task_type, api_base=HF_HUB_URL, timeout=timeout + ) elif call_type == "async": return self._async_transform_input(model=model, task_type=task_type, embed_url=embed_url, input=input) @@ -253,6 +258,7 @@ class HuggingFaceEmbedding(BaseLLM): call_type="sync", optional_params=optional_params, embed_url=api_base, + timeout=timeout, ) ## LOGGING @@ -269,9 +275,10 @@ class HuggingFaceEmbedding(BaseLLM): if client is None: client = get_async_httpx_client( llm_provider=litellm.LlmProviders.HUGGINGFACE, + params={"timeout": timeout}, ) - response: Final = await client.post(api_base, headers=headers, data=json.dumps(data)) + response: Final = await client.post(api_base, headers=headers, data=json.dumps(data), timeout=timeout) ## LOGGING logging_obj.post_call( @@ -321,7 +328,9 @@ class HuggingFaceEmbedding(BaseLLM): litellm_params=litellm_params, ) task_type: Final = optional_params.get("input_type", None) - task: Final = get_hf_task_embedding_for_model(model=model, task_type=task_type, api_base=HF_HUB_URL) + task: Final = get_hf_task_embedding_for_model( + model=model, task_type=task_type, api_base=HF_HUB_URL, timeout=timeout + ) # print_verbose(f"{model}, {task}") embed_url = "" if model.startswith(("http://", "https://")): @@ -359,6 +368,7 @@ class HuggingFaceEmbedding(BaseLLM): call_type="sync", optional_params=optional_params, embed_url=embed_url, + timeout=timeout, ) ## LOGGING @@ -374,7 +384,7 @@ class HuggingFaceEmbedding(BaseLLM): ## COMPLETION CALL if client is None or not isinstance(client, HTTPHandler): client = HTTPHandler(concurrent_limit=1) - response: Final = client.post(embed_url, headers=headers, data=json.dumps(data)) + response: Final = client.post(embed_url, headers=headers, data=json.dumps(data), timeout=timeout) ## LOGGING logging_obj.post_call( diff --git a/litellm/main.py b/litellm/main.py index b5d1398ecad..b9fd86274b4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7400,6 +7400,7 @@ def embedding( aembedding=aembedding, litellm_params=litellm_params_dict, headers=headers, + timeout=timeout, ) elif custom_llm_provider == "bedrock": if isinstance(input, str): diff --git a/tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py b/tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py new file mode 100644 index 00000000000..f5cb874c6ef --- /dev/null +++ b/tests/unit/llms/cohere/embed/test_cohere_embedding_handler.py @@ -0,0 +1,35 @@ +from typing import Final + +import httpx +import pytest +import respx + +import litellm + +BEDROCK_COHERE_INVOKE_URL: Final = ( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-english-v3/invoke" +) +COHERE_EMBED_RESPONSE: Final = {"embeddings": [[0.1, 0.2]], "meta": {"billed_units": {"input_tokens": 1}}} + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_bedrock_cohere_embedding_sends_the_caller_timeout( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(BEDROCK_COHERE_INVOKE_URL).mock( + return_value=httpx.Response(200, json=COHERE_EMBED_RESPONSE) + ) + request: Final = { + "model": "bedrock/cohere.embed-english-v3", + "input": ["hello"], + "timeout": 45, + "aws_region_name": "us-east-1", + "aws_access_key_id": "mockaws_access_key_id", + "aws_secret_access_key": "mockaws_secret_access_key", + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert route.calls.last.request.extensions["timeout"]["read"] == 45 + assert response.data[0]["embedding"] == [0.1, 0.2] diff --git a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py index ea1e04291ca..7040fbe5fcc 100644 --- a/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py +++ b/tests/unit/llms/huggingface/embedding/test_huggingface_embedding_handler.py @@ -1,12 +1,12 @@ import json from typing import Final -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch - -import litellm +import httpx import pytest import respx +import litellm MOCK_EMBEDDING_RESPONSE = [[0.1, 0.2, 0.3, 0.4, 0.5]] @@ -52,12 +52,10 @@ def mock_embedding_async_http_handler(): class TestHuggingFaceEmbedding: @pytest.fixture(autouse=True) def setup(self, mock_embedding_http_handler, mock_embedding_async_http_handler, mock_hf_config_fetch): - self.mock_get_task_patcher = patch( - "litellm.llms.huggingface.embedding.handler.get_hf_task_embedding_for_model" - ) + self.mock_get_task_patcher = patch("litellm.llms.huggingface.embedding.handler.get_hf_task_embedding_for_model") self.mock_get_task = self.mock_get_task_patcher.start() - def mock_get_task_side_effect(model, task_type, api_base): + def mock_get_task_side_effect(model, task_type, api_base, timeout=None): if task_type is not None: return task_type return "sentence-similarity" @@ -180,7 +178,9 @@ async def test_hf_embedding_sentence_sim_sends_one_similarity_request( monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setenv("HUGGINGFACE_API_KEY", "hf-test-token") with respx.mock(assert_all_called=False) as hf: - hf.get(url__regex=r"https://huggingface\.co/api/models/.*").respond(json={"pipeline_tag": "sentence-similarity"}) + hf.get(url__regex=r"https://huggingface\.co/api/models/.*").respond( + json={"pipeline_tag": "sentence-similarity"} + ) hf.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(json={"max_position_embeddings": 512}) inference: Final = hf.post( "https://router.huggingface.co/hf-inference/pipeline/sentence-similarity/sentence-transformers/TaylorAI/bge-micro-v2" @@ -197,3 +197,33 @@ async def test_hf_embedding_sentence_sim_sends_one_similarity_request( } assert inference.calls.last.request.headers["Authorization"] == "Bearer hf-test-token" assert response.data == [{"object": "embedding", "index": 0, "embedding": 0.7708950042724609}] + + +HF_MODEL_LOOKUP_URL: Final = "https://huggingface.co/api/models/BAAI/bge-m3" +HF_EMBED_URL: Final = "https://router.huggingface.co/hf-inference/pipeline/feature-extraction/BAAI/bge-m3" + + +@pytest.mark.respx(assert_all_called=False) +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_huggingface_embedding_sends_the_caller_timeout( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.get(url__regex=r"https://huggingface\.co/.*/config\.json").respond(json={"max_position_embeddings": 512}) + lookup: Final = respx_mock.get(HF_MODEL_LOOKUP_URL).mock( + return_value=httpx.Response(200, json={"pipeline_tag": "feature-extraction"}) + ) + route: Final = respx_mock.post(HF_EMBED_URL).mock(return_value=httpx.Response(200, json=MOCK_EMBEDDING_RESPONSE)) + request: Final = { + "model": "huggingface/BAAI/bge-m3", + "input": ["hello"], + "api_key": "hf_test", + "timeout": 45, + } + + response: Final = litellm.embedding(**request) if sync_mode else await litellm.aembedding(**request) + + assert all(call.request.extensions["timeout"]["read"] == 45 for call in lookup.calls) + assert lookup.called + assert route.calls.last.request.extensions["timeout"]["read"] == 45 + assert response.data[0]["embedding"] == MOCK_EMBEDDING_RESPONSE[0] From 0d697017b5f8a54b94a0346d45a647e1ad3a8b9f Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 2 Nov 2026 22:59:40 -0800 Subject: [PATCH 15/34] fix(cost-map)!: price Mistral Large 4 at its current list price (#49814) Mistral lowered Large 4 to $0.70 in, $0.07 cached in and $2.10 out per million tokens, so both rows in both cost map files move to that price and point their source at the model page that lists it Breaking change: Large 4 calls are logged at about half the old cost, so budgets last longer and cost-based routing may favor it --- .../model_prices_and_context_window_backup.json | 16 ++++++++-------- model_prices_and_context_window.json | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 75364a382f7..376cd816c66 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43106,18 +43106,18 @@ "mode": "embedding" }, "mistral/mistral-large-4": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -43127,18 +43127,18 @@ "supports_vision": true }, "mistral/mistral-large-4-0": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 75364a382f7..376cd816c66 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43106,18 +43106,18 @@ "mode": "embedding" }, "mistral/mistral-large-4": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -43127,18 +43127,18 @@ "supports_vision": true }, "mistral/mistral-large-4-0": { - "cache_read_input_token_cost": 1.4e-07, - "input_cost_per_token": 1.36e-06, + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 7e-07, "litellm_provider": "mistral", "max_input_tokens": 1048576, "max_output_tokens": 1048576, "mode": "chat", - "output_cost_per_token": 4.18e-06, + "output_cost_per_token": 2.1e-06, "reasoning_effort_levels": [ "none", "high" ], - "source": "https://docs.mistral.ai/models/mistral-large-4", + "source": "https://docs.mistral.ai/models/mistral-large-4-0", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_prompt_caching": true, From f4a2e2b3af95c2628715ff46aedf41fe3ab3e255 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:05:15 -0800 Subject: [PATCH 16/34] fix(anthropic): send an https PDF image_url to Claude as a document (#49708) * fix(anthropic): send an https PDF image_url to Claude as a document A PDF link sent as image_url to Claude on Anthropic went out as an image block and Anthropic answered 400. A link whose path ends in .pdf is now downloaded like Vertex links are, and the served Content-Type picks the document or image block, so CDN links that serve a PNG from a .pdf path keep working. An explicit format of application/pdf sends a url document with no download. PDF media types match regardless of case or parameters, and count_tokens picks the same block as the request. * fix(anthropic): download https PDF links off the event loop The proxy downloaded an https link ending in .pdf synchronously while building the Anthropic request, so on an async call every other request on that worker waited for the download. Those links are now inlined before the request is built, the same way http links already are, and the /v1/messages guardrail write-back does the same. A non-string format on an image_url is ignored the way it already was inside the image_url mapping, instead of raising an AttributeError. * fix(anthropic): send a PDF link the proxy cannot download as a link and keep guardrail url blocks as sent * test(anthropic): expect the PNG media type the served bytes carry at a .jpg path * fix(anthropic): bound the PDF link download and keep query-string PDF names Give up on a .pdf link after NON_LLM_CONNECTION_TIMEOUT instead of the client's 6000 s default, fall back to main's whole-URL extension check so download?file=report.pdf still counts as a PDF, treat application/x-pdf as a PDF, and set the url document's application/pdf format only for the guardrail write-back so other /v1/messages providers send what main sends * fix(anthropic): probe a .pdf link and send it to Claude as a url document Instead of downloading a .pdf link and inlining it as base64, ask the host for its first kilobyte and send the link as a url document when it serves a PDF. Every other link goes out as the url image main sends, so robots.txt handling, caching and the wire format stay as they are on main. Also restore the guardrail adapter and the content type inference to main, keep the client's url document blocks (title, context, citations, cache_control) through the guardrail write-back, treat a PDF filename in the query string as a PDF link, probe links concurrently on the sync path, cap the async probe's read at the download limit, and catch only transport, URL and SSRF failures in the probe * fix(anthropic): read only the first kilobyte when probing a PDF link, and restore each url document from its own place The probe now streams the response and keeps the first 1 KB, so a host that ignores Range no longer fills memory with up to 20 parallel bodies. The guardrail write-back now matches url documents inside a tool_result and outside one separately, since the write-back moves tool results ahead of other content * fix(anthropic): remember each PDF link probe's verdict and cap its total time, and skip the probe on Bedrock invoke * fix(anthropic): keep a PDF probe's error status for a minute, not ten --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../prompt_templates/factory.py | 48 +-- .../prompt_templates/image_handling.py | 263 ++++++++++++++- .../chat/guardrail_translation/handler.py | 6 +- litellm/llms/anthropic/chat/transformation.py | 15 +- .../anthropic/count_tokens/transformation.py | 7 +- .../anthropic/messages/convertible_blocks.py | 70 ++++ .../anthropic_claude_transformation.py | 5 + tests/unit/conftest.py | 1 + ...llm_core_utils_prompt_templates_factory.py | 65 ++++ .../litellm_core_utils/test_image_handling.py | 305 ++++++++++++++++++ .../chat/test_anthropic_chat_handler.py | 190 ++++++++++- ...t_anthropic_count_tokens_transformation.py | 22 +- ...mations_anthropic_claude_transformation.py | 60 +++- .../test_structured_messages_writeback.py | 75 +++++ 14 files changed, 1078 insertions(+), 54 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index d6d753635f6..b3c8833811f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -77,7 +77,7 @@ from .common_utils import ( parse_tool_call_arguments, salvage_concatenated_tool_arguments, ) -from .image_handling import convert_url_to_base64 +from .image_handling import convert_url_to_base64, is_pdf_media_type def default_pt(messages): @@ -911,8 +911,17 @@ def convert_to_anthropic_image_obj(openai_image_url: str, format: str | None) -> ) +def _anthropic_https_url_block( + image_url: str, image_format: str | None +) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: + source: Final = AnthropicContentParamSourceUrl(type="url", url=image_url) + if image_format and is_pdf_media_type(image_format): + return AnthropicMessagesDocumentParam(type="document", source=source) + return AnthropicMessagesImageParam(type="image", source=source) + + def create_anthropic_image_param( - image_url_input: str | dict, + image_url_input: str | Mapping[str, object], format: str | None = None, is_bedrock_invoke: bool = False, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: @@ -921,13 +930,11 @@ def create_anthropic_image_param( Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding. """ - # Extract URL and format from input - if isinstance(image_url_input, str): - image_url = image_url_input - else: - image_url = image_url_input.get("url", "") - if format is None: - format = image_url_input.get("format") + url_value: Final = image_url_input if isinstance(image_url_input, str) else image_url_input.get("url") + format_value: Final = None if isinstance(image_url_input, str) else image_url_input.get("format") + image_url: Final = url_value if isinstance(url_value, str) else "" + explicit_format: Final = format if format is not None else format_value + image_format: Final = explicit_format if isinstance(explicit_format, str) else None # Check if the image URL is an HTTP/HTTPS URL if image_url.startswith("http://") or image_url.startswith("https://"): @@ -935,21 +942,14 @@ def create_anthropic_image_param( # as these providers don't support URL sources for images if is_bedrock_invoke or image_url.startswith("http://"): base64_url: Final = convert_url_to_base64(url=image_url) - image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=format) - return _anthropic_content_element_factory(image_chunk) + image_chunk = convert_to_anthropic_image_obj(openai_image_url=base64_url, format=image_format) + return anthropic_content_element_factory(image_chunk) else: - # HTTPS URL - pass directly for regular Anthropic - return AnthropicMessagesImageParam( - type="image", - source=AnthropicContentParamSourceUrl( - type="url", - url=image_url, - ), - ) + return _anthropic_https_url_block(image_url, image_format) else: # Convert to base64 for data URIs or other formats - image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=format) - return _anthropic_content_element_factory(image_chunk) + image_chunk = convert_to_anthropic_image_obj(openai_image_url=image_url, format=image_format) + return anthropic_content_element_factory(image_chunk) # The following XML functions will be deprecated once JSON schema support is available on Bedrock and Vertex @@ -2255,16 +2255,16 @@ def add_cache_control_to_content( return anthropic_content_element -def _anthropic_content_element_factory( +def anthropic_content_element_factory( image_chunk: GenericImageParsingChunk, ) -> AnthropicMessagesImageParam | AnthropicMessagesDocumentParam: - if image_chunk["media_type"] == "application/pdf": + if is_pdf_media_type(image_chunk["media_type"]): _anthropic_content_element: AnthropicMessagesDocumentParam | AnthropicMessagesImageParam = ( AnthropicMessagesDocumentParam( type="document", source=AnthropicContentParamSource( type="base64", - media_type=image_chunk["media_type"], + media_type="application/pdf", data=image_chunk["data"], ), ) diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py index 3287515c81d..ffc0e82e410 100644 --- a/litellm/litellm_core_utils/prompt_templates/image_handling.py +++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py @@ -5,28 +5,33 @@ Helper functions to handle images passed in messages import asyncio import base64 import time -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from itertools import chain +from functools import partial +from itertools import accumulate, chain, takewhile, tee from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar +from urllib.parse import urlparse import httpx -from httpx import Response +from httpx import USE_CLIENT_DEFAULT, Response import litellm from litellm import verbose_logger from litellm.caching.caching import InMemoryCache -from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB +from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, NON_LLM_CONNECTION_TIMEOUT from litellm.litellm_core_utils.prompt_templates.common_utils import infer_content_type_from_url_and_content from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get from litellm.types.llms.openai import AllMessageValues MAX_IMGS_IN_MEMORY: Final = 10 MAX_CONCURRENT_REMOTE_MEDIA_FETCHES: Final = 20 +_Fetched = TypeVar("_Fetched") _FETCH_ATTEMPTS: Final = 3 in_memory_cache: Final = InMemoryCache(max_size_in_memory=MAX_IMGS_IN_MEMORY) +pdf_probe_verdicts: Final = InMemoryCache() def _header_value(response: Response, name: str) -> str | None: @@ -209,6 +214,173 @@ def inline_remote_image_urls(media: RemoteMedia) -> bool: return media.part_type == "image_url" +_PDF_MEDIA_TYPES: Final = frozenset({"application/pdf", "application/x-pdf"}) +_GENERIC_MEDIA_TYPES: Final = frozenset({"application/octet-stream", "binary/octet-stream"}) +_PDF_SIGNATURE: Final = b"%PDF-" +_PDF_SIGNATURE_WINDOW: Final = 1024 +_PDF_PROBE_HEADERS: Final = MappingProxyType({"Range": f"bytes=0-{_PDF_SIGNATURE_WINDOW - 1}"}) +_PDF_PROBE_STATUSES: Final = frozenset({200, 206}) +_PDF_PROBE_FAILURES: Final = (httpx.HTTPError, httpx.InvalidURL, ValueError) +_ASYNC_PDF_PROBE_FAILURES: Final = (*_PDF_PROBE_FAILURES, asyncio.TimeoutError) +_PDF_PROBE_VERDICT_TTL: Final = 600 +_PDF_PROBE_FAILURE_TTL: Final = 60 +_DECODED_BODY_HEADERS: Final = frozenset({"content-encoding", "content-length", "transfer-encoding"}) + + +def _bare_media_type(media_type: str) -> str: + return media_type.split(";", 1)[0].strip().lower() + + +def is_pdf_media_type(media_type: str) -> bool: + return _bare_media_type(media_type) in _PDF_MEDIA_TYPES + + +def is_unformatted_pdf_link(url: str, image_format: str | None) -> bool: + lowered: Final = url.lower() + return not image_format and (urlparse(lowered).path.endswith(".pdf") or lowered.endswith(".pdf")) + + +def is_unformatted_pdf_link_image(media: RemoteMedia) -> bool: + image_format: Final = media.fields.get("format") + return inline_remote_image_urls(media) and is_unformatted_pdf_link( + media.url, image_format if isinstance(image_format, str) else None + ) + + +def _serves_pdf(response: Response) -> bool: + if response.status_code not in _PDF_PROBE_STATUSES: + return False + served_type: Final = _bare_media_type(_header_value(response, "Content-Type") or "") + if served_type and served_type not in _GENERIC_MEDIA_TYPES: + return served_type in _PDF_MEDIA_TYPES + return _PDF_SIGNATURE in response.content[:_PDF_SIGNATURE_WINDOW] + + +def _signature_window_chunks(chunks: Iterable[bytes], deadline: float) -> Iterator[bytes]: + reading, counting = tee(chunks) + read_so_far: Final = chain((0,), accumulate(map(len, counting))) + still_reading: Final = takewhile( + lambda total: total < _PDF_SIGNATURE_WINDOW and time.monotonic() < deadline, read_so_far + ) + return (chunk for _, chunk in zip(still_reading, reading)) + + +def _with_body_prefix(response: Response, prefix: bytes) -> Response: + return Response( + response.status_code, + headers=[(name, value) for name, value in response.headers.multi_items() if name not in _DECODED_BODY_HEADERS], + content=prefix, + request=response.request, + ) + + +@dataclass(frozen=True, slots=True) +class _BodyPrefixReader: + client: httpx.Client + deadline: float + + def get( + self, + url: str, + *, + headers: Mapping[str, str] | None = None, + follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, + ) -> Response: + with self.client.stream( + "GET", + url, + headers=headers, + follow_redirects=follow_redirects, + timeout=USE_CLIENT_DEFAULT if timeout is None else timeout, + ) as response: + prefix: Final = b"".join(_signature_window_chunks(response.iter_bytes(), self.deadline)) + return _with_body_prefix(response, prefix[:_PDF_SIGNATURE_WINDOW]) + + +@dataclass(frozen=True, slots=True) +class _AsyncBodyPrefixReader: + client: httpx.AsyncClient + + async def get( + self, + url: str, + *, + headers: Mapping[str, str] | None = None, + follow_redirects: bool = False, + timeout: float | httpx.Timeout | None = None, + max_response_bytes: int | None = None, + ) -> Response: + async with self.client.stream( + "GET", + url, + headers=headers, + follow_redirects=follow_redirects, + timeout=USE_CLIENT_DEFAULT if timeout is None else timeout, + ) as response: + return _with_body_prefix(response, await anext(response.aiter_bytes(max_response_bytes), b"")) + + +def _cached_pdf_verdict(url: str) -> bool | None: + cached: Final = pdf_probe_verdicts.get_cache(url) + return cached if isinstance(cached, bool) else None + + +def _remember_pdf_verdict(url: str, verdict: bool, ttl: int) -> bool: + pdf_probe_verdicts.set_cache(url, verdict, ttl=ttl) + return verdict + + +def _remember_probe_response(url: str, response: Response) -> bool: + ttl: Final = _PDF_PROBE_VERDICT_TTL if response.status_code in _PDF_PROBE_STATUSES else _PDF_PROBE_FAILURE_TTL + return _remember_pdf_verdict(url, _serves_pdf(response), ttl) + + +def _failed_pdf_probe(url: str, error: Exception) -> bool: + verbose_logger.debug("PDF probe of %s failed, sending it as an image link: %s", url, error) + return _remember_pdf_verdict(url, False, _PDF_PROBE_FAILURE_TTL) + + +def serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool: + if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0: + return False + cached: Final = _cached_pdf_verdict(url) + if cached is not None: + return cached + try: + response: Final = safe_get( + _BodyPrefixReader(litellm.module_level_client.client, deadline=time.monotonic() + timeout), + url, + headers=dict(_PDF_PROBE_HEADERS), + timeout=timeout, + ) + except _PDF_PROBE_FAILURES as e: + return _failed_pdf_probe(url, e) + return _remember_probe_response(url, response) + + +async def async_serves_pdf(url: str, timeout: float = NON_LLM_CONNECTION_TIMEOUT) -> bool: + if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0: + return False + cached: Final = _cached_pdf_verdict(url) + if cached is not None: + return cached + try: + response: Final = await asyncio.wait_for( + async_safe_get( + _AsyncBodyPrefixReader(litellm.module_level_aclient.client), + url, + headers=dict(_PDF_PROBE_HEADERS), + timeout=timeout, + max_response_bytes=_PDF_SIGNATURE_WINDOW, + ), + timeout, + ) + except _ASYNC_PDF_PROBE_FAILURES as e: + return _failed_pdf_probe(url, e) + return _remember_probe_response(url, response) + + def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None: if fields.get("type") != "image_url": return None @@ -292,15 +464,40 @@ def _inline_part(part: object, data_urls: Mapping[str, str], should_inline: Call return _inline(remote, data_url) if data_url is not None else part -def _inline_message( - message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool] -) -> AllMessageValues: +def _with_parts(message: AllMessageValues, rewrite: Callable[[object], object]) -> AllMessageValues: parts: Final = _content_parts(message) if not parts: return message - inlined_parts: Final = [_inline_part(part, data_urls, should_inline) for part in parts] - inlined_message: Final = {**message, "content": inlined_parts} - return inlined_message # pyright: ignore[reportReturnType] # the same message with its remote parts inlined + rewritten: Final = {**message, "content": [rewrite(part) for part in parts]} + return rewritten # pyright: ignore[reportReturnType] # the same message with some of its parts rewritten + + +def _inline_message( + message: AllMessageValues, data_urls: Mapping[str, str], should_inline: Callable[[RemoteMedia], bool] +) -> AllMessageValues: + return _with_parts(message, partial(_inline_part, data_urls=data_urls, should_inline=should_inline)) + + +def _pdf_formatted_part(part: object, pdf_links: frozenset[str], should_probe: Callable[[RemoteMedia], bool]) -> object: + remote: Final = _parse_remote_part(part) + match remote: + case _RemoteImage(fields, image_url, url) if url in pdf_links and should_probe(_remote_media(remote)): + image_url_fields: Final = image_url if image_url is not None else _NO_FIELDS + return {**fields, "image_url": {**image_url_fields, "url": url, "format": "application/pdf"}} + case _: + return part + + +def _with_pdf_formats( + messages: Iterable[AllMessageValues], + probed: Iterable[tuple[str, bool]], + should_probe: Callable[[RemoteMedia], bool], +) -> tuple[AllMessageValues, ...]: + pdf_links: Final = frozenset(url for url, is_pdf in probed if is_pdf) + if not pdf_links: + return tuple(messages) + rewrite: Final = partial(_pdf_formatted_part, pdf_links=pdf_links, should_probe=should_probe) + return tuple(_with_parts(message, rewrite) for message in messages) async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float | httpx.Timeout | None) -> str: @@ -308,11 +505,12 @@ async def _fetch_data_url(url: str, in_flight: asyncio.Semaphore, timeout: float return await async_convert_url_to_base64(url, timeout=timeout) -async def async_convert_urls_to_base64( - remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None -) -> tuple[str, ...]: - in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) - fetches: Final = tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls) +async def _serves_pdf_within(url: str, in_flight: asyncio.Semaphore) -> bool: + async with in_flight: + return await async_serves_pdf(url) + + +async def _gather_or_cancel_all(fetches: tuple["asyncio.Task[_Fetched]", ...]) -> tuple[_Fetched, ...]: try: return tuple(await asyncio.gather(*fetches)) except BaseException: @@ -322,6 +520,15 @@ async def async_convert_urls_to_base64( raise +async def async_convert_urls_to_base64( + remote_urls: tuple[str, ...], timeout: float | httpx.Timeout | None = None +) -> tuple[str, ...]: + in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) + return await _gather_or_cancel_all( + tuple(asyncio.create_task(_fetch_data_url(url, in_flight, timeout)) for url in remote_urls) + ) + + def _remote_urls_to_inline( messages: Iterable[AllMessageValues], should_inline: Callable[[RemoteMedia], bool] ) -> tuple[str, ...]: @@ -351,3 +558,27 @@ async def async_inline_remote_media( data_urls: Final = await async_convert_urls_to_base64(remote_urls) inlined: Final = MappingProxyType(dict(zip(remote_urls, data_urls, strict=True))) return [_inline_message(message, inlined, should_inline) for message in messages] + + +def format_pdf_links( + messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool] +) -> tuple[AllMessageValues, ...]: + links: Final = _remote_urls_to_inline(messages, should_probe) + if not links: + return tuple(messages) + with ThreadPoolExecutor(max_workers=min(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, len(links))) as probes: + verdicts: Final = tuple(probes.map(serves_pdf, links)) + return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe) + + +async def async_format_pdf_links( + messages: Sequence[AllMessageValues], should_probe: Callable[[RemoteMedia], bool] +) -> tuple[AllMessageValues, ...]: + links: Final = _remote_urls_to_inline(messages, should_probe) + if not links: + return tuple(messages) + in_flight: Final = asyncio.Semaphore(MAX_CONCURRENT_REMOTE_MEDIA_FETCHES) + verdicts: Final = await _gather_or_cancel_all( + tuple(asyncio.create_task(_serves_pdf_within(url, in_flight)) for url in links) + ) + return _with_pdf_formats(messages, zip(links, verdicts, strict=True), should_probe) diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py index 7f688ae733b..7e3fda06b1a 100644 --- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py +++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py @@ -34,6 +34,7 @@ from litellm.llms.anthropic.messages.adapters.transformation import ( ) from litellm.llms.anthropic.messages.convertible_blocks import ( keep_non_string_tool_result_texts, + keep_url_documents, scannable_tool_result_request, ) from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details @@ -1187,7 +1188,10 @@ class AnthropicMessagesHandler(BaseTranslation): cast(Sequence[object], converted), # cast-ok: Anthropic message payload original_tool_names, ) - data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages")) + original_messages: Final = data.get("messages") + data["messages"] = keep_url_documents( + keep_non_string_tool_result_texts(converted, original_messages), original_messages + ) @staticmethod def _extract_midturn_system_text( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2e908e66626..1977e3e2a7d 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -27,6 +27,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, ) +from litellm.litellm_core_utils.asyncify import is_event_loop_running from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( DEFS_MAX_INLINED_BYTES, @@ -47,8 +48,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( RemoteMedia, + async_format_pdf_links, async_inline_remote_media, + format_pdf_links, inline_remote_image_urls, + is_unformatted_pdf_link_image, ) from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( message_field, @@ -2333,6 +2337,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def inlines_remote_media(self, media: RemoteMedia) -> bool: return inline_remote_image_urls(media) and media.url.startswith("http://") + def probes_pdf_link(self, media: RemoteMedia) -> bool: + return is_unformatted_pdf_link_image(media) and not self.inlines_remote_media(media) + + def _with_pdf_links_formatted(self, messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + return messages if is_event_loop_running() else format_pdf_links(messages, should_probe=self.probes_pdf_link) + async def async_transform_request( self, model: str, @@ -2341,9 +2351,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): litellm_params: dict[str, object], # mutable-ok: BaseConfig signature headers: dict[str, object], # mutable-ok: BaseConfig signature ) -> dict[str, object]: # mutable-ok: BaseConfig signature + inlined: Final = await async_inline_remote_media(messages, should_inline=self.inlines_remote_media) return self.transform_request( model=model, - messages=await async_inline_remote_media(messages, should_inline=self.inlines_remote_media), + messages=list(await async_format_pdf_links(inlined, should_probe=self.probes_pdf_link)), optional_params=optional_params, litellm_params=litellm_params, headers=headers, @@ -2477,7 +2488,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): try: anthropic_messages = anthropic_messages_pt( model=model, - messages=list(conversation), + messages=list(self._with_pdf_links_formatted(conversation)), llm_provider=self._resolved_provider, drop_params=drop_params, ) diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 1c8f0c206cd..3034927c612 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -49,7 +49,10 @@ _PLAIN_MESSAGE_KEYS: Final = frozenset({"role", "content"}) def _count_image(block: JsonValue) -> JsonValue: if not isinstance(block, dict) or block.get("type") != "image_url": return block - from litellm.litellm_core_utils.prompt_templates.factory import convert_to_anthropic_image_obj + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_content_element_factory, + convert_to_anthropic_image_obj, + ) image_block: Final = _IMAGE_BLOCK.validate_python(block) image_url: Final = image_block["image_url"] @@ -57,7 +60,7 @@ def _count_image(block: JsonValue) -> JsonValue: openai_image_url=image_url if isinstance(image_url, str) else image_url["url"], format=image_url.get("format") if isinstance(image_url, dict) else None, ) - image: Final = _COUNT_REQUEST.validate_python({"type": "image", "source": source}) + image: Final = _COUNT_REQUEST.validate_python(anthropic_content_element_factory(source)) return {**{key: value for key, value in block.items() if key not in {"type", "image_url"}}, **image} diff --git a/litellm/llms/anthropic/messages/convertible_blocks.py b/litellm/llms/anthropic/messages/convertible_blocks.py index 09d662da543..b98b4cf3f75 100644 --- a/litellm/llms/anthropic/messages/convertible_blocks.py +++ b/litellm/llms/anthropic/messages/convertible_blocks.py @@ -314,6 +314,76 @@ def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: obj return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten] +def _url_source_url(block: object) -> str | None: + match as_str_mapping(block): + case {"type": "image" | "document", "source": {"type": "url", "url": str(url)}}: + return url + case _: + return None + + +def _is_document(block: object) -> bool: + match as_str_mapping(block): + case {"type": "document"}: + return True + case _: + return False + + +@dataclass(frozen=True, slots=True, order=True) +class _UrlBlockPlace: + in_tool_result: bool + url: str + + +def _url_sourced_blocks(block: object, in_tool_result: bool = False) -> tuple[tuple[_UrlBlockPlace, object], ...]: + url: Final = _url_source_url(block) + if url is not None: + return ((_UrlBlockPlace(in_tool_result, url), block),) + match as_str_mapping(block): + case {"type": "tool_result", "content": list() as content}: + in_this_tool_result: Final = partial(_url_sourced_blocks, in_tool_result=True) + return tuple(chain.from_iterable(map(in_this_tool_result, _list_items(content)))) + case _: + return () + + +def _entry_place(entry: tuple[_UrlBlockPlace, object]) -> _UrlBlockPlace: + return entry[0] + + +def _entry_block(entry: tuple[_UrlBlockPlace, object]) -> object: + return entry[1] + + +def _restored_url_document( + block: object, originals: Mapping[_UrlBlockPlace, Iterator[object]], in_tool_result: bool = False +) -> object: + url: Final = _url_source_url(block) + if url is not None: + remaining: Final = originals.get(_UrlBlockPlace(in_tool_result, url)) + original: Final = None if remaining is None else next(remaining, None) + return original if _is_document(original) else block + match as_str_mapping(block): + case {"type": "tool_result", "content": list() as content} as tool_result: + restore: Final = partial(_restored_url_document, originals=originals, in_tool_result=True) + return {**tool_result, "content": list(map(restore, _list_items(content)))} + case _: + return block + + +def keep_url_documents(rewritten: Sequence[object], original: object) -> Sequence[object]: + blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original))) + by_place: Final = tuple(sorted(chain.from_iterable(map(_url_sourced_blocks, blocks)), key=_entry_place)) + if not any(_is_document(block) for _, block in by_place): + return rewritten + originals_in_order: Final = MappingProxyType( + {place: iter(tuple(map(_entry_block, group))) for place, group in groupby(by_place, key=_entry_place)} + ) + restore: Final = partial(_restored_url_document, originals=originals_in_order) + return [_with_user_blocks(message, restore) for message in rewritten] + + def _without_non_string_text(block: object) -> object: match as_str_mapping(block): case {"type": "tool_result", "content": list()} as tool_result: diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py index e2f12b123de..37acb5423ff 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude_transformation.py @@ -13,8 +13,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) from litellm.litellm_core_utils.prompt_templates.image_handling import ( + RemoteMedia, async_inline_remote_media, convert_url_to_base64, + inline_every_remote_url, ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.common_utils import ( @@ -117,6 +119,9 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def should_strip_billing_metadata(self) -> bool: return True + def inlines_remote_media(self, media: RemoteMedia) -> bool: + return inline_every_remote_url(media) + def get_supported_openai_params(self, model: str) -> list[str]: return AnthropicConfig.get_supported_openai_params(self, model) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ed7dd5ec146..914eb1f2508 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -216,6 +216,7 @@ def _reset_aws_auth_caches() -> None: def _flush_client_caches() -> None: litellm.in_memory_llm_clients_cache.flush_cache() image_handling_module.in_memory_cache.flush_cache() + image_handling_module.pdf_probe_verdicts.flush_cache() _reset_aws_auth_caches() diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8a755d47817..d65f094971a 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1298,6 +1298,71 @@ def test_anthropic_messages_pt_uses_pdf_format_for_generic_mime_data_uri(llm_pro } +@pytest.mark.parametrize( + ("image_url", "expected_type"), + ( + ({"url": "https://example.com/download?id=7", "format": "application/pdf"}, "document"), + ({"url": "https://example.com/download?id=7", "format": "Application/PDF; charset=binary"}, "document"), + ({"url": "https://example.com/files/report.pdf", "format": "image/png"}, "image"), + ({"url": "https://example.com/pdf/photo.png"}, "image"), + ({"url": "https://example.com/pdf/photo.png", "format": 5}, "image"), + ({"url": "https://example.com/pdf/photo.png", "format": ["application/pdf"]}, "image"), + ), +) +def test_anthropic_messages_pt_picks_block_type_for_https_image_url( + image_url: dict[str, object], expected_type: str +) -> None: + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": image_url}, + ], + } + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": expected_type, + "source": {"type": "url", "url": image_url["url"]}, + } + + +@pytest.mark.parametrize( + "image_url", + ( + {"url": "data:Application/PDF;base64,JVBERi0xLjQK"}, + {"url": "data:application/x-pdf;base64,JVBERi0xLjQK"}, + {"url": "data:application/octet-stream;base64,JVBERi0xLjQK", "format": "application/pdf; charset=binary"}, + ), +) +def test_anthropic_messages_pt_normalizes_a_pdf_media_type_with_case_or_parameters( + image_url: dict[str, str], +) -> None: + result: Final = anthropic_messages_pt( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "image_url", "image_url": image_url}, + ], + } + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) + + assert result[0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"}, + } + + def test_anthropic_messages_pt_keeps_png_image_url_as_image() -> None: image_data = "iVBORw0KGgo=" result: Final = anthropic_messages_pt( diff --git a/tests/unit/litellm_core_utils/test_image_handling.py b/tests/unit/litellm_core_utils/test_image_handling.py index d1bebde9917..2d4cd4b9a6d 100644 --- a/tests/unit/litellm_core_utils/test_image_handling.py +++ b/tests/unit/litellm_core_utils/test_image_handling.py @@ -1,23 +1,36 @@ import asyncio import base64 import copy +import threading import time import uuid +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final +from unittest.mock import MagicMock import pytest from httpx import Request, Response import litellm +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils import url_utils from litellm.litellm_core_utils.prompt_templates import image_handling from litellm.litellm_core_utils.prompt_templates.image_handling import ( MAX_CONCURRENT_REMOTE_MEDIA_FETCHES, RemoteMedia, async_convert_url_to_base64, + async_format_pdf_links, async_inline_remote_media, + async_serves_pdf, convert_url_to_base64, + format_pdf_links, inline_remote_media, + is_unformatted_pdf_link_image, + serves_pdf, ) from litellm.litellm_core_utils.url_utils import SSRFError +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @pytest.fixture(autouse=True) @@ -695,3 +708,295 @@ async def test_async_inline_remote_media_raises_image_fetch_error_when_the_fetch with pytest.raises(litellm.ImageFetchError, match="Status code: 404"): await async_inline_remote_media([{"role": "user", "content": [{"type": "image_url", "image_url": url}]}]) + + +PDF_BODY: Final = b"%PDF-1.7\n" + b"0" * 4096 +PNG_BODY: Final = b"\x89PNG\r\n\x1a\n" + b"0" * 64 + + +@dataclass(frozen=True, slots=True) +class _Served: + status: int = 200 + content_type: str | None = "application/octet-stream" + body: bytes = PDF_BODY + honors_range: bool = True + barrier: threading.Barrier | None = None + declared_length: int | None = None + hold_open_until: threading.Event | None = None + respond_after: threading.Event | None = None + drip_every: float | None = None + + +@dataclass(frozen=True, slots=True) +class _ProbeServer: + base_url: str + routes: dict[str, _Served] + seen: list[tuple[str, str | None]] + + def serve(self, path: str, served: _Served) -> str: + self.routes[path] = served + return f"{self.base_url}{path}" + + +@pytest.fixture +def probe_server(monkeypatch): + monkeypatch.setattr(litellm, "user_url_validation", False) + monkeypatch.setattr(image_handling, "safe_get", url_utils.safe_get) + monkeypatch.setattr(litellm, "module_level_client", HTTPHandler()) + monkeypatch.setattr(litellm, "module_level_aclient", AsyncHTTPHandler()) + routes: Final[dict[str, _Served]] = {} + seen: Final[list[tuple[str, str | None]]] = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + served = routes[self.path] + requested_range = self.headers.get("Range") + seen.append((self.path, requested_range)) + if served.barrier is not None: + served.barrier.wait() + if served.respond_after is not None: + served.respond_after.wait(timeout=30) + ranged = served.honors_range and requested_range == "bytes=0-1023" + body = served.body[:1024] if ranged else served.body + self.send_response(206 if ranged and served.status == 200 else served.status) + if served.content_type is not None: + self.send_header("Content-Type", served.content_type) + self.send_header("Content-Length", str(served.declared_length or len(body))) + self.end_headers() + if served.drip_every is None: + self.wfile.write(body) + else: + for byte in body: + self.wfile.write(bytes([byte])) + self.wfile.flush() + time.sleep(served.drip_every) + if served.hold_open_until is not None: + served.hold_open_until.wait(timeout=30) + + def log_message(self, *args): + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield _ProbeServer(f"http://127.0.0.1:{server.server_port}", routes, seen) + server.shutdown() + server.server_close() + + +async def _serves_pdf(mode: str, url: str, **kwargs) -> bool: + return serves_pdf(url, **kwargs) if mode == "sync" else await async_serves_pdf(url, **kwargs) + + +async def _format_pdf_links(mode, messages): + if mode == "sync": + return format_pdf_links(messages, should_probe=is_unformatted_pdf_link_image) + return await async_format_pdf_links(messages, should_probe=is_unformatted_pdf_link_image) + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_asks_for_only_the_leading_kilobyte(probe_server, mode): + url = probe_server.serve("/files/report.pdf", _Served()) + + assert await _serves_pdf(mode, url) is True + assert probe_server.seen == [("/files/report.pdf", "bytes=0-1023")] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize( + ("served", "expected"), + ( + (_Served(content_type="application/pdf", body=PNG_BODY), True), + (_Served(content_type="Application/X-PDF; charset=binary", body=b""), True), + (_Served(content_type="binary/octet-stream", body=PDF_BODY), True), + (_Served(content_type=None, body=PDF_BODY), True), + (_Served(content_type="application/octet-stream", body=b"\n" * 1000 + PDF_BODY), True), + (_Served(content_type="application/octet-stream", body=b"\n" * 1024 + PDF_BODY), False), + (_Served(content_type="binary/octet-stream", body=PNG_BODY), False), + (_Served(content_type="image/png", body=PDF_BODY), False), + (_Served(content_type="text/html", body=b"%PDF-"), False), + (_Served(status=404, content_type="application/pdf"), False), + (_Served(honors_range=False), True), + ), + ids=( + "served-pdf", + "served-x-pdf", + "generic-type-pdf-bytes", + "no-type-pdf-bytes", + "signature-inside-the-first-kilobyte", + "signature-past-the-first-kilobyte", + "generic-type-png-bytes", + "served-png", + "served-html", + "not-found", + "range-ignored", + ), +) +async def test_serves_pdf_trusts_a_specific_served_type_and_sniffs_a_generic_one(probe_server, mode, served, expected): + url = probe_server.serve(f"/{uuid.uuid4()}/report.pdf", served) + + assert await _serves_pdf(mode, url) is expected + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_makes_no_request_when_url_downloads_are_disabled(probe_server, monkeypatch, mode): + monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0) + url = probe_server.serve("/report.pdf", _Served(content_type="application/pdf")) + + assert await _serves_pdf(mode, url) is False + assert probe_server.seen == [] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_is_false_when_the_host_refuses_the_connection(probe_server, mode): + assert await _serves_pdf(mode, "http://127.0.0.1:1/report.pdf") is False + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_is_false_without_a_request_when_the_ssrf_check_blocks_the_link(probe_server, monkeypatch, mode): + monkeypatch.setattr(litellm, "user_url_validation", True) + url = probe_server.serve("/report.pdf", _Served(content_type="application/pdf")) + + assert await _serves_pdf(mode, url) is False + assert probe_server.seen == [] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_stops_reading_after_the_leading_kilobyte_when_the_host_ignores_range(probe_server, mode): + rest_of_body_sent = threading.Event() + url = probe_server.serve( + "/report.pdf", _Served(honors_range=False, declared_length=1024**3, hold_open_until=rest_of_body_sent) + ) + + try: + assert await _serves_pdf(mode, url) is True + finally: + rest_of_body_sent.set() + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_gives_up_within_its_timeout_when_the_host_sends_one_byte_at_a_time(probe_server, mode): + url = probe_server.serve("/report.pdf", _Served(body=b"0" * 1024, honors_range=False, drip_every=0.02)) + started = time.monotonic() + + assert await _serves_pdf(mode, url, timeout=0.5) is False + assert time.monotonic() - started < 3 + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_reuses_a_link_verdict_instead_of_probing_the_link_again(probe_server, mode): + pdf_link = probe_server.serve("/a/report.pdf", _Served()) + png_link = probe_server.serve("/a/page.pdf", _Served(content_type="image/png", body=PNG_BODY)) + + verdicts = [await _serves_pdf(mode, url) for url in (pdf_link, png_link, pdf_link, png_link)] + + assert verdicts == [True, False, True, False] + assert [path for path, _ in probe_server.seen] == ["/a/report.pdf", "/a/page.pdf"] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_serves_pdf_remembers_a_timed_out_probe_so_the_next_turn_does_not_wait_again(probe_server, mode): + host_answers = threading.Event() + url = probe_server.serve("/report.pdf", _Served(respond_after=host_answers)) + + try: + first = await _serves_pdf(mode, url, timeout=0.3) + started = time.monotonic() + second = await _serves_pdf(mode, url, timeout=0.3) + second_took = time.monotonic() - started + finally: + host_answers.set() + + assert (first, second) == (False, False) + assert second_took < 0.1 + assert probe_server.seen == [("/report.pdf", "bytes=0-1023")] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("status", (404, 503)) +async def test_serves_pdf_probes_a_link_again_a_minute_after_its_host_answered_an_error( + probe_server, monkeypatch, mode, status +): + clock: Final = MagicMock(return_value=1_000.0) + monkeypatch.setattr(image_handling, "pdf_probe_verdicts", InMemoryCache(clock=clock)) + url = probe_server.serve("/report.pdf", _Served(status=status)) + + while_down = await _serves_pdf(mode, url) + probe_server.serve("/report.pdf", _Served()) + clock.return_value = 1_030.0 + within_the_minute = await _serves_pdf(mode, url) + clock.return_value = 1_061.0 + after_the_minute = await _serves_pdf(mode, url) + + assert (while_down, within_the_minute, after_the_minute) == (False, False, True) + assert len(probe_server.seen) == 2 + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_format_pdf_links_marks_only_links_that_serve_a_pdf(probe_server, mode): + pdf_link = probe_server.serve("/a/report.pdf", _Served()) + query_link = probe_server.serve("/a/download?file=summary.pdf", _Served(content_type="application/pdf")) + png_at_pdf_link = probe_server.serve("/a/page.pdf?fm=png", _Served(content_type="image/png", body=PNG_BODY)) + png_link = f"{probe_server.base_url}/a/photo.png" + messages = [ + {"role": "system", "content": "read the files"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + {"type": "image_url", "image_url": {"url": pdf_link, "detail": "high"}, "cache_control": {"type": "ephemeral"}}, + {"type": "image_url", "image_url": query_link}, + {"type": "image_url", "image_url": {"url": png_at_pdf_link}}, + {"type": "image_url", "image_url": {"url": png_link}}, + {"type": "image_url", "image_url": {"url": pdf_link, "format": "image/png"}}, + ], + }, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link}}]}, + ] + original = copy.deepcopy(messages) + + formatted = await _format_pdf_links(mode, messages) + + assert messages == original + assert formatted == ( + messages[0], + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + { + "type": "image_url", + "image_url": {"url": pdf_link, "detail": "high", "format": "application/pdf"}, + "cache_control": {"type": "ephemeral"}, + }, + {"type": "image_url", "image_url": {"url": query_link, "format": "application/pdf"}}, + {"type": "image_url", "image_url": {"url": png_at_pdf_link}}, + {"type": "image_url", "image_url": {"url": png_link}}, + {"type": "image_url", "image_url": {"url": pdf_link, "format": "image/png"}}, + ], + }, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link, "format": "application/pdf"}}]}, + ) + assert sorted(path for path, _ in probe_server.seen) == ["/a/download?file=summary.pdf", "/a/page.pdf?fm=png", "/a/report.pdf"] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_format_pdf_links_probes_every_link_at_once(probe_server, mode): + both_in_flight = threading.Barrier(2, timeout=5) + links = [probe_server.serve(f"/{name}.pdf", _Served(barrier=both_in_flight)) for name in ("first", "second")] + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": link}} for link in links]}] + + formatted = await _format_pdf_links(mode, messages) + + assert [part["image_url"]["format"] for part in formatted[0]["content"]] == ["application/pdf", "application/pdf"] + + +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_format_pdf_links_leaves_messages_without_pdf_links_alone(probe_server, mode): + messages = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": f"{probe_server.base_url}/a.png"}}]}, + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:application/pdf;base64,JVBERi0="}}]}, + ] + + assert await _format_pdf_links(mode, messages) == tuple(messages) + assert probe_server.seen == [] diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py index ce14adab4a0..7e2786327dc 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -2,6 +2,7 @@ import json import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from itertools import chain +from collections.abc import Callable from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -13,7 +14,8 @@ from openai.types.chat import ChatCompletionChunk import litellm from litellm._uuid import uuid -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, RESPONSE_FORMAT_TOOL_NAME +from litellm.litellm_core_utils.prompt_templates import image_handling from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.anthropic.chat.handler import ModelResponseIterator, _envelope_verdict, make_call, make_sync_call from litellm.llms.base_llm.base_utils import response_format_schema, unwrap_response_format_envelope @@ -253,6 +255,7 @@ def test_anthropic_tool_call_error_is_mapped_from_the_provider_response(): async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch): http_image_url = f"http://img.example/{uuid.uuid4()}.png" https_image_url = f"https://img.example/{uuid.uuid4()}.png" + https_pdf_link = f"https://img.example/{uuid.uuid4()}/page.pdf?w=600" captured = {} def handle(request): @@ -282,6 +285,7 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop {"type": "text", "text": "What colour is this?"}, {"type": "image_url", "image_url": {"url": http_image_url}}, {"type": "image_url", "image_url": {"url": https_image_url}}, + {"type": "image_url", "image_url": {"url": https_pdf_link}}, ], } ], @@ -290,11 +294,193 @@ async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop ) assert response.choices[0].message.content == "Green" - assert async_only_image_fetch.fetched == [http_image_url] + assert async_only_image_fetch.fetched == [http_image_url, https_pdf_link] sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"] assert sources == [ {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png}, {"type": "url", "url": https_image_url}, + {"type": "url", "url": https_pdf_link}, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("download_limit_mb", (0, 50), ids=("downloads-disabled", "link-unreachable")) +async def test_anthropic_async_completion_sends_a_pdf_path_link_it_cannot_download_as_a_link( + monkeypatch: pytest.MonkeyPatch, download_limit_mb: int +) -> None: + pdf_link: Final = f"https://img.example/{uuid.uuid4()}/page.pdf?fm=png" + captured: Final[dict[str, object]] = {} + + async def unreachable(client: object, url: str, **kwargs: object) -> httpx.Response: + return httpx.Response(404, request=httpx.Request("GET", url)) + + def handle(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "A chart"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + monkeypatch.setattr(image_handling, "async_safe_get", unreachable) + monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", download_limit_mb) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + + response: Final = await litellm.acompletion( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link}}]}], + client=client, + api_key="sk-ant-unit-test", + ) + + assert response.choices[0].message.content == "A chart" + assert captured["body"]["messages"][0]["content"] == [{"type": "image", "source": {"type": "url", "url": pdf_link}}] + + +@pytest.mark.asyncio +async def test_anthropic_async_completion_gives_up_on_a_pdf_link_within_the_non_llm_connection_timeout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pdf_link: Final = f"https://unreachable.example/{uuid.uuid4()}/report.pdf" + fetch_timeouts: Final[list[object]] = [] + captured: Final[dict[str, object]] = {} + + async def unreachable(client: object, url: str, **kwargs: object) -> httpx.Response: + fetch_timeouts.append(kwargs.get("timeout")) + raise httpx.ConnectTimeout("no route to host") + + def handle(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "A chart"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + monkeypatch.setattr(image_handling, "async_safe_get", unreachable) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) + + await litellm.acompletion( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": pdf_link}}]}], + client=client, + api_key="sk-ant-unit-test", + ) + + assert captured["body"]["messages"][0]["content"] == [{"type": "image", "source": {"type": "url", "url": pdf_link}}] + assert fetch_timeouts == [NON_LLM_CONNECTION_TIMEOUT] + + +def _anthropic_reply(captured: dict[str, object]) -> Callable[[httpx.Request], httpx.Response]: + def handle(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "A report"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + return handle + + +def _served_pdf_slice(url: str) -> httpx.Response: + return httpx.Response( + 206, content=b"%PDF-1.7\n", headers={"content-type": "binary/octet-stream"}, request=httpx.Request("GET", url) + ) + + +@pytest.mark.asyncio +async def test_anthropic_async_completion_sends_a_link_that_serves_a_pdf_as_a_url_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pdf_link: Final = f"https://files.example/{uuid.uuid4()}/download?file=report.pdf" + probes: Final[list[object]] = [] + captured: Final[dict[str, object]] = {} + + async def serve_pdf_slice(client: object, url: str, **kwargs: object) -> httpx.Response: + probes.append(kwargs.get("headers")) + return _served_pdf_slice(url) + + monkeypatch.setattr(image_handling, "async_safe_get", serve_pdf_slice) + client: Final = AsyncHTTPHandler() + client.client = httpx.AsyncClient(transport=httpx.MockTransport(_anthropic_reply(captured))) + + await litellm.acompletion( + model="anthropic/claude-haiku-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": pdf_link}, "cache_control": {"type": "ephemeral"}} + ], + } + ], + client=client, + api_key="sk-ant-unit-test", + ) + + assert captured["body"]["messages"][0]["content"] == [ + {"type": "document", "source": {"type": "url", "url": pdf_link}, "cache_control": {"type": "ephemeral"}} + ] + assert probes == [{"Range": "bytes=0-1023"}] + + +def test_anthropic_sync_completion_sends_a_link_that_serves_a_pdf_as_a_url_document( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pdf_link: Final = f"https://files.example/{uuid.uuid4()}/report.pdf" + png_at_pdf_link: Final = f"https://files.example/{uuid.uuid4()}/page.pdf?fm=png" + captured: Final[dict[str, object]] = {} + + def serve(client: object, url: str, **kwargs: object) -> httpx.Response: + if url == pdf_link: + return _served_pdf_slice(url) + return httpx.Response(200, content=b"\x89PNG", headers={"content-type": "image/png"}, request=httpx.Request("GET", url)) + + monkeypatch.setattr(image_handling, "safe_get", serve) + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_anthropic_reply(captured)))) + + litellm.completion( + model="anthropic/claude-haiku-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": pdf_link}}, + {"type": "image_url", "image_url": {"url": png_at_pdf_link}}, + ], + } + ], + client=client, + api_key="sk-ant-unit-test", + ) + + assert captured["body"]["messages"][0]["content"] == [ + {"type": "document", "source": {"type": "url", "url": pdf_link}}, + {"type": "image", "source": {"type": "url", "url": png_at_pdf_link}}, ] diff --git a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py index aa7fa2727b4..13c17ebd2bf 100644 --- a/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/anthropic/test_anthropic_count_tokens_transformation.py @@ -50,20 +50,30 @@ COMPACTED_HISTORY = [ "config_type", (AnthropicCountTokensConfig, AzureAIAnthropicCountTokensConfig) ) @pytest.mark.parametrize( - ("image_url", "source"), + ("image_url", "block"), ( - ("data:image/png;base64,aW1hZ2U=", {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}), - ({"url": "data:image/png;base64,aW1hZ2U="}, {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}), + ( + "data:image/png;base64,aW1hZ2U=", + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}}, + ), + ( + {"url": "data:image/png;base64,aW1hZ2U="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aW1hZ2U="}}, + ), ( {"url": "data:image/png;base64,aW1hZ2U=", "format": "image/jpeg", "detail": "high"}, - {"type": "base64", "media_type": "image/jpeg", "data": "aW1hZ2U="}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "aW1hZ2U="}}, + ), + ( + "data:application/pdf;base64,JVBERi0=", + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="}}, ), ), ) def test_count_translates_openai_images_without_mutating_input( config_type: type[AnthropicCountTokensConfig], image_url: str | dict[str, JsonValue], - source: dict[str, JsonValue], + block: dict[str, JsonValue], ) -> None: cache_control: Final[dict[str, JsonValue]] = {"type": "ephemeral"} messages: Final[list[dict[str, JsonValue]]] = [{ @@ -81,7 +91,7 @@ def test_count_translates_openai_images_without_mutating_input( "model": "claude-opus-5-5", "messages": [{ "role": "user", "content": [ {"type": "text", "text": "Count this image"}, - {"type": "image", "source": source, "cache_control": cache_control}, + {**block, "cache_control": cache_control}, ], }], } diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py index 05d755b1215..34a4cf5225b 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude_transformation.py @@ -18,7 +18,7 @@ from litellm.constants import DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude_transformation import ( AmazonAnthropicClaudeConfig, ) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.unit.conftest import NativeStructuredOutputKey from tests.unit.llms.anthropic.anthropic_docs_refusal import ANTHROPIC_DOCS_REFUSAL_MESSAGE @@ -1052,6 +1052,64 @@ async def test_bedrock_invoke_claude_async_completion_inlines_document_url_sourc } in captured["body"]["messages"][0]["content"] +def test_bedrock_invoke_claude_sync_completion_downloads_a_pdf_link_once_and_sends_it_as_a_base64_document(monkeypatch): + from litellm.litellm_core_utils.prompt_templates import image_handling + + pdf_url = f"https://docs.example/{uuid.uuid4()}/report.pdf" + pdf_bytes = b"%PDF-1.7\n" + b"0" * 64 + fetched = [] + captured = {} + + def serve_pdf(client, url, **kwargs): + fetched.append(url) + return httpx.Response( + 200, content=pdf_bytes, headers={"content-type": "application/pdf"}, request=httpx.Request("GET", url) + ) + + def handle(request): + captured["body"] = json.loads(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "us.anthropic.claude-sonnet-5", + "content": [{"type": "text", "text": "A lease"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + monkeypatch.setattr(image_handling, "safe_get", serve_pdf) + client = HTTPHandler() + client.client = httpx.Client(transport=httpx.MockTransport(handle)) + + response = litellm.completion( + model="bedrock/invoke/us.anthropic.claude-sonnet-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this document?"}, + {"type": "image_url", "image_url": {"url": pdf_url}}, + ], + } + ], + aws_access_key_id="AKIAEXAMPLE", + aws_secret_access_key="fake-secret", + aws_region_name="us-east-1", + client=client, + ) + + assert response.choices[0].message.content == "A lease" + assert fetched == [pdf_url] + assert captured["body"]["messages"][0]["content"][1] == { + "type": "document", + "source": {"type": "base64", "media_type": "application/pdf", "data": base64.b64encode(pdf_bytes).decode()}, + } + + @pytest.mark.parametrize( "model, expected_betas", [ diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py index d2e5b407e30..1baf9504208 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_structured_messages_writeback.py @@ -200,3 +200,78 @@ async def test_write_back_keeps_real_tool_results_under_modify_params(): assert "skipped" not in serialized assert "Please continue" not in serialized assert [m["role"] for m in written] == ["user", "assistant", "user", "user"] + + +@pytest.mark.asyncio +async def test_write_back_sends_the_client_url_blocks_as_the_client_sent_them(async_only_image_fetch): + image_link = "https://img.example/presskit/page.pdf?fm=png" + document_link = "https://files.example/reports/q3.pdf" + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu_1", "name": "fetch_report", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [ + {"type": "document", "source": {"type": "url", "url": document_link}, "title": "Q3 attachment"} + ], + }, + {"type": "text", "text": "compare these"}, + {"type": "image", "source": {"type": "url", "url": image_link}}, + { + "type": "document", + "source": {"type": "url", "url": document_link}, + "title": "Q3 report", + "context": "Board copy", + "citations": {"enabled": True}, + "cache_control": {"type": "ephemeral"}, + }, + {"type": "document", "source": {"type": "url", "url": document_link}, "title": "Q3 report, again"}, + ], + }, + ] + + written = await _write_back_identity([dict(m) for m in messages]) + + assert async_only_image_fetch.fetched == [] + assert [m["content"] for m in written[1:]] == [messages[1]["content"][:1], messages[1]["content"][1:]] + + +@pytest.mark.asyncio +async def test_write_back_keeps_each_url_document_fields_when_a_tool_result_after_it_has_the_same_link( + async_only_image_fetch, +): + document_link = "https://files.example/reports/q3.pdf" + messages = [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "tu_1", "name": "fetch_report", "input": {}}], + }, + { + "role": "user", + "content": [ + { + "type": "document", + "source": {"type": "url", "url": document_link}, + "title": "Q3 report", + "context": "Board copy", + }, + { + "type": "tool_result", + "tool_use_id": "tu_1", + "content": [ + {"type": "document", "source": {"type": "url", "url": document_link}, "title": "Q3 attachment"} + ], + }, + ], + }, + ] + + written = await _write_back_identity([dict(m) for m in messages]) + + assert [m["content"] for m in written[1:]] == [messages[1]["content"][1:], messages[1]["content"][:1]] From ecb8ef5ca8834f0010783b3de59a0584ed62dc6e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 3 Nov 2026 01:27:52 -0600 Subject: [PATCH 17/34] feat(realtime)!: support latest OpenAI audio models (#35600) * feat(realtime): support latest OpenAI audio models * fix(transcription): type SDK stream duration metadata explicitly * fix(realtime): preserve transport settings and translation duration * refactor(realtime): reuse SDK client initialization * fix(audio): bill finalized streams and authorize transcription models * refactor(audio): keep transcription pricing selection together * fix(realtime): reconcile model metadata and test contracts * fix(realtime): preserve provider duration and model capabilities * fix(realtime): keep transcription validation compatible with cost map guard * fix(realtime): address transcription and billing review findings * fix(realtime): preserve input usage and provider routing * fix(realtime): bill retained output and route Azure aliases * fix(realtime): reject malformed provider durations * fix(realtime): preserve batch cache rates and Azure v1 routes * fix(transcription): reject nonfinite provider durations * refactor(realtime): align annotations with current lint rules * fix(realtime): preserve translation deployment pricing and input usage * test(transcription): verify synchronous stream finalization * fix(realtime): retain missing translation input usage at close * fix(transcription): reject streams that bypass output guardrails * fix(realtime): reconcile audio protocols billing and current CI * fix(realtime): trust confirmed formats and authorize default models * fix(realtime): preserve absent models and reject unsupported guardrails * fix(realtime): reject translation policy pipelines * chore(ci): rerun transient infrastructure timing failure * fix(realtime): reconcile audio protocols with current upstream * fix(audio): validate provider responses at SDK boundaries * fix(realtime): preserve minimum OpenAI SDK compatibility * fix(realtime): preserve deferred Live session authorization * fix(realtime): preserve upstream authentication and transcription behavior * fix(audio): scope provider duration precedence to configured models * fix(typing): avoid Final declarations inside transcription cost loop * refactor(audio): reuse transcription mapping and duration resolution * fix(realtime): enforce model budgets and preserve translation routing * fix(audio): log streamed transcripts whose done event reports detected languages * fix(audio): pass stream, languages and keywords through unchecked for non-OpenAI transcription providers Providers without a streaming transcription handler got UnsupportedParamsError for stream=True, languages and keywords, where the merge base forwarded them as provider-specific params. Only openai and azure validate them now. * fix(audio): bill streamed gpt-transcribe from the provider's reported duration The stream path computed cost after logging had replaced the transcription's duration usage, so it fell back to the measured file length while the unstreamed call billed the provider's seconds. * fix(transcription): log streamed transcription usage from the provider response The stream logging path built the spend payload from the response after its duration usage was transformed away, so whisper-1 with stream=true and streamed gpt-transcribe logged an empty usage object. Bill and log from the provider's TranscriptionResponse in both the sync and async paths. * fix(realtime): keep main's routing, protocol and transport behavior around the new translation routes - Send OpenAI realtime and translation client_secrets and calls through litellm's httpx client again, so ssl_verify and custom CA settings apply and upstream errors keep main's shape - Use the input transcription model as the session model only for transcription sessions - Drop the new /openai/v1 translation aliases and record each lazy feature's own routes, so the /openai pass-through and the MCP routes resolve as on main - Let a configured Azure realtime_protocol win for every model, with GA as the default only when none is set - Return x-litellm headers on streamed transcriptions and mark the request done - Refuse translation sessions before connecting upstream when realtime guardrails apply or the provider is not OpenAI or Azure - Accept json and text for gpt-transcribe, and drop other formats under drop_params instead of failing - Bill translation sessions on output audio seconds only, at the single published rate - Document the 201 SDP answer on the translation calls routes and the transcription 200 response * fix(realtime): restore main behavior for guardrails, Azure transcription, and realtime clients Streamed transcription runs unstreamed when a deployment post_call guardrail applies, so the guardrail still sees the transcript. Azure transcription maps v1, latest and preview to the default dated version on the deployments route for every model. The SDK client secret keeps session.model ahead of model, realtime WebSockets connect to api_base again, realtime metadata keeps only guardrails, and managed agents need an explicit transcription model. Reverts the two unrelated streamed Responses error codes and the duplicate cached_tokens_details field, and the gpt-transcribe format check only reads the request provider's cost map entry. * fix(realtime): keep main's transcription, client secret and route behavior where the PR changed it Transcription reaches every provider as on main again, sends stream only when asked, forwards language and languages together, keeps form key order, bills provider seconds only when above zero, honors base_model for the new flags, keeps main's Azure api_version on the proxy path, and streams carry chat's headers, an error event on a mid-stream failure, and a spend row when the stream ends early. Translation cost reads both per-second rates from one source and spend rows carry the billed seconds. Realtime transcription WebSockets keep model in the upstream URL outside OpenAI's hosts, client secrets forward explicit nulls, accept conversation and delay, and answer 400 on a malformed body, the /openai translation paths are back to main, translation refuses only for guardrails realtime sessions run, nested aliases resolve and later session.update models are authorized, translation client secrets run per-model auth, beta clients bridged to GA get beta-shaped session events, and transcription sessions send the deployment model in both shapes. * fix(realtime): address review findings on transcription billing, streaming and realtime auth Per-second pricing replaces token billing only when the priced entry sets no token rates, translation rates fall back per key to the cost map, transcription cache keys include languages and keywords, a stream request a post_call guardrail forces unstreamed comes back as redacted SSE, Azure beta clients are bridged only for GA-only deployments, nested transcription models on the realtime WebSocket are checked against the key, and translation spend carries key and team metadata * fix(azure): pick the GA realtime protocol for deployments whose model_info.base_model is GA-only * fix(realtime): preserve prior merge resolutions after rebase * fix(realtime): enforce translation policies and normalize Azure transcription URLs * fix(cookbook): stop translation once output idles after the upload ends * fix(realtime): bill translation on input audio and drop pre-read stream cost headers Translation spend now uses the input audio seconds the session sent, priced through input_cost_per_second, instead of the translated output duration. A streamed transcription no longer returns cost headers before the stream is read, since the cost is only known after the last event. Also removes an unused realtime guardrail helper. * fix(realtime): drop top-level session nulls, bill dropped transcription streams, and bridge beta response fields Client secret, translation and transcription session bodies drop top-level nulls again, as main did, while nested nulls such as turn_detection still pass through. Transcription client secrets send the deployment model as session.model. A streamed transcription the client drops still runs its success callbacks, and the stream ends with data: [DONE] like OpenAI's. The beta to GA bridge maps max_response_output_tokens, speed and noise reduction, remaps response.create, and drops temperature under drop_params. The Entra token for realtime HTTP routes is fetched off the event loop * test(e2e): cover realtime translation, live transcription, client secrets, and streamed transcription * fix(realtime): close review gaps in translation billing, transcription streams, and Azure GA Bill translation sessions from the deployment's own per-second rates, add per-second prices for Azure gpt-4o transcribe models, keep stream-scoped guardrails seeing the caller's stream flag, map mid-stream transcription errors to LiteLLM exceptions with a [DONE] frame, map GA responses back to the beta shape for bridged beta clients, honor the minting key's guardrail opt-out on translation calls, restore the Azure transcription session URL, and flag GA-only Azure realtime models * fix(realtime): bill a translation rate the deployment omits from the session model * test(e2e): check translated speech is not English instead of exact French wording * fix(realtime): keep main's behavior for non-bridged backends, stream error frames, gateway URLs and Azure transcribe prices Remap beta events to GA only for bridged beta clients and translation sessions, bridge beta clients on GA-pinned Azure deployments, log error-only upstream sessions as failures, flag dated Azure GA realtime models, drop the unpublished Azure per-second transcribe rates, log a mid-stream transcription failure once, keep main's SSE error frame outside transcription, drop unsupported transcription params under drop_params, keep the caller's Azure auth headers, keep a gateway /v1 path, let key opt-outs release string-callback guardrails on translation, and keep the cookbook waiting while a long upload is still sending * fix(realtime): keep the proxy's realtime session gate off translation sessions A key that opts out of a legacy callback guardrail now gets a working translation session, since the transcription-guardrail gate no longer sends a realtime session.update the translation upstream rejects * fix(realtime): keep websocket model limits, bill measured transcription length, route transcription sessions on the allowed model A WebSocket opened without a query model now authorizes as an empty model, transcription cost prefers the measured upload length over the provider's reported seconds, transcription sessions overwrite nested models with the authorized top-level model again, and a non-boolean transcription stream value answers 400 or is dropped under drop_params. * fix(ci): copy the rust bridge extension into the openapi docs gate's base worktree * fix(model-catalog): accept the realtime translation endpoints in the cost map * fix(cookbook): close the translation session and read until session.closed * fix(realtime): treat an empty transcription stream as unset and pass the cost-map guard An empty `stream` form value runs the transcription unstreamed again, as OpenAI does, instead of answering 400. The four translate cost-map entries drop the translation routes from supported_endpoints until the main branch's catalog parser accepts them, and the nested transcription model lookup uses the router registry entry point. * test(proxy): drop the mid-stream error frame pin that main's error shape change replaced * fix(transcription): bill uploaded seconds when a stream fails mid-transcript * fix(proxy): bill uploaded seconds when a streamed transcription breaks mid-stream * fix(audio): keep transcription's positional arguments and redact streamed error details - Move the new languages, keywords and stream options to the end of transcription's parameters, so a prompt or format passed by position keeps its meaning - Redact internal hostnames, paths and tracebacks from a streamed transcription's mid-stream error event, as chat streams already do - Move the CA-bundle check for the realtime HTTP routes, which serves real TLS on loopback, from the unit suite to the SDK integration suite * fix(audio): stream the guarded transcript to SDK callers that asked for a stream * fix(logging): keep a streamed transcript out of callbacks that turned off message logging A streamed transcription's assembled response reached success callbacks with its text intact under turn_off_message_logging or a callback's message_logging=False, because the shared streaming redactor had no transcription branch. It now redacts the text, as the unstreamed call already does * fix(proxy): stream a transcription the deployment streamed when the client sent no stream field --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/test-unit.yml | 1 + cookbook/gpt_realtime_translate.py | 245 ++++ .../crates/model-catalog/src/vocabulary.rs | 6 + litellm/__init__.py | 16 +- litellm/constants.py | 3 + litellm/cost_calculator.py | 212 +++- litellm/integrations/custom_guardrail.py | 13 +- .../audio_utils/transcription_streaming.py | 341 ++++++ .../litellm_core_utils/audio_utils/utils.py | 10 + litellm/litellm_core_utils/litellm_logging.py | 41 +- .../litellm_core_utils/llm_cost_calc/utils.py | 79 +- .../convert_dict_to_response.py | 12 +- .../litellm_core_utils/model_param_helper.py | 8 +- .../litellm_core_utils/realtime_streaming.py | 718 ++++++++++-- litellm/litellm_core_utils/redact_messages.py | 2 + litellm/llms/azure/audio_transcriptions.py | 41 +- litellm/llms/azure/realtime/handler.py | 96 +- .../azure/realtime/http_transformation.py | 38 +- .../audio_transcription/transformation.py | 119 +- .../base_llm/realtime/http_transformation.py | 21 + litellm/llms/custom_httpx/llm_http_handler.py | 109 +- litellm/llms/openai/realtime/handler.py | 15 +- .../openai/realtime/http_transformation.py | 12 + .../transcriptions/gpt_transformation.py | 15 +- litellm/llms/openai/transcriptions/handler.py | 42 +- litellm/main.py | 119 +- ...odel_prices_and_context_window_backup.json | 247 +++- litellm/proxy/_lazy_features.py | 7 +- litellm/proxy/_lazy_openapi_snapshot.json | 968 +++++++++++++++ litellm/proxy/_types.py | 6 + .../auth/managed_authorization.py | 16 +- litellm/proxy/auth/auth_utils.py | 1 + litellm/proxy/auth/user_api_key_auth.py | 17 +- litellm/proxy/common_request_processing.py | 2 + .../proxy/common_utils/http_parsing_utils.py | 28 +- litellm/proxy/common_utils/realtime_utils.py | 2 +- litellm/proxy/proxy_server.py | 138 ++- litellm/proxy/realtime_endpoints/endpoints.py | 198 +++- litellm/proxy/route_llm_request.py | 2 + litellm/realtime_api/main.py | 484 +++++++- litellm/router_python_core.py | 18 +- litellm/types/llms/openai.py | 15 + litellm/types/realtime.py | 47 +- litellm/types/utils.py | 53 +- litellm/utils.py | 153 ++- model_prices_and_context_window.json | 247 +++- scripts/check_openapi_docs.py | 12 +- .../llm_nonconversational.yaml | 6 + tests/e2e/coverage_registry/schema.py | 11 +- .../realtime/REALTIME_COVERAGE_MATRIX.md | 9 + .../realtime/realtime_client.py | 99 +- .../realtime/test_realtime_e2e.py | 227 +++- .../test_audio_transcriptions_e2e.py | 106 +- .../llm_translation/test_realtime_http_e2e.py | 291 ++++- .../test_audio_transcriptions_wire.py | 4 - ...est_openai_realtime_http_ca_bundle_wire.py | 95 ++ tests/unit/cookbook/__init__.py | 0 .../cookbook/test_gpt_realtime_translate.py | 141 +++ .../test_transcription_streaming.py | 319 +++++ .../llm_cost_calc/test_llm_cost_calc_utils.py | 61 + .../test_model_param_helper.py | 19 + .../test_realtime_streaming.py | 1012 +++++++++++++++- .../realtime/test_azure_realtime_handler.py | 47 + .../realtime/test_http_transformation.py | 66 +- .../llms/azure/test_audio_transcriptions.py | 68 ++ .../llms/azure/test_azure_common_utils.py | 5 + .../realtime/test_openai_realtime_handler.py | 84 ++ .../realtime/test_transcription_sessions.py | 65 +- .../llms/openai/realtime/test_translation.py | 185 +++ .../transcriptions/test_gpt_transcribe.py | 951 +++++++++++++++ .../test_transcription_duration_hidden.py | 63 +- .../auth/test_managed_authorization.py | 71 +- tests/unit/proxy/auth/test_route_checks.py | 46 + .../unit/proxy/auth/test_user_api_key_auth.py | 37 + .../test_user_api_key_auth_request_flow.py | 182 +++ .../common_utils/test_http_parsing_utils.py | 11 + .../proxy/common_utils/test_realtime_cache.py | 9 + .../proxy/proxy_server/test_routes_audio.py | 698 ++++++++++- .../public_endpoints/test_public_endpoints.py | 29 +- .../test_realtime_webrtc_endpoints.py | 737 ++++++++++-- tests/unit/proxy/test__lazy_features.py | 43 + tests/unit/proxy/test_proxy_server.py | 86 ++ ...test_proxy_server_endpoints_and_startup.py | 13 +- tests/unit/realtime_api/test_main.py | 1042 ++++++++++++++++- tests/unit/test_check_openapi_docs.py | 22 +- tests/unit/test_cost_calculator.py | 408 +++++++ tests/unit/test_main.py | 73 ++ .../test_model_prices_and_context_window.py | 61 + tests/unit/test_utils.py | 47 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 552 ++++++++- 90 files changed, 12390 insertions(+), 706 deletions(-) create mode 100644 cookbook/gpt_realtime_translate.py create mode 100644 litellm/litellm_core_utils/audio_utils/transcription_streaming.py create mode 100644 tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py create mode 100644 tests/unit/cookbook/__init__.py create mode 100644 tests/unit/cookbook/test_gpt_realtime_translate.py create mode 100644 tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py create mode 100644 tests/unit/llms/openai/realtime/test_translation.py create mode 100644 tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2c05ee827a3..0a490bb8f45 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -441,6 +441,7 @@ jobs: tests/unit/ocr tests/unit/passthrough tests/unit/rag + tests/unit/cookbook tests/unit/realtime_api tests/unit/rerank_api tests/unit/search diff --git a/cookbook/gpt_realtime_translate.py b/cookbook/gpt_realtime_translate.py new file mode 100644 index 00000000000..827c65cdbfe --- /dev/null +++ b/cookbook/gpt_realtime_translate.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 + +import argparse +import asyncio +import base64 +import json +import os +import sys +import wave +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlencode, urlsplit, urlunsplit + +import websockets +from websockets.asyncio.client import ClientConnection + +SAMPLE_RATE = 24_000 +CHANNELS = 1 +SAMPLE_WIDTH = 2 +CHUNK_DURATION_SECONDS = 0.1 +CHUNK_BYTES = int(SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH * CHUNK_DURATION_SECONDS) +RESPONSE_TIMEOUT_SECONDS = 30.0 +AUDIO_EVENT_TYPES = frozenset( + { + "session.output_audio.delta", + "response.audio.delta", + "response.output_audio.delta", + } +) +TRANSCRIPT_EVENT_TYPES = frozenset( + { + "session.output_transcript.delta", + "response.text.delta", + "response.output_audio_transcript.delta", + } +) + + +@dataclass(frozen=True, slots=True) +class Settings: + input_wav: Path + output_wav: Path + base_url: str + model: str + target_language: str + trailing_silence_seconds: float + api_key: str + + +def write_stdout(message: str = "", *, end: str = "\n", flush: bool = False) -> None: + sys.stdout.write(f"{message}{end}") + if flush: + sys.stdout.flush() + + +def write_stderr(message: str) -> None: + sys.stderr.write(f"{message}\n") + + +def parse_args(argv: Sequence[str] | None = None) -> Settings | str: + parser = argparse.ArgumentParser( + description="Stream a 24 kHz PCM16 WAV through gpt-realtime-translate and save the translated audio", + ) + parser.add_argument("input_wav", type=Path) + parser.add_argument("--output", type=Path, default=Path("translated.wav")) + parser.add_argument("--base-url", default=os.getenv("LITELLM_BASE_URL", "http://localhost:4000")) + parser.add_argument("--model", default=os.getenv("REALTIME_TRANSLATE_MODEL", "gpt-realtime-translate")) + parser.add_argument("--target-language", default="fr") + parser.add_argument("--trailing-silence", type=float, default=1.5) + parsed = parser.parse_args(argv) + api_key = os.getenv("LITELLM_API_KEY") or os.getenv("OPENAI_API_KEY") + if not api_key: + return "Set LITELLM_API_KEY or OPENAI_API_KEY before running the script" + if parsed.trailing_silence < 0: + return "--trailing-silence must be zero or greater" + return Settings( + input_wav=parsed.input_wav, + output_wav=parsed.output, + base_url=parsed.base_url, + model=parsed.model, + target_language=parsed.target_language, + trailing_silence_seconds=parsed.trailing_silence, + api_key=api_key, + ) + + +def translation_url(base_url: str, model: str) -> str | None: + parsed = urlsplit(base_url.rstrip("/")) + scheme = {"http": "ws", "https": "wss", "ws": "ws", "wss": "wss"}.get(parsed.scheme) + if not scheme or not parsed.netloc: + return None + base_path = parsed.path.rstrip("/") + realtime_path = ( + f"{base_path}/realtime/translations" if base_path.endswith("/v1") else f"{base_path}/v1/realtime/translations" + ) + return urlunsplit((scheme, parsed.netloc, realtime_path, urlencode({"model": model}), "")) + + +def read_pcm16_wav(path: Path) -> bytes | str: + try: + with wave.open(str(path), "rb") as source: + actual_format = ( + source.getnchannels(), + source.getsampwidth(), + source.getframerate(), + source.getcomptype(), + ) + expected_format = (CHANNELS, SAMPLE_WIDTH, SAMPLE_RATE, "NONE") + if actual_format != expected_format: + return ( + f"{path} must be mono, 16-bit PCM, 24 kHz WAV; received " + f"channels={actual_format[0]}, sample_width={actual_format[1]}, " + f"sample_rate={actual_format[2]}, compression={actual_format[3]}" + ) + return source.readframes(source.getnframes()) + except (OSError, EOFError, wave.Error) as exc: + return f"Unable to read {path}: {exc}" + + +def audio_chunks(audio: bytes) -> Iterator[bytes]: + return (audio[offset : offset + CHUNK_BYTES] for offset in range(0, len(audio), CHUNK_BYTES)) + + +def audio_message(audio: bytes) -> str: + return json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(audio).decode("ascii"), + } + ) + + +async def configure_session(connection: ClientConnection, target_language: str) -> str | None: + await connection.send( + json.dumps( + { + "type": "session.update", + "session": {"audio": {"output": {"language": target_language}}}, + } + ) + ) + while True: + raw_event = await asyncio.wait_for(connection.recv(), timeout=20) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type == "session.created": + write_stdout(f"Session: {event.get('session', {}).get('id', 'created')}") + if event_type == "session.updated": + return None + if event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + + +async def send_audio( + connection: ClientConnection, pcm: bytes, trailing_silence_seconds: float, finished: asyncio.Event +) -> None: + silence = bytes(round(trailing_silence_seconds * SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + try: + for chunk in audio_chunks(pcm + silence): + await connection.send(audio_message(chunk)) + await asyncio.sleep(len(chunk) / (SAMPLE_RATE * CHANNELS * SAMPLE_WIDTH)) + await connection.send(json.dumps({"type": "session.close"})) + finally: + finished.set() + + +async def receive_translation( + connection: ClientConnection, output_path: Path, sender_finished: asyncio.Event +) -> str | None: + audio_received = asyncio.Event() + try: + with wave.open(str(output_path), "wb") as output: + output.setnchannels(CHANNELS) + output.setsampwidth(SAMPLE_WIDTH) + output.setframerate(SAMPLE_RATE) + write_stdout("Translation: ", end="", flush=True) + while True: + try: + raw_event = await asyncio.wait_for(connection.recv(), timeout=RESPONSE_TIMEOUT_SECONDS) + except TimeoutError: + if not sender_finished.is_set(): + continue + return ( + f"The translation stream went quiet for {RESPONSE_TIMEOUT_SECONDS:g} s without closing the session" + ) + event = json.loads(raw_event) + event_type = event.get("type") + if event_type in AUDIO_EVENT_TYPES: + output.writeframes(base64.b64decode(event.get("delta", ""), validate=True)) + audio_received.set() + elif event_type in TRANSCRIPT_EVENT_TYPES: + write_stdout(event.get("delta", event.get("text", "")), end="", flush=True) + elif event_type == "error": + return f"Realtime API error: {json.dumps(event.get('error', event), ensure_ascii=False)}" + elif event_type == "session.closed": + write_stdout() + return None if audio_received.is_set() else "The translation stream ended without translated audio" + except (OSError, wave.Error) as exc: + return f"Unable to write {output_path}: {exc}" + + +async def translate(settings: Settings, pcm: bytes) -> str | None: + url = translation_url(settings.base_url, settings.model) + if not url: + return f"Invalid --base-url: {settings.base_url}" + sender_finished = asyncio.Event() + try: + async with websockets.connect( + url, + additional_headers={"Authorization": f"Bearer {settings.api_key}"}, + proxy=None, + open_timeout=20, + close_timeout=5, + ) as connection: + configuration_error = await configure_session(connection, settings.target_language) + if configuration_error: + return configuration_error + async with asyncio.TaskGroup() as tasks: + receiver = tasks.create_task(receive_translation(connection, settings.output_wav, sender_finished)) + tasks.create_task(send_audio(connection, pcm, settings.trailing_silence_seconds, sender_finished)) + return receiver.result() + except Exception as exc: + return f"Translation failed: {type(exc).__name__}: {exc}" + + +def main(argv: Sequence[str] | None = None) -> int: + settings = parse_args(argv) + if isinstance(settings, str): + write_stderr(settings) + return 2 + pcm = read_pcm16_wav(settings.input_wav) + if isinstance(pcm, str): + write_stderr(pcm) + return 2 + error = asyncio.run(translate(settings, pcm)) + if error: + write_stderr(error) + return 1 + write_stdout(f"Translated audio: {settings.output_wav}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/litellm-rust/crates/model-catalog/src/vocabulary.rs b/litellm-rust/crates/model-catalog/src/vocabulary.rs index 2b7ac8bd1ab..6bf2cc55224 100644 --- a/litellm-rust/crates/model-catalog/src/vocabulary.rs +++ b/litellm-rust/crates/model-catalog/src/vocabulary.rs @@ -93,6 +93,12 @@ pub enum Endpoint { Realtime, #[serde(rename = "/v1/realtime/transcription_sessions")] RealtimeTranscriptionSessions, + #[serde(rename = "/v1/realtime/translations")] + RealtimeTranslations, + #[serde(rename = "/v1/realtime/translations/calls")] + RealtimeTranslationCalls, + #[serde(rename = "/v1/realtime/translations/client_secrets")] + RealtimeTranslationClientSecrets, #[serde(rename = "/v1/videos")] Videos, #[serde(rename = "/v1/ocr")] diff --git a/litellm/__init__.py b/litellm/__init__.py index 041e3fc7837..102c4d1440f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -8,7 +8,7 @@ _apply_process_defaults() import threading import os -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from typing import ( Any, Callable, @@ -1562,7 +1562,9 @@ from .realtime_api.main import ( _arealtime, acreate_realtime_client_secret, acreate_realtime_transcription_session, + acreate_realtime_translation_client_secret as acreate_realtime_translation_client_secret, arealtime_calls, + arealtime_translation_calls as arealtime_translation_calls, ) from .responses.main import _aresponses_websocket @@ -1811,6 +1813,9 @@ if TYPE_CHECKING: from .llms.vertex_ai.rerank.transformation import ( VertexAIRerankConfig as VertexAIRerankConfig, ) + from .llms.together_ai.chat.transformation import ( + TogetherAIChatConfig as TogetherAIChatConfig, + ) from .llms.fireworks_ai.rerank.transformation import ( FireworksAIRerankConfig as FireworksAIRerankConfig, ) @@ -1856,9 +1861,6 @@ if TYPE_CHECKING: BedrockMantleAnthropicMessagesConfig as BedrockMantleAnthropicMessagesConfig, ) from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig - from .llms.together_ai.chat.transformation import ( - TogetherAIChatConfig as TogetherAIChatConfig, - ) from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig as VertexGeminiConfig, @@ -2017,6 +2019,9 @@ if TYPE_CHECKING: from .llms.xai.responses.transformation import ( XAIResponsesAPIConfig as XAIResponsesAPIConfig, ) + from .llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig as VertexAIInteractionsConfig, + ) from .llms.litellm_proxy.responses.transformation import ( LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig, ) @@ -2044,9 +2049,6 @@ if TYPE_CHECKING: from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) - from .llms.vertex_ai.interactions.transformation import ( - VertexAIInteractionsConfig as VertexAIInteractionsConfig, - ) from .llms.openai.chat.o_series_transformation import ( OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config, diff --git a/litellm/constants.py b/litellm/constants.py index 14a2a9f9eca..96caa162604 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -971,7 +971,10 @@ OPENAI_CHAT_COMPLETION_PARAMS: Final = [ OPENAI_TRANSCRIPTION_PARAMS: Final = [ "language", + "languages", + "keywords", "response_format", + "stream", "timestamp_granularities", ] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index ef89a5ca8e7..527981a1dc6 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -23,7 +23,8 @@ from litellm.constants import ( OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, REALTIME_METERED_USAGE_KEY, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, billed_usage_cost +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import ( decisions_pricing_fields, @@ -324,12 +325,41 @@ def _get_additional_costs( return None -def _transcription_usage_has_token_details( +_TranscriptionRateKey: TypeAlias = Literal[ + "input_cost_per_second", + "output_cost_per_second", + "input_cost_per_audio_token", + "output_cost_per_audio_token", + "input_cost_per_token", + "output_cost_per_token", +] +_TRANSCRIPTION_TOKEN_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = ( + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token", +) +_TRANSCRIPTION_SECOND_RATE_KEYS: Final[tuple[_TranscriptionRateKey, ...]] = ( + "input_cost_per_second", + "output_cost_per_second", +) + + +def _transcription_uses_token_pricing( + model: str, + custom_llm_provider: str | None, usage_block: Usage | None, ) -> bool: if usage_block is None: return False + model_info: Final = cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider) + if ( + _declared_transcription_rate(model_info, _TRANSCRIPTION_TOKEN_RATE_KEYS) is None + and _declared_transcription_rate(model_info, _TRANSCRIPTION_SECOND_RATE_KEYS) is not None + ): + return False + prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0 completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0 prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None) @@ -343,13 +373,38 @@ def _transcription_usage_has_token_details( return (prompt_tokens_val > 0) or (completion_tokens_val > 0) -def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool: - if model is None or custom_llm_provider != LlmProviders.OPENAI.value: +def _bills_transcription_usage_duration(pricing_model: str | None, custom_llm_provider: str | None) -> bool: + if pricing_model is None or custom_llm_provider is None: return False + transcription_info: Final = _lookup_model_info_or_none(pricing_model, custom_llm_provider) + transcription_metadata: Final = ( + as_str_mapping(transcription_info.get("provider_specific_entry")) if transcription_info is not None else None + ) + return transcription_metadata is not None and transcription_metadata.get("transcription_usage_duration") == 1 + + +def _transcription_duration_for_cost( + completion_response: object, pricing_model: str | None, model: str | None, custom_llm_provider: str | None +) -> float: + hidden_params: Final = as_str_mapping(attribute_or_none(completion_response, "_hidden_params")) or {} + measured_duration: Final = normalized_audio_duration_seconds( + hidden_params.get("audio_transcription_duration", attribute_or_none(completion_response, "duration")) + ) + if measured_duration is not None and measured_duration > 0: + return measured_duration + provider_duration: Final = _get_transcription_usage_duration(completion_response) + if ( + provider_duration is not None + and provider_duration > 0 + and any(_bills_transcription_usage_duration(name, custom_llm_provider) for name in (pricing_model, model)) + ): + return provider_duration + if model is None or custom_llm_provider != LlmProviders.OPENAI.value: + return 0.0 transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI ) - return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) + return (provider_duration or 0.0) if isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) else 0.0 OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"] @@ -701,7 +756,7 @@ def cost_per_token( vertex_location=vertex_location, ) elif call_type == "atranscription" or call_type == "transcription": - if _transcription_usage_has_token_details(usage_block): + if _transcription_uses_token_pricing(model_without_prefix, custom_llm_provider, usage_block): return generic_cost_per_token( model=model, usage=usage_block, @@ -1283,6 +1338,23 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: ) +def _get_transcription_usage_duration(completion_response: object) -> float | None: + response_fields: Final = as_str_mapping(completion_response) + usage_object: Final = ( + response_fields.get("usage") if response_fields is not None else attribute_or_none(completion_response, "usage") + ) + usage_fields: Final = as_str_mapping(usage_object) + usage_type: Final = ( + usage_fields.get("type") if usage_fields is not None else attribute_or_none(usage_object, "type") + ) + if usage_type != "duration": + return None + seconds: Final = ( + usage_fields.get("seconds") if usage_fields is not None else attribute_or_none(usage_object, "seconds") + ) + return normalized_audio_duration_seconds(seconds) + + def _extraction_usage(usage: ExtractionUsage) -> Usage: return Usage( prompt_tokens=usage.input_tokens, @@ -2008,20 +2080,8 @@ def completion_cost( prompt, ) elif call_type in _TRANSCRIPTION_CALL_TYPES: - # Check _hidden_params first (duration stored there to - # avoid polluting the response body), then fall back to - # the response attribute (for verbose_json responses that - # naturally include duration from the provider). - _hidden = getattr(completion_response, "_hidden_params", {}) or {} - audio_transcription_file_duration = _hidden.get( - "audio_transcription_duration", - getattr(completion_response, "duration", 0.0), - ) or ( - completion_response.usage.seconds - if isinstance(completion_response, TranscriptionResponse) - and isinstance(completion_response.usage, TranscriptionUsageDurationObject) - and _bills_provider_transcription_seconds(requested_model, custom_llm_provider) - else 0.0 + audio_transcription_file_duration = _transcription_duration_for_cost( + completion_response, model, requested_model, custom_llm_provider ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): @@ -2131,6 +2191,7 @@ def completion_cost( litellm_logging_obj=litellm_logging_obj, custom_pricing_model=selected_model if custom_pricing else None, base_pricing_model=(selected_model if base_model is not None and not custom_pricing else None), + deployment_models=(base_model, requested_model), ) elif call_type == _MCP_CALL_TYPE: from litellm.proxy.mcp_server.cost_calculator import ( @@ -3409,8 +3470,11 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor): def create_logging_realtime_object( usage: Usage, results: OpenAIRealtimeStreamList ) -> LiteLLMRealtimeStreamLoggingObject: + translated_input_seconds, _ = _translation_session_seconds(results) return LiteLLMRealtimeStreamLoggingObject( - usage=usage, + usage=usage.model_copy(update={"seconds": translated_input_seconds}) + if translated_input_seconds > 0 + else usage, results=results, ) @@ -3495,6 +3559,7 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +_TRANSLATION_CLOSED_EVENT_TYPE: Final = "session.closed" class _LiveResponsePayload(BaseModel): @@ -3717,6 +3782,7 @@ def handle_realtime_stream_cost_calculation( litellm_logging_obj: LitellmLoggingObject | None = None, custom_pricing_model: str | None = None, base_pricing_model: str | None = None, + deployment_models: Sequence[str | None] = (), ) -> float: """ Handles the cost calculation for realtime stream responses. @@ -3761,6 +3827,13 @@ def handle_realtime_stream_cost_calculation( if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results) else 0.0 ) + translation_cost: Final = handle_realtime_translation_cost_calculation( + results=results, + custom_llm_provider=custom_llm_provider, + litellm_model_name=litellm_model_name, + deployment_pricing_models=(custom_pricing_model, base_pricing_model, *deployment_models), + session_model=received_model, + ) live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"}) live_session_started: Final = any(result["type"] == "session.started" for result in results) live_session_closed: Final = any(result["type"] == "session.closed" for result in results) @@ -3817,12 +3890,19 @@ def handle_realtime_stream_cost_calculation( input_cost: Final = input_cost_per_token + live_response_input_cost output_cost: Final = output_cost_per_token + live_response_output_cost total_cost: Final = ( - input_cost + output_cost + transcription_cost + live_voice_cost + audio_duration_cost + text_message_cost + input_cost + + output_cost + + transcription_cost + + translation_cost + + live_voice_cost + + audio_duration_cost + + text_message_cost ) additional_costs: Final = tuple( (cost_name, cost) for cost_name, cost in ( ("transcription_cost", transcription_cost), + ("translation_cost", translation_cost), ("live_voice_cost", live_voice_cost), ("audio_duration_cost", audio_duration_cost), ("text_message_cost", text_message_cost), @@ -3843,6 +3923,55 @@ def handle_realtime_stream_cost_calculation( return total_cost +def _translation_session_seconds(results: OpenAIRealtimeStreamList) -> tuple[float, float]: + usage_events: Final = tuple( + usage + for result in results + if (event := as_str_mapping(result)) is not None + and event.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE + and (usage := as_str_mapping(event.get("usage"))) is not None + ) + input_seconds: Final = sum( + seconds + for usage in usage_events + if (seconds := normalized_audio_duration_seconds(usage.get("input_seconds"))) is not None + ) + output_seconds: Final = sum( + output_duration + for usage in usage_events + if (output_duration := normalized_audio_duration_seconds(usage.get("output_seconds"))) is not None + ) + return float(input_seconds), float(output_seconds) + + +def _translation_per_second_rates( + models: Sequence[str | None], custom_llm_provider: str +) -> tuple[float | None, float | None]: + model_infos: Final = tuple( + _lookup_model_info_or_none(model, custom_llm_provider) for model in models if model is not None + ) + return ( + _first_declared_transcription_rate(("input_cost_per_second",), model_infos), + _first_declared_transcription_rate(("output_cost_per_second",), model_infos), + ) + + +def handle_realtime_translation_cost_calculation( + results: OpenAIRealtimeStreamList, + custom_llm_provider: str, + litellm_model_name: str, + deployment_pricing_models: Sequence[str | None] = (), + session_model: str | None = None, +) -> float: + input_seconds, output_seconds = _translation_session_seconds(results) + if input_seconds <= 0 and output_seconds <= 0: + return 0.0 + input_rate, output_rate = _translation_per_second_rates( + (*deployment_pricing_models, litellm_model_name, session_model), custom_llm_provider + ) + return input_seconds * (input_rate or 0.0) + output_seconds * (output_rate or 0.0) + + def handle_realtime_transcription_cost_calculation( results: OpenAIRealtimeStreamList, custom_llm_provider: str, @@ -3919,17 +4048,16 @@ def _get_transcription_model_name_from_results( ) -_TranscriptionRateKey: TypeAlias = Literal[ - "input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token" -] - - -def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None: +def _model_info_rate(info: ModelInfoBase, key: _TranscriptionRateKey) -> float | None: match key: case "input_cost_per_second": return info.get("input_cost_per_second") + case "output_cost_per_second": + return info.get("output_cost_per_second") case "input_cost_per_audio_token": return info.get("input_cost_per_audio_token") + case "output_cost_per_audio_token": + return info.get("output_cost_per_audio_token") case "input_cost_per_token": return info.get("input_cost_per_token") case "output_cost_per_token": @@ -3938,7 +4066,7 @@ def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | Non assert_never(key) -def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: +def _declared_transcription_rate(info: ModelInfoBase | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None: """First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry because ``get_model_info`` synthesizes zero token rates for entries that omit them.""" if info is None: @@ -3956,15 +4084,19 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_Transcript ) -def _transcription_rate( - keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None -) -> float: - rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base)) - return next((rate for rate in rates if rate is not None), 0.0) +def _first_declared_transcription_rate( + keys: tuple[_TranscriptionRateKey, ...], infos: Sequence[ModelInfoBase | None] +) -> float | None: + rates: Final = (_declared_transcription_rate(info, keys) for info in infos) + return next((rate for rate in rates if rate is not None), None) + + +def _transcription_rate(keys: tuple[_TranscriptionRateKey, ...], *infos: ModelInfoBase | None) -> float: + return _first_declared_transcription_rate(keys, infos) or 0.0 def _transcription_usage_cost( - usage: dict, + usage: Mapping[str, object], model_info: ModelInfo | None, override_info: ModelInfo | None = None, ) -> float: @@ -3973,13 +4105,13 @@ def _transcription_usage_cost( usage_type: Final = usage.get("type") if usage_type == "duration": - seconds: Final = usage.get("seconds") or 0.0 + seconds: Final = normalized_audio_duration_seconds(usage.get("seconds")) or 0.0 return float(seconds) * _transcription_rate(("input_cost_per_second",), override_info, model_info) if usage_type == "tokens": - input_token_details: Final = usage.get("input_token_details") or {} - audio_tokens: Final = input_token_details.get("audio_tokens") or 0 - text_tokens: Final = input_token_details.get("text_tokens") or 0 - output_tokens: Final = usage.get("output_tokens") or 0 + input_token_details: Final = as_str_mapping(usage.get("input_token_details")) or {} + audio_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("audio_tokens")) or 0.0 + text_tokens: Final = normalized_audio_duration_seconds(input_token_details.get("text_tokens")) or 0.0 + output_tokens: Final = normalized_audio_duration_seconds(usage.get("output_tokens")) or 0.0 audio_cost: Final = float(audio_tokens) * _transcription_rate( ("input_cost_per_audio_token", "input_cost_per_token"), override_info, model_info ) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index dfe508f3630..68876812c3e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1089,6 +1089,12 @@ class CustomGuardrail(CustomLogger): enrich_http_exception_with_guardrail_context(result, self) raise result + def runs_post_call_deployment_hook(self, request_data: Mapping[str, object]) -> bool: + return ( + isinstance(request_data.get("guardrails"), list) + and self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is True + ) + async def async_post_call_success_deployment_hook( self, request_data: dict, @@ -1098,12 +1104,7 @@ class CustomGuardrail(CustomLogger): """ Allow modifying / reviewing the response just after it's received from the deployment. """ - # should run guardrail - litellm_guardrails: Final = request_data.get("guardrails") - if litellm_guardrails is None or not isinstance(litellm_guardrails, list): - return None - - if self.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call) is not True: + if not self.runs_post_call_deployment_hook(request_data): return None target: Final = self._deployment_hook_target() diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py new file mode 100644 index 00000000000..4650b7db2f2 --- /dev/null +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import datetime +import json +import traceback +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from typing import Final, Protocol + +import anyio +from openai import AsyncStream, Stream +from openai.types.audio import ( + TranscriptionStreamEvent, + TranscriptionTextDeltaEvent, + TranscriptionTextDoneEvent, +) +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.core_helpers import process_response_headers +from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, +) +from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage + +_STREAM_FLAG: Final = TypeAdapter(bool) + + +def transcription_stream_flag(value: object) -> bool | None: + try: + return _STREAM_FLAG.validate_python(value) + except ValidationError: + return None + + +def transcription_stream_unset(value: object) -> bool: + return value is None or value == "" + + +def transcription_stream_requested(value: object) -> bool: + return transcription_stream_flag(value) is True + + +TRANSCRIPTION_STREAM_DONE_FRAME: Final = "data: [DONE]\n\n" + +TranscriptionStreamExceptionMapper = Callable[[Exception], Exception] + + +def litellm_exception_mapper(call_details: Mapping[str, object]) -> TranscriptionStreamExceptionMapper: + from litellm.litellm_core_utils.exception_mapping_utils import exception_type, with_deployment_api_base + + model: Final = call_details.get("model") + custom_llm_provider: Final = call_details.get("custom_llm_provider") + extra_kwargs: Final = with_deployment_api_base(call_details.get("litellm_params")) + + def map_exception(exception: Exception) -> Exception: + try: + mapped: Final[object] = exception_type( + model=model if isinstance(model, str) else None, + original_exception=exception, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + extra_kwargs=extra_kwargs, + ) + except Exception as mapped_exception: # noqa: BLE001 # exception_type raises the mapped exception + return mapped_exception + return mapped if isinstance(mapped, Exception) else exception + + return map_exception + + +def _guarded_transcription_event_payloads( + response: TranscriptionResponse, +) -> tuple[dict[str, object], dict[str, object]]: + text: Final = response.text or "" + usage: Final = response.usage + languages: Final = TranscriptionResponse.detected_languages(getattr(response, "languages", None)) + return ( + {"type": "transcript.text.delta", "delta": text}, + { + "type": "transcript.text.done", + "text": text, + **({} if usage is None else {"usage": usage.model_dump(exclude_none=True)}), + **({} if languages is None else {"languages": [language.model_dump() for language in languages]}), + }, + ) + + +def guarded_transcription_events(response: TranscriptionResponse) -> tuple[str, ...]: + return ( + *( + f"data: {json.dumps(event, separators=(',', ':'))}\n\n" + for event in _guarded_transcription_event_payloads(response) + ), + TRANSCRIPTION_STREAM_DONE_FRAME, + ) + + +class GuardedTranscriptionReplay: + def __init__(self, response: TranscriptionResponse) -> None: + self.response: Final = response + self._hidden_params: dict[str, object] = response.hidden_params + + def __aiter__(self) -> AsyncIterator[TranscriptionStreamEvent]: + return self._events() + + async def _events(self) -> AsyncIterator[TranscriptionStreamEvent]: + delta, done = _guarded_transcription_event_payloads(self.response) + yield TranscriptionTextDeltaEvent.model_construct(**delta) + yield TranscriptionTextDoneEvent.model_construct(**done) + + +class TranscriptionStreamLogging(Protocol): + def success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + async def async_success_handler( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def handle_sync_success_callbacks_for_async_calls( + self, + result: TranscriptionResponse, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + *, + for_async_request: bool = False, + ) -> None: ... + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: ... + + def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ... + + def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ... + + +class _TranscriptionEventCollector: + def __init__(self, duration: float | None) -> None: + self.duration = duration + self.text_deltas: list[str] = [] # mutable-ok: streaming deltas accumulate until the terminal event + self.done_event: TranscriptionTextDoneEvent | None = None + + def add(self, event: TranscriptionStreamEvent) -> None: + if isinstance(event, TranscriptionTextDeltaEvent): + self.text_deltas.append(event.delta) + elif isinstance(event, TranscriptionTextDoneEvent): + self.done_event = event + + def response(self) -> TranscriptionResponse: + done_event: Final = self.done_event + response: Final = TranscriptionResponse( + text=done_event.text if done_event is not None else "".join(self.text_deltas), + usage=done_event.usage.model_dump() if done_event is not None and done_event.usage is not None else None, + languages=getattr(done_event, "languages", None) if done_event is not None else None, + ) + if self.duration is not None: + response.set_audio_transcription_duration(self.duration) + return response + + +def _billed_usage(response: TranscriptionResponse) -> Usage: + usage: Final[object] = getattr(response, "usage", None) + token_usage: Final = ( + TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage) + if isinstance(usage, TranscriptionUsageTokensObject) + else None + ) + return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + + +def _record_billable_partial_spend( + logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector +) -> None: + partial_response: Final = collector.response() + partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0 + if partial_cost <= 0: + return + logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost) + + +class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): + def __init__( + self, + stream: Stream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + map_exception: TranscriptionStreamExceptionMapper, + ) -> None: + self.__dict__.update(stream.__dict__) + self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata + "additional_headers": process_response_headers(stream.response.headers) + } + self._logging_obj = logging_obj + self._start_time = start_time + self._map_exception = map_exception + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + def _logging_iterator( + self, source_iterator: Iterator[TranscriptionStreamEvent] + ) -> Iterator[TranscriptionStreamEvent]: + try: + for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + _record_billable_partial_spend(self._logging_obj, self._collector) + mapped_exception: Final = self._map_exception(exception) + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time) + raise mapped_exception from exception + finally: + self._finalize() + + def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + self._logging_obj.success_handler( + self._collector.response(), + self._start_time, + datetime.datetime.now(), # noqa: DTZ005 # callback timestamps use the legacy naive contract + ) + + def close(self) -> None: + try: + super().close() + finally: + self._finalize() + + @property + def logging_obj(self) -> TranscriptionStreamLogging: + return self._logging_obj + + +class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): + def __init__( + self, + stream: AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + map_exception: TranscriptionStreamExceptionMapper, + ) -> None: + self.__dict__.update(stream.__dict__) + self._hidden_params: dict[str, object] = { # mutable-ok: ResponseMetadata fills in the call's metadata + "additional_headers": process_response_headers(stream.response.headers) + } + self._logging_obj = logging_obj + self._start_time = start_time + self._map_exception = map_exception + self._collector = _TranscriptionEventCollector(getattr(stream, "_litellm_audio_duration", None)) + self._finalized = False + self._failed = False + source_iterator: Final = self._iterator + self._iterator = self._logging_iterator(source_iterator) + + async def _logging_iterator( + self, source_iterator: AsyncIterator[TranscriptionStreamEvent] + ) -> AsyncIterator[TranscriptionStreamEvent]: + try: + async for event in source_iterator: + self._collector.add(event) + yield event + except Exception as exception: + self._failed = True + _record_billable_partial_spend(self._logging_obj, self._collector) + mapped_exception: Final = self._map_exception(exception) + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.failure_handler( + mapped_exception, traceback.format_exc(), self._start_time, end_time, for_async_request=True + ) + await self._logging_obj.async_failure_handler( + mapped_exception, + traceback.format_exc(), + self._start_time, + end_time, + ) + raise mapped_exception from exception + finally: + await self._finalize() + + async def _finalize(self) -> None: + if self._finalized or self._failed: + return + self._finalized = True + response: Final = self._collector.response() + end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract + self._logging_obj.handle_sync_success_callbacks_for_async_calls( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + with anyio.CancelScope(shield=True): + await self._logging_obj.async_success_handler( + result=response, + start_time=self._start_time, + end_time=end_time, + ) + + async def close(self) -> None: + try: + await super().close() + finally: + await self._finalize() + + @property + def logging_obj(self) -> TranscriptionStreamLogging: + return self._logging_obj + + +def wrap_transcription_stream( + stream: Stream[TranscriptionStreamEvent] | AsyncStream[TranscriptionStreamEvent], + logging_obj: TranscriptionStreamLogging, + start_time: datetime.datetime, + map_exception: TranscriptionStreamExceptionMapper, +) -> LoggingTranscriptionStream | LoggingAsyncTranscriptionStream: + if isinstance(stream, AsyncStream): + return LoggingAsyncTranscriptionStream(stream, logging_obj, start_time, map_exception) + return LoggingTranscriptionStream(stream, logging_obj, start_time, map_exception) diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index 640559ac7e2..29b10ac3c83 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -21,6 +21,16 @@ from litellm.types.files import ( from litellm.types.utils import FileTypes +def normalized_audio_duration_seconds(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + seconds: Final = float(value) + except OverflowError: + return None + return seconds if math.isfinite(seconds) and seconds >= 0 else None + + @dataclass class ProcessedAudioFile: """ diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 37657a94f16..22c5f051d5c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -22,6 +22,8 @@ from types import MappingProxyType, TracebackType from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast from httpx import Response, Timeout +from openai import AsyncStream as OpenAIAsyncStream +from openai import Stream as OpenAIStream from pydantic import BaseModel, JsonValue, TypeAdapter import litellm @@ -2130,6 +2132,9 @@ class Logging(LiteLLMLoggingBaseClass): if resolved_cache_hit is True: return 0.0 + if isinstance(result, (OpenAIStream, OpenAIAsyncStream)): + return None + if is_unbilled_non_inference_call( self.call_type, StandardLoggingPayloadSetup.merge_litellm_metadata(self.litellm_params), result ): @@ -3625,6 +3630,7 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit) if not self.should_run_logging(event_type="sync_success"): # prevent double logging return + provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None start_time, end_time, result, request_logging_payload = self.prepare_success_logging( start_time=start_time, end_time=end_time, @@ -3639,7 +3645,12 @@ class Logging(LiteLLMLoggingBaseClass): try: ## BUILD COMPLETE STREAMED RESPONSE complete_streaming_response: ( - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None ) = None if "complete_streaming_response" in self.model_call_details and not for_async_request: return # break out of this. @@ -3655,7 +3666,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_streaming_response"] = complete_streaming_response self._surface_response_headers_from_result(complete_streaming_response) self.model_call_details["response_cost"] = self.response_cost_calculator( - result=complete_streaming_response + result=complete_streaming_response if provider_transcription is None else provider_transcription ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) @@ -3665,7 +3676,9 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD if not for_async_request: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - logged_streaming_response, start_time, end_time + logged_streaming_response if provider_transcription is None else provider_transcription, + start_time, + end_time, ) standard_logging_payload: Final[StandardLoggingPayload | None] = self.model_call_details.get( "standard_logging_object" @@ -4192,6 +4205,7 @@ class Logging(LiteLLMLoggingBaseClass): kwargs=self.model_call_details, messages=self.model_call_details.get("messages") ) ) + provider_transcription: Final = result if isinstance(result, TranscriptionResponse) else None start_time, end_time, result = self._success_handler_helper_fn( start_time=start_time, end_time=end_time, @@ -4211,9 +4225,7 @@ class Logging(LiteLLMLoggingBaseClass): ## BUILD COMPLETE STREAMED RESPONSE if "async_complete_streaming_response" in self.model_call_details: return # break out of this. - complete_streaming_response: Final[ - ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None - ] = self._get_assembled_streaming_response( + complete_streaming_response: Final = self._get_assembled_streaming_response( result=result, start_time=start_time, end_time=end_time, @@ -4235,7 +4247,7 @@ class Logging(LiteLLMLoggingBaseClass): get_base_model_from_metadata(model_call_details=self.model_call_details) # base_model defaults to None if not set on model_info self.model_call_details["response_cost"] = self.response_cost_calculator( - result=complete_streaming_response + result=complete_streaming_response if provider_transcription is None else provider_transcription ) verbose_logger.debug("Model=%s; cost=%s", self.model, self.model_call_details["response_cost"]) @@ -4262,7 +4274,9 @@ class Logging(LiteLLMLoggingBaseClass): ## STANDARDIZED LOGGING PAYLOAD try: self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( - logged_streaming_response, start_time, end_time + logged_streaming_response if provider_transcription is None else provider_transcription, + start_time, + end_time, ) except Exception: # noqa: BLE001 # payload build must never block later callbacks (slot release) verbose_logger.exception( @@ -5132,10 +5146,17 @@ class Logging(LiteLLMLoggingBaseClass): end_time: datetime.datetime, is_async: bool, streaming_chunks: list[object], - ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None: + ) -> ( + ModelResponse + | TextCompletionResponse + | ResponsesAPIResponse + | InteractionsAPIResponse + | TranscriptionResponse + | None + ): if self.stream is not True: return None - if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse): + if isinstance(result, (ModelResponse, TextCompletionResponse, TranscriptionResponse)): return result elif isinstance( result, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 50c26f4d4ca..c553bfeaad8 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -1040,6 +1040,11 @@ def calculate_cache_writing_cost( class PromptTokensDetailsResult(TypedDict): cache_hit_tokens: int cache_hit_audio_tokens: ReadOnly[int] + + cached_text_tokens: ReadOnly[int] + cached_audio_tokens: ReadOnly[int] + cached_image_tokens: ReadOnly[int] + has_cached_tokens_details: ReadOnly[bool] cache_creation_tokens: int cache_creation_token_details: CacheCreationTokenDetails | None text_tokens: int @@ -1126,6 +1131,10 @@ def parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult: return PromptTokensDetailsResult( cache_hit_tokens=cache_hit_tokens, cache_hit_audio_tokens=cached_audio_tokens, + cached_text_tokens=cached_text_tokens, + cached_audio_tokens=cached_audio_tokens, + cached_image_tokens=cached_image_tokens, + has_cached_tokens_details=cached_tokens_details is not None, cache_creation_tokens=cache_creation_tokens, cache_creation_token_details=cache_creation_token_details, text_tokens=text_tokens, @@ -1234,15 +1243,11 @@ def _calculate_input_cost( prompt_cost = 0.0 if bills_characters else float(prompt_tokens_details["text_tokens"]) * prompt_base_cost ### CACHE READ COST - Now uses tiered pricing - cache_hit_audio_tokens: Final = prompt_tokens_details["cache_hit_audio_tokens"] - audio_cache_read_rate: Final = get_cost_per_unit( - model_info, - get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier), - None, - ) - prompt_cost += float(prompt_tokens_details["cache_hit_tokens"] - cache_hit_audio_tokens) * cache_read_cost - prompt_cost += float(cache_hit_audio_tokens) * ( - audio_cache_read_rate if audio_cache_read_rate is not None else cache_read_cost + prompt_cost += _calculate_cache_read_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=model_info, + cache_read_cost=cache_read_cost, + service_tier=service_tier, ) ### AUDIO COST @@ -1331,6 +1336,38 @@ def _calculate_input_cost( return prompt_cost +def _calculate_cache_read_cost( + prompt_tokens_details: PromptTokensDetailsResult, + model_info: ModelInfo, + cache_read_cost: float, + service_tier: str | None, +) -> float: + cached_text_tokens: Final = prompt_tokens_details["cached_text_tokens"] + cached_audio_tokens: Final = prompt_tokens_details["cached_audio_tokens"] + cached_image_tokens: Final = prompt_tokens_details["cached_image_tokens"] + classified_cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens + unclassified_cached_tokens: Final = max(prompt_tokens_details["cache_hit_tokens"] - classified_cached_tokens, 0) + total_cost = ( # rebind-ok: cached modality components accumulate into one cache-read cost + float(cached_text_tokens + unclassified_cached_tokens) * cache_read_cost + ) + + if cached_audio_tokens: + cached_audio_cost_key: Final = get_service_tier_cost_key("cache_read_input_audio_token_cost", service_tier) + cached_audio_cost: Final = get_cost_per_unit(model_info, cached_audio_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached audio contributes to cache-read cost + float(cached_audio_tokens) * float(cached_audio_cost or 0.0) + ) + + if cached_image_tokens: + cached_image_cost_key: Final = get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier) + cached_image_cost: Final = get_cost_per_unit(model_info, cached_image_cost_key, cache_read_cost) + total_cost += ( # rebind-ok: cached images contribute to cache-read cost + float(cached_image_tokens) * float(cached_image_cost or 0.0) + ) + + return total_cost + + def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float: """ Resolve the per-model regional-processing uplift multiplier for a given @@ -1502,6 +1539,10 @@ def generic_cost_per_token( prompt_tokens_details = PromptTokensDetailsResult( cache_hit_tokens=0, cache_hit_audio_tokens=0, + cached_text_tokens=0, + cached_audio_tokens=0, + cached_image_tokens=0, + has_cached_tokens_details=False, cache_creation_tokens=0, cache_creation_token_details=None, text_tokens=usage.prompt_tokens, @@ -1680,6 +1721,7 @@ class BilledTokenRates: cache_creation_input_token_cost: float cache_creation_input_token_cost_by_ttl: Mapping[str, float] output_cost_per_reasoning_token: float + cache_read_input_image_token_cost: float | None = None def scaled(self, multiplier: float) -> "BilledTokenRates": if multiplier == 1.0: @@ -1694,6 +1736,11 @@ class BilledTokenRates: {ttl: rate * multiplier for ttl, rate in self.cache_creation_input_token_cost_by_ttl.items()} ), output_cost_per_reasoning_token=self.output_cost_per_reasoning_token * multiplier, + cache_read_input_image_token_cost=( + self.cache_read_input_image_token_cost * multiplier + if self.cache_read_input_image_token_cost is not None + else None + ), ) @@ -1802,6 +1849,11 @@ def _cost_map_billed_rates( cache_creation_input_token_cost=base_costs.cache_creation_rate, cache_creation_input_token_cost_by_ttl=base_costs.cache_creation_rates_by_ttl, output_cost_per_reasoning_token=reasoning_rate, + cache_read_input_image_token_cost=_get_cost_per_unit( + model_info, + _get_service_tier_cost_key("cache_read_input_image_token_cost", service_tier), + None, + ), ).scaled(multiplier) @@ -1882,6 +1934,12 @@ def get_token_type_cost_breakdown( cache_read_tokens, cached_audio_tokens, cache_creation_tokens, cache_creation_token_details = _cache_token_counts( usage ) + cached_image_tokens: Final = parse_prompt_tokens_details(usage)["cached_image_tokens"] + image_cache_read_rate: Final = ( + rates.cache_read_input_image_token_cost + if rates.cache_read_input_image_token_cost is not None + else rates.cache_read_input_token_cost + ) cache_creation_cost: Final = ( float(cache_creation_tokens) * rates.cache_creation_input_token_cost if custom_cost_per_token is not None @@ -1895,8 +1953,9 @@ def get_token_type_cost_breakdown( return TokenTypeCostBreakdown( reasoning_cost=float(_reasoning_token_count(usage)) * rates.output_cost_per_reasoning_token, cache_read_cost=( - float(cache_read_tokens - cached_audio_tokens) * rates.cache_read_input_token_cost + float(cache_read_tokens - cached_audio_tokens - cached_image_tokens) * rates.cache_read_input_token_cost + float(cached_audio_tokens) * rates.cache_read_input_audio_token_cost + + float(cached_image_tokens) * image_cache_read_rate ), cache_creation_cost=cache_creation_cost, rates=rates, diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 6bb036e9080..9a0a35979db 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -970,9 +970,11 @@ def convert_to_model_response_object( model_response_object.data = response_object["data"] if "usage" in response_object and response_object["usage"] is not None: - model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) - model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) - model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) + embedding_usage: Final = model_response_object.usage or Usage() + embedding_usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) + embedding_usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) + embedding_usage.total_tokens = response_object["usage"].get("total_tokens", 0) + model_response_object.usage = embedding_usage if start_time is not None and end_time is not None: model_response_object._response_ms = ( @@ -1015,6 +1017,10 @@ def convert_to_model_response_object( if key in response_object: setattr(model_response_object, key, response_object[key]) + detected_languages: Final = TranscriptionResponse.detected_languages(response_object.get("languages")) + if detected_languages is not None: + model_response_object.languages = detected_languages + if "usage" in response_object and response_object["usage"] is not None: tr_usage_object: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None diff --git a/litellm/litellm_core_utils/model_param_helper.py b/litellm/litellm_core_utils/model_param_helper.py index b08596e5244..3e82077f9d6 100644 --- a/litellm/litellm_core_utils/model_param_helper.py +++ b/litellm/litellm_core_utils/model_param_helper.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from functools import lru_cache -from typing import ClassVar, Final +from typing import ClassVar, Final, get_args from openai.types.chat.completion_create_params import ( CompletionCreateParamsNonStreaming, @@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams from litellm.types.rerank import RerankRequest @@ -178,6 +179,7 @@ class ModelParamHelper: This follows the OpenAI API Spec """ + litellm_transcription_kwargs: Final = set(get_args(OpenAIAudioTranscriptionOptionalParams)) try: from openai.types.audio.transcription_create_params import ( TranscriptionCreateParamsNonStreaming, @@ -187,11 +189,11 @@ class ModelParamHelper: non_streaming_kwargs = set(getattr(TranscriptionCreateParamsNonStreaming, "__annotations__", {}).keys()) streaming_kwargs: Final = set(getattr(TranscriptionCreateParamsStreaming, "__annotations__", {}).keys()) - all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs) + all_transcription_kwargs: Final = non_streaming_kwargs.union(streaming_kwargs, litellm_transcription_kwargs) return all_transcription_kwargs except Exception as e: verbose_logger.debug("Error getting transcription kwargs %s", str(e)) - return set() + return litellm_transcription_kwargs @staticmethod def _get_litellm_supported_responses_api_kwargs() -> set[str]: diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 46f5ca87a93..e9e9a5a75b9 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,4 +1,6 @@ import asyncio +import base64 +import functools import json import time import traceback @@ -8,7 +10,7 @@ from enum import Enum, auto from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast, runtime_checkable -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly import litellm @@ -22,6 +24,8 @@ from litellm.constants import ( REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) +from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import ( @@ -35,6 +39,8 @@ from litellm.types.llms.openai import ( OpenAIRealtimeResponseDelta, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeTranslationClosedEvent, + OpenAIRealtimeTranslationDurationUsage, ) from litellm.types.realtime import ALL_DELTA_TYPES @@ -42,9 +48,11 @@ from .litellm_logging import Logging as LiteLLMLogging from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason if TYPE_CHECKING: + from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats from websockets.asyncio.client import ClientConnection from websockets.exceptions import ConnectionClosed + from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks CLIENT_CONNECTION_CLASS = ClientConnection @@ -96,6 +104,7 @@ class _ClientEventFrame(TypedDict, total=False): type: ReadOnly[str] item: ReadOnly[_ClientEventItem] session: ReadOnly[Mapping[str, object]] + response: ReadOnly[Mapping[str, object]] class _ResponseDoneBody(TypedDict, total=False): @@ -212,6 +221,87 @@ def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, ob return _json_object(container.get(key)) +_BETA_AUDIO_FORMAT_BY_GA_TYPE: Final[Mapping[str, str]] = MappingProxyType( + {"audio/pcm": "pcm16", "audio/pcmu": "g711_ulaw", "audio/pcma": "g711_alaw"} +) +_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"output_modalities": "modalities", "max_output_tokens": "max_response_output_tokens"} +) +_GA_TO_BETA_AUDIO_INPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + { + "format": "input_audio_format", + "transcription": "input_audio_transcription", + "noise_reduction": "input_audio_noise_reduction", + "turn_detection": "turn_detection", + } +) +_GA_TO_BETA_AUDIO_OUTPUT_FIELDS: Final[Mapping[str, str]] = MappingProxyType( + {"format": "output_audio_format", "voice": "voice", "speed": "speed"} +) +_GA_ONLY_SESSION_FIELDS: Final = frozenset(("type", "audio", *_GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS)) + + +def _beta_session_value(ga_field: str, value: object) -> object: + if ga_field == "output_modalities": + return ["audio", "text"] if value == ["audio"] else value + if ga_field != "format": + return value + ga_format: Final = _json_object(value) + format_type: Final = None if ga_format is None else ga_format.get("type") + return _BETA_AUDIO_FORMAT_BY_GA_TYPE.get(format_type, value) if isinstance(format_type, str) else value + + +def _beta_session_fields(ga_fields: Mapping[str, object], beta_names: Mapping[str, str]) -> Mapping[str, object]: + return { + beta_names[name]: _beta_session_value(name, value) for name, value in ga_fields.items() if name in beta_names + } + + +def ga_session_to_beta(session: Mapping[str, object]) -> Mapping[str, object]: + audio: Final = _mapping_field(session, "audio") or _EMPTY_SESSION + return { + **{key: value for key, value in session.items() if key not in _GA_ONLY_SESSION_FIELDS}, + **_beta_session_fields(session, _GA_TO_BETA_TOP_LEVEL_SESSION_FIELDS), + **_beta_session_fields(_mapping_field(audio, "input") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_INPUT_FIELDS), + **_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS), + } + + +_GA_ONLY_RESPONSE_FIELDS: Final = frozenset(("audio", "output_modalities")) +_GA_TO_BETA_RESPONSE_FIELDS: Final[Mapping[str, str]] = MappingProxyType({"output_modalities": "modalities"}) + + +def ga_response_to_beta(response: Mapping[str, object]) -> Mapping[str, object]: + audio: Final = _mapping_field(response, "audio") or _EMPTY_SESSION + return { + **{key: value for key, value in response.items() if key not in _GA_ONLY_RESPONSE_FIELDS}, + **_beta_session_fields(response, _GA_TO_BETA_RESPONSE_FIELDS), + **_beta_session_fields(_mapping_field(audio, "output") or _EMPTY_SESSION, _GA_TO_BETA_AUDIO_OUTPUT_FIELDS), + } + + +_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...]) +_GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE: Final[Mapping[str, str]] = MappingProxyType( + {"text": "output_text", "audio": "output_audio"} +) + + +def _ga_assistant_content_block(block: object) -> object: + content_block: Final = _json_object(block) + block_type: Final = None if content_block is None else content_block.get("type") + if content_block is None or not isinstance(block_type, str): + return block + ga_type: Final = _GA_ASSISTANT_CONTENT_TYPE_BY_BETA_TYPE.get(block_type) + return block if ga_type is None else {**content_block, "type": ga_type} + + +def beta_item_to_ga(item: Mapping[str, object]) -> Mapping[str, object]: + content: Final = item.get("content") + if item.get("role") != "assistant" or not isinstance(content, list): + return item + return {**item, "content": [_ga_assistant_content_block(block) for block in _JSON_ARRAY.validate_python(content)]} + + def _guardrail_event_id() -> str: return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}" @@ -270,6 +360,26 @@ def _gate_session_update( return {**client_session, "turn_detection": gate(None)} +@functools.cache +def _translation_audio_format_adapter() -> "TypeAdapter[str | RealtimeAudioFormats | None]": + from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats + + return TypeAdapter(str | RealtimeAudioFormats | None) + + +def _translation_audio_bytes_per_second(audio_format: "str | RealtimeAudioFormats | None") -> float | None: + from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU + + match audio_format: + case "pcm16" | AudioPCM(): + return 48000.0 + case "g711_ulaw" | "g711_alaw" | AudioPCMU() | AudioPCMA(): + return 8000.0 + case _: + pass + return None + + class RealtimeEventNormalizer(Protocol): def should_drop(self, event: object) -> bool: ... def normalize(self, event: dict) -> dict: ... @@ -294,6 +404,79 @@ DefaultLoggedRealTimeEventTypes: Final = [ ] +TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE: Final = "Translation sessions cannot enforce configured realtime guardrails" + + +def _guardrails_run_for_event_hooks( + request_data: Mapping[str, object], + event_hooks: Sequence["GuardrailEventHooks"], + callbacks: Sequence[object], +) -> bool: + from litellm.integrations.custom_guardrail import CustomGuardrail + + return any( + isinstance(cb, CustomGuardrail) + and any(cb.should_run_guardrail(data={**request_data, "stream": True}, event_type=et) for et in event_hooks) + for cb in callbacks + ) + + +def _opted_out_of_always_on_guardrail(callback: object, request_data: Mapping[str, object]) -> bool: + from litellm.integrations.custom_guardrail import CustomGuardrail + + return ( + isinstance(callback, CustomGuardrail) + and callback.event_hook is None + and callback.get_disable_global_guardrail(dict(request_data)) is True + ) + + +def _realtime_runner_guardrail( + callback: object, + check_data: Mapping[str, object], + event_hooks: Sequence["GuardrailEventHooks"], +) -> "CustomGuardrail | None": + from litellm.integrations.custom_guardrail import CustomGuardrail + + if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks: + return None + runs: Final = any( + callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks + ) + return callback if runs else None + + +def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool: + from litellm.types.guardrails import GuardrailEventHooks + + callbacks: Final = tuple( + callback for callback in litellm.callbacks if not _opted_out_of_always_on_guardrail(callback, request_data) + ) + if _guardrails_run_for_event_hooks(request_data, (GuardrailEventHooks.post_call,), callbacks): + return True + if any( + isinstance(bucket, Mapping) and "_guardrail_pipelines" in bucket + for bucket in (request_data.get("metadata"), request_data.get("litellm_metadata")) + ): + from litellm.proxy.utils import pipeline_managed_guardrail_names + + if pipeline_managed_guardrail_names(request_data, "pre_call") or pipeline_managed_guardrail_names( + request_data, "post_call" + ): + return True + return realtime_guardrails_apply(request_data, callbacks) + + +def realtime_guardrails_apply(request_data: Mapping[str, object], callbacks: Sequence[object]) -> bool: + from litellm.types.guardrails import GuardrailEventHooks + + check_data: Final = {**request_data, "stream": True} + runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call) + return any( + _realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks + ) + + class RealTimeStreaming: def __init__( self, @@ -303,12 +486,15 @@ class RealTimeStreaming: provider_config: BaseRealtimeConfig | None = None, model: str = "", user_api_key_dict: object | None = None, - request_data: dict | None = None, + request_data: Mapping[str, object] | None = None, backend_uses_beta_protocol: bool | None = None, force_transcription_model: str | None = None, event_normalizer: RealtimeEventNormalizer | None = None, usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + translation_session: bool = False, + bridges_beta_client_to_ga: bool = False, + drop_params: bool = False, lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS, lazy_setup_content_grace_seconds: float = REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, clock: Callable[[], float] = time.monotonic, @@ -326,14 +512,24 @@ class RealTimeStreaming: self._logging_worker = logging_worker self.messages: list[OpenAIRealtimeEvents] = [] self._backend_sent_frames: bool = False + self._backend_sent_non_error_frames: bool = False self.input_message: dict = {} self.input_messages: list[dict[str, str]] = [] self.session_tools: list[dict] = [] self.tool_calls: list[dict] = [] + self._is_translation_session = translation_session + self._bridges_beta_client_to_ga: Final = bridges_beta_client_to_ga + self._remaps_all_beta_fields_to_ga: Final = bridges_beta_client_to_ga or translation_session + self._drop_params: Final = drop_params + self._translation_input_seconds = 0.0 + self._translation_input_bytes_per_second = 48000.0 + self._translation_output_seconds = 0.0 + self._translation_output_bytes_per_second = 48000.0 + self._translation_usage_finalized = False # Detect whether the client is explicitly opting into the beta protocol. self._client_wants_beta = self.detect_beta_header(websocket) - self._backend_uses_beta_protocol = ( + self._backend_uses_beta_protocol = not translation_session and ( self._client_wants_beta if backend_uses_beta_protocol is None else backend_uses_beta_protocol ) @@ -352,7 +548,7 @@ class RealTimeStreaming: self.current_delta_type: ALL_DELTA_TYPES | None = None self.session_configuration_request: str | None = None self.user_api_key_dict = user_api_key_dict - self.request_data: dict = request_data or {} + self.request_data: Mapping[str, object] = request_data or {} # Violation counter for end_session_after_n_fails support self._violation_count: int = 0 # When a text message is blocked, hold the guardrail reason so the next @@ -384,6 +580,7 @@ class RealTimeStreaming: # their input_audio_transcription.completed usage drives duration-based cost. self._force_transcription_model = force_transcription_model self._is_transcription_session: bool = force_transcription_model is not None + self._bound_nested_transcription_model: str | None = None # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer). self._event_normalizer = event_normalizer self._gate_turn_detection: Final[_TurnDetectionGate] = ( @@ -407,6 +604,7 @@ class RealTimeStreaming: _MAX_BUFFERED_BYTES: int = 10 * 1024 * 1024 # 10 MB _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) + _RESPONSE_EVENT_TYPES: Final = frozenset(("response.created", "response.done")) _CLIENT_AUDIO_BUFFER_TYPES = frozenset( [ "input_audio_buffer.append", @@ -429,8 +627,8 @@ class RealTimeStreaming: _DEFERRED_SETUP_HELD_CONTENT_TYPES: Final = frozenset(["conversation.item.create"]) _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, - "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, - "g711_alaw": {"type": "audio/G711-alaw", "rate": 8000}, + "g711_ulaw": {"type": "audio/pcmu"}, + "g711_alaw": {"type": "audio/pcma"}, } # GA name → beta name (when client WebSocket includes OpenAI-Beta: realtime=v1) _GA_TO_BETA_EVENT_TYPES: dict[str, str] = { @@ -449,12 +647,12 @@ class RealTimeStreaming: def _should_store_message( self, - message_obj: dict | OpenAIRealtimeEvents, + message_obj: Mapping[str, object], ) -> bool: _msg_type: Final = message_obj["type"] if "type" in message_obj else None if self.logged_real_time_event_types == "*": return True - if _msg_type and _msg_type in self.logged_real_time_event_types: + if isinstance(_msg_type, str) and _msg_type in self.logged_real_time_event_types: return True return False @@ -555,7 +753,7 @@ class RealTimeStreaming: input_messages, so it is not duplicated into the response log here. """ try: - usage: Final = event_obj.get("usage") + usage: Final = as_str_mapping(event_obj.get("usage")) if usage is None: return # If this event type is already captured by store_message (e.g. the user @@ -623,6 +821,7 @@ class RealTimeStreaming: async def log_messages(self): """Log messages in list""" + self._finalize_translation_usage() if self.logging_obj: if self._usage_meter is not None: self.logging_obj.model_call_details[REALTIME_METERED_USAGE_KEY] = self._usage_meter.snapshot() @@ -641,6 +840,142 @@ class RealTimeStreaming: ) self.logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True + def _capture_translation_output_audio(self, event_obj: Mapping[str, object]) -> None: + if not self._is_translation_session: + return + if event_obj.get("type") == "session.closed": + usage: Final = as_str_mapping(event_obj.get("usage")) + output_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("output_seconds")) if usage is not None else None + ) + reported_input_seconds: Final = ( + normalized_audio_duration_seconds(usage.get("input_seconds")) if usage is not None else None + ) + input_seconds: Final = ( + reported_input_seconds + if reported_input_seconds is not None + else self._translation_input_seconds or None + ) + synthetic_output_seconds: Final = self._translation_output_seconds or None + resolved_output_seconds: Final = output_seconds if output_seconds is not None else synthetic_output_seconds + if input_seconds is not None or resolved_output_seconds is not None: + if self._should_store_message(event_obj): + supplemental_usage: Final = OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=float(input_seconds or 0.0) if reported_input_seconds is None else 0.0, + output_seconds=float(synthetic_output_seconds or 0.0) if output_seconds is None else 0.0, + ) + if ( + supplemental_usage.get("input_seconds", 0.0) > 0 + or supplemental_usage.get("output_seconds", 0.0) > 0 + ): + self.messages.append( + OpenAIRealtimeTranslationClosedEvent( + type="session.closed", + usage=supplemental_usage, + ) + ) + else: + normalized_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=input_seconds, + output_seconds=float(resolved_output_seconds or 0.0), + ) + if input_seconds is not None + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=float(resolved_output_seconds or 0.0) + ) + ) + self.messages.append( + OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=normalized_usage) + ) + self._translation_usage_finalized = True + return + self._capture_translation_audio_formats(event_obj) + if event_obj.get("type") not in ( + "session.output_audio.delta", + "response.output_audio.delta", + "response.audio.delta", + ): + return + delta: Final = event_obj.get("delta") + if not isinstance(delta, str): + return + try: + decoded: Final = base64.b64decode(delta, validate=True) + except (ValueError, TypeError): + return + self._translation_output_seconds += len(decoded) / self._translation_output_bytes_per_second + + def _capture_translation_input_audio(self, message: str) -> None: + if not self._is_translation_session: + return + try: + event: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return + if event.get("type") != "session.input_audio_buffer.append" or not isinstance(audio := event.get("audio"), str): + return + try: + decoded: Final = base64.b64decode(audio, validate=True) + except (ValueError, TypeError): + return + self._translation_input_seconds += len(decoded) / self._translation_input_bytes_per_second + + def _capture_translation_audio_formats(self, event_obj: Mapping[str, object]) -> None: + if event_obj.get("type") not in ("session.created", "session.updated"): + return + session: Final = as_str_mapping(event_obj.get("session")) + if session is None: + return + audio: Final = as_str_mapping(session.get("audio")) + if audio is None: + return + for direction, raw_configuration in (("input", audio.get("input")), ("output", audio.get("output"))): + if (configuration := as_str_mapping(raw_configuration)) is None: + continue + try: + if ( + bytes_per_second := _translation_audio_bytes_per_second( + _translation_audio_format_adapter().validate_python(configuration.get("format")) + ) + ) is None: + continue + except ValidationError: + continue + if direction == "input": + self._translation_input_bytes_per_second = bytes_per_second + else: + self._translation_output_bytes_per_second = bytes_per_second + + def _finalize_translation_usage(self) -> None: + if self._translation_usage_finalized: + return + for event in self.messages: + if event.get("type") != "session.closed": + continue + if (event_usage := as_str_mapping(event.get("usage"))) is not None and normalized_audio_duration_seconds( + event_usage.get("output_seconds") + ) is not None: + self._translation_usage_finalized = True + return + if self._translation_output_seconds == 0 and self._translation_input_seconds == 0: + return + synthetic_usage: Final = ( + OpenAIRealtimeTranslationDurationUsage( + type="duration", + input_seconds=self._translation_input_seconds, + output_seconds=self._translation_output_seconds, + ) + if self._translation_input_seconds > 0 + else OpenAIRealtimeTranslationDurationUsage( + type="duration", output_seconds=self._translation_output_seconds + ) + ) + self.messages.append(OpenAIRealtimeTranslationClosedEvent(type="session.closed", usage=synthetic_usage)) + self._translation_usage_finalized = True + async def _send_to_backend(self, message: str) -> bool: """Send a message to the backend WebSocket. @@ -653,10 +988,25 @@ class RealTimeStreaming: backend, False if the provider transformation produced no output and the message was effectively dropped. """ - message = self._enforce_transcription_session_model(message) + from litellm.proxy._types import ProxyException + + try: + authorized_message: Final = await self._apply_nested_transcription_model_policy(message) + except ProxyException as denied: + denial_message: Final = redact_internal_details_from_client_message(denied.message) + try: + await self.websocket.send_text(realtime_error_event(denial_message, error_type="permission_error")) + finally: + await self.websocket.close( + code=1008, + reason=websocket_close_reason( + denial_message, fallback="Nested transcription model is not authorized" + ), + ) + return False if self.provider_config: transformed: Final = self.provider_config.transform_realtime_request( - message, self.model, self.session_configuration_request + authorized_message, self.model, self.session_configuration_request ) sent = False for msg in transformed: @@ -691,14 +1041,94 @@ class RealTimeStreaming: if is_content_message: self._content_sent_after_setup = True sent = True + if sent: + self._capture_translation_input_audio(authorized_message) return sent - await self.backend_ws.send(message) + await self.backend_ws.send(authorized_message) if self._usage_meter is not None: - sent_event: Final = self._parse_backend_event(message) + sent_event: Final = self._parse_backend_event(authorized_message) if sent_event is not None: self._usage_meter.observe_client_event(sent_event) + self._capture_translation_input_audio(authorized_message) return True + async def _apply_nested_transcription_model_policy(self, message: str) -> str: + if self._force_transcription_model is not None: + return self._enforce_transcription_session_model(message) + if self._is_translation_session: + return await self._enforce_translation_nested_transcription_model(message) + return message + + def _session_update_message_obj(self, message: str) -> Mapping[str, object] | None: + try: + message_obj: Final = _decode_json_object(message) + except (json.JSONDecodeError, TypeError): + return None + if message_obj.get("type") not in ( + "session.update", + "transcription_session.update", + ): + return None + return message_obj + + def _nested_transcription_models_from_session( + self, + session: Mapping[str, object], + ) -> tuple[str, ...]: + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + nested_transcription: Final = audio_input.get("transcription") if audio_input is not None else None + nested_model: Final = self._transcription_model_value(nested_transcription) + flat_model: Final = self._transcription_model_value(session.get("input_audio_transcription")) + return tuple(dict.fromkeys(model for model in (nested_model, flat_model) if model is not None)) + + def _transcription_model_value(self, transcription_config: object) -> str | None: + if (fields := as_str_mapping(transcription_config)) is None: + return None + model: Final = fields.get("model") + if isinstance(model, str) and model: + return model + return None + + def _rewrite_session_update_transcription_model(self, message: str, authorized_model: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: + return message + session: Final = as_str_mapping(message_obj.get("session")) + if session is None: + return message + transcription: Final = as_str_mapping(session.get("input_audio_transcription")) + rewrite_flat: Final = transcription is not None and transcription.get("model") != authorized_model + audio: Final = as_str_mapping(session.get("audio")) + audio_input: Final = as_str_mapping(audio.get("input")) if audio is not None else None + nested_transcription: Final = ( + as_str_mapping(audio_input.get("transcription")) if audio_input is not None else None + ) + rewrite_nested: Final = ( + nested_transcription is not None and nested_transcription.get("model") != authorized_model + ) + if not rewrite_flat and not rewrite_nested: + return message + updated_session: Final = { + **session, + **( + {"input_audio_transcription": {**transcription, "model": authorized_model}} + if transcription is not None and rewrite_flat + else {} + ), + **( + { + "audio": { + **audio, + "input": {**audio_input, "transcription": {**nested_transcription, "model": authorized_model}}, + } + } + if audio is not None and audio_input is not None and nested_transcription is not None and rewrite_nested + else {} + ), + } + return json.dumps({**message_obj, "session": updated_session}) + def _builds_setup_lazily(self) -> bool: if self.provider_config is None: return False @@ -877,56 +1307,52 @@ class RealTimeStreaming: if self._force_transcription_model is None: return message - try: - message_obj: Final = _decode_json_object(message) - except (json.JSONDecodeError, TypeError): + message_obj: Final = self._session_update_message_obj(message) + if message_obj is None: return message - - if message_obj.get("type") not in ( - "session.update", - "transcription_session.update", - ): - return message - - session: Final = message_obj.get("session") - if not isinstance(session, dict): - return message - - if session.get("type") == "transcription": + session: Final = as_str_mapping(message_obj.get("session")) + if session is not None and session.get("type") == "transcription": self._is_transcription_session = True + return self._rewrite_session_update_transcription_model(message, self._force_transcription_model) - authorized_model: Final = self._force_transcription_model - changed = False + async def _enforce_translation_nested_transcription_model(self, message: str) -> str: + message_obj: Final = self._session_update_message_obj(message) + session: Final = as_str_mapping(message_obj.get("session")) if message_obj is not None else None + bound_model: Final = self._bound_nested_transcription_model + requested_models: Final = ( + tuple(model for model in self._nested_transcription_models_from_session(session) if model != bound_model) + if session is not None + else () + ) + if not requested_models: + return ( + message + if bound_model is None + else self._rewrite_session_update_transcription_model(message, bound_model) + ) - transcription: Final = session.get("input_audio_transcription") - if isinstance(transcription, dict) and transcription.get("model") != authorized_model: - session["input_audio_transcription"] = { - **transcription, - "model": authorized_model, - } - changed = True + valid_token: Final = self.user_api_key_dict - audio: Final = session.get("audio") - if isinstance(audio, dict): - audio_input: Final = audio.get("input") - if isinstance(audio_input, dict): - nested_transcription: Final = audio_input.get("transcription") - if isinstance(nested_transcription, dict) and nested_transcription.get("model") != authorized_model: - session["audio"] = { - **audio, - "input": { - **audio_input, - "transcription": { - **nested_transcription, - "model": authorized_model, - }, - }, - } - changed = True + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import can_key_call_resolved_model + from litellm.proxy.proxy_server import llm_model_list, llm_router + from litellm.proxy.realtime_endpoints.endpoints import nested_transcription_upstream_model - if not changed: + if not isinstance(valid_token, UserAPIKeyAuth): return message - return json.dumps(message_obj) + + for requested_model in requested_models: + await can_key_call_resolved_model( + model=requested_model, + valid_token=valid_token, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + upstream_model: Final = nested_transcription_upstream_model( + requested_models[0], valid_token.team_id, llm_router + ) + self._bound_nested_transcription_model = upstream_model + return self._rewrite_session_update_transcription_model(message, upstream_model) def _uses_deferred_backend_setup(self) -> bool: """True when setup is deferred until the client's first session.update.""" @@ -1047,7 +1473,19 @@ class RealTimeStreaming: return event def _event_to_client_json(self, event: dict) -> str: - return json.dumps(self._normalize_event_for_ga_client(event)) + return json.dumps(self._session_event_for_client(self._normalize_event_for_ga_client(event))) + + def _session_event_for_client(self, event: Mapping[str, object]) -> Mapping[str, object]: + if not self._bridges_beta_client_to_ga: + return event + event_type: Final = event.get("type") + if event_type in self._SESSION_EVENT_TYPES: + session: Final = _mapping_field(event, "session") + return event if session is None else {**event, "session": ga_session_to_beta(session)} + if event_type in self._RESPONSE_EVENT_TYPES: + response: Final = _mapping_field(event, "response") + return event if response is None else {**event, "response": ga_response_to_beta(response)} + return event async def _send_event_to_client(self, event: object, event_str: str) -> bool: if self._should_drop_event_from_client(event): @@ -1060,7 +1498,7 @@ class RealTimeStreaming: translated: Final = self._translate_event_to_beta(event) if translated is None: return False - await self.websocket.send_text(json.dumps(translated)) + await self.websocket.send_text(json.dumps(self._session_event_for_client(translated))) return True except Exception as e: verbose_logger.warning( @@ -1183,32 +1621,7 @@ class RealTimeStreaming: self, event_hooks: Sequence["GuardrailEventHooks"], ) -> bool: - """Return True if any callback would run for one of ``event_hooks``.""" - from litellm.integrations.custom_guardrail import CustomGuardrail - - return any( - isinstance(cb, CustomGuardrail) - and any( - cb.should_run_guardrail( - data={**self.request_data, "stream": True}, - event_type=et, - ) - for et in event_hooks - ) - for cb in litellm.callbacks - ) - - def _has_realtime_guardrails(self) -> bool: - """Return True if any callback is registered for realtime guardrail event types.""" - from litellm.types.guardrails import GuardrailEventHooks - - return self._has_realtime_guardrails_for_event_hooks( - [ - GuardrailEventHooks.realtime_input_transcription, - GuardrailEventHooks.pre_call, - GuardrailEventHooks.post_call, - ] - ) + return _guardrails_run_for_event_hooks(self.request_data, event_hooks, litellm.callbacks) def _has_audio_transcription_guardrails(self) -> bool: """Return True when a guardrail is configured for the audio/VAD transcript path. @@ -1222,8 +1635,12 @@ class RealTimeStreaming: return self._has_realtime_guardrails_for_event_hooks([GuardrailEventHooks.realtime_input_transcription]) def _should_disable_vad_auto_response(self) -> bool: - """Transcription-only sessions have no assistant turn to gate and reject realtime session updates.""" - return not self._is_transcription_session and self._has_audio_transcription_guardrails() + """Transcription and translation sessions have no assistant turn to gate and reject realtime session updates.""" + return ( + not self._is_transcription_session + and not self._is_translation_session + and self._has_audio_transcription_guardrails() + ) def _refuses_unscreened_audio(self, msg_type: str | None) -> bool: return ( @@ -1264,7 +1681,6 @@ class RealTimeStreaming: transcript completion uses ``realtime_input_transcription`` only; typed user messages and tool outputs use ``pre_call``. """ - from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.guardrails import GuardrailEventHooks if event_hooks is None: @@ -1273,14 +1689,11 @@ class RealTimeStreaming: _check_data: Final = {**self.request_data, "transcript": transcript, "stream": True} _already_run: Final[set] = set() - for callback in litellm.callbacks: - if not isinstance(callback, CustomGuardrail): - continue - if callback.use_native_lifecycle_hooks: - continue - if id(callback) in _already_run: - continue - if not any(callback.should_run_guardrail(data=_check_data, event_type=et) for et in _realtime_event_types): + runner_guardrails: Final = ( + _realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks + ) + for callback in runner_guardrails: + if callback is None or id(callback) in _already_run: continue _already_run.add(id(callback)) try: @@ -1369,7 +1782,10 @@ class RealTimeStreaming: async def _handle_provider_config_message(self, raw_response: str) -> None: """Process a backend message when a provider_config is set (transformed path).""" - returned_object: Final = self.provider_config.transform_realtime_response( + provider_config: Final = self.provider_config + if provider_config is None: + raise RuntimeError("Provider response handling requires a provider configuration") + returned_object: Final = provider_config.transform_realtime_response( raw_response, self.model, self.logging_obj, @@ -1398,6 +1814,7 @@ class RealTimeStreaming: continue if isinstance(event, dict): self._detect_transcription_session_from_backend(event) + self._capture_translation_output_audio(event) is_session_created_event = isinstance(event, dict) and event.get("type") == "session.created" if is_session_created_event: if self._uses_deferred_backend_setup() and not self._backend_setup_complete: @@ -1514,6 +1931,7 @@ class RealTimeStreaming: continue if self.provider_config: + self._backend_sent_non_error_frames = True try: await self._handle_provider_config_message(raw_response) except Exception as e: @@ -1521,6 +1939,8 @@ class RealTimeStreaming: continue else: event = self._parse_backend_event(raw_response) + if event is None or event.get("type") != "error": + self._backend_sent_non_error_frames = True if event is None: await self.websocket.send_text(raw_response) continue @@ -1535,6 +1955,7 @@ class RealTimeStreaming: verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event) continue + self._capture_translation_output_audio(event) if await self._handle_raw_backend_message(event, raw_response): continue @@ -1548,7 +1969,7 @@ class RealTimeStreaming: translated = self._translate_event_to_beta(event) if translated is None: continue - await self.websocket.send_text(json.dumps(translated)) + await self.websocket.send_text(json.dumps(self._session_event_for_client(translated))) async def backend_to_client_send_messages(self) -> BackendClose: import websockets @@ -1573,8 +1994,8 @@ class RealTimeStreaming: def _backend_refused_session(self, close: BackendClose) -> bool: return ( - close.code != 1000 - and not self._backend_sent_frames + not self._backend_sent_non_error_frames + and (close.code != 1000 or self._backend_sent_frames) and not self._holds_session_updates_until_first_content() ) @@ -1617,7 +2038,7 @@ class RealTimeStreaming: _detect_beta_header = detect_beta_header @staticmethod - def _remap_beta_session_to_ga(session: dict) -> dict: + def _remap_beta_session_to_ga(session: dict, all_beta_fields: bool = False, drop_temperature: bool = False) -> dict: """ Convert a beta-style session.update payload to the GA nested schema. @@ -1630,6 +2051,11 @@ class RealTimeStreaming: session.output_audio_format → session.audio.output.format (with type/rate) session.turn_detection → session.audio.input.turn_detection session.input_audio_transcription → session.audio.input.transcription + Only with all_beta_fields: + session.input_audio_noise_reduction → session.audio.input.noise_reduction + session.speed → session.audio.output.speed + session.max_response_output_tokens → session.max_output_tokens + session.temperature (dropped when drop_temperature) ───────────────────────────────────────────────────────────────────── Fields not in the mapping (instructions, tools, etc.) are passed through. GA clients that already use the nested shape are unaffected. @@ -1681,6 +2107,19 @@ class RealTimeStreaming: if "input_audio_transcription" in session: inp["transcription"] = session.pop("input_audio_transcription") + if all_beta_fields and "input_audio_noise_reduction" in session: + inp["noise_reduction"] = session.pop("input_audio_noise_reduction") + + if all_beta_fields and "speed" in session: + out["speed"] = session.pop("speed") + + if all_beta_fields and "max_response_output_tokens" in session: + max_tokens: Final = session.pop("max_response_output_tokens") + session.setdefault("max_output_tokens", max_tokens) + + if all_beta_fields and drop_temperature: + session.pop("temperature", None) + if inp: audio["input"] = inp if out: @@ -1699,6 +2138,53 @@ class RealTimeStreaming: return session + @staticmethod + def _remap_beta_response_to_ga(response: Mapping[str, object], drop_temperature: bool) -> dict[str, object]: + moved: Final = frozenset( + ("modalities", "voice", "output_audio_format", "max_response_output_tokens") + + (("temperature",) if drop_temperature else ()) + ) + raw_format: Final = response.get("output_audio_format") + output_audio: Final = { + **({} if "voice" not in response else {"voice": response["voice"]}), + **( + {} + if "output_audio_format" not in response + else { + "format": RealTimeStreaming._AUDIO_FORMAT_MAP.get(raw_format, raw_format) + if isinstance(raw_format, str) + else raw_format + } + ), + } + modalities: Final = response.get("modalities") + modality_names: Final = ( + frozenset(m.lower() for m in modalities if isinstance(m, str)) + if isinstance(modalities, list) + else frozenset() + ) + output_modalities: Final = ( + ["audio"] if "audio" in modality_names else ["text"] if "text" in modality_names else None + ) + existing_audio: Final = as_str_mapping(response.get("audio")) or {} + existing_output: Final = as_str_mapping(existing_audio.get("output")) or {} + return { + **{key: value for key, value in response.items() if key not in moved}, + **( + {} + if output_modalities is None or "output_modalities" in response + else {"output_modalities": output_modalities} + ), + **( + {} + if "max_response_output_tokens" not in response or "max_output_tokens" in response + else {"max_output_tokens": response["max_response_output_tokens"]} + ), + **( + {} if not output_audio else {"audio": {**existing_audio, "output": {**existing_output, **output_audio}}} + ), + } + @staticmethod def _translate_event_to_beta(event: dict) -> dict | None: """Translate a single GA event dict to its beta equivalent. @@ -1932,10 +2418,34 @@ class RealTimeStreaming: if msg_type == "session.update" and not self._backend_uses_beta_protocol: session = client_event.get("session", {}) if isinstance(session, dict): - session = self._remap_beta_session_to_ga(session) + session = self._remap_beta_session_to_ga( + session, + all_beta_fields=self._remaps_all_beta_fields_to_ga, + drop_temperature=self._drop_params, + ) + if self._is_translation_session: + session.pop("type", None) msg_obj["session"] = session message = json.dumps(msg_obj) + if ( + msg_type == "response.create" + and self._remaps_all_beta_fields_to_ga + and (beta_response := as_str_mapping(client_event.get("response"))) is not None + ): + msg_obj["response"] = self._remap_beta_response_to_ga( + beta_response, drop_temperature=self._drop_params + ) + message = json.dumps(msg_obj) + + if ( + msg_type == "conversation.item.create" + and self._bridges_beta_client_to_ga + and (beta_item := as_str_mapping(client_event.get("item"))) is not None + ): + msg_obj["item"] = beta_item_to_ga(beta_item) + message = json.dumps(msg_obj) + if msg_type == "session.update" and self._event_normalizer: session = client_event.get("session") if isinstance(session, dict): @@ -2019,6 +2529,12 @@ class RealTimeStreaming: return ClientLoopExit.CLIENT_DISCONNECTED async def bidirectional_forward(self) -> None: + if self._is_translation_session and translation_guardrails_apply(self.request_data): + await self.websocket.send_text( + realtime_error_event(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, error_type="guardrail_error") + ) + await self.websocket.close(code=1008, reason=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) + return forward_task: Final = asyncio.create_task(self.backend_to_client_send_messages()) client_task: Final = asyncio.create_task(self.client_ack_messages()) try: diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 2f8ecdcc9b9..078ec18e2db 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -171,6 +171,8 @@ def _redact_streaming_response(streaming_response): _redact_bedrock_guardrail_trace(streaming_response) elif isinstance(streaming_response, litellm.ResponsesAPIResponse): return perform_redaction({}, streaming_response) + elif isinstance(streaming_response, litellm.TranscriptionResponse): + streaming_response.text = REDACTED_BY_LITELLM elif hasattr(streaming_response, "output"): _redact_responses_api_output(streaming_response.output) if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None: diff --git a/litellm/llms/azure/audio_transcriptions.py b/litellm/llms/azure/audio_transcriptions.py index df8a0614665..386815412b5 100644 --- a/litellm/llms/azure/audio_transcriptions.py +++ b/litellm/llms/azure/audio_transcriptions.py @@ -1,11 +1,17 @@ -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final -from openai import AsyncAzureOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncStream, AzureOpenAI, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel from litellm._uuid import uuid from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name +from litellm.llms.base_llm.audio_transcription.transformation import ( + acreate_sdk_transcription, + create_sdk_transcription, + sdk_compatible_transcription_request_data, +) from litellm.types.utils import FileTypes from litellm.utils import ( TranscriptionResponse, @@ -38,13 +44,18 @@ class AzureAudioTranscription(AzureChatCompletion): atranscription: bool = False, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: + ) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ): data: Final = {"model": model, "file": audio_file, **optional_params} + sdk_data: Final = sdk_compatible_transcription_request_data(data) if atranscription is True: return self.async_audio_transcriptions( audio_file=audio_file, - data=data, + data=sdk_data, model_response=model_response, timeout=timeout, api_key=api_key, @@ -58,7 +69,7 @@ class AzureAudioTranscription(AzureChatCompletion): ) azure_client: Final = self.get_azure_openai_client( - api_version=api_version, + api_version=None if self.is_azure_v1_api_version(api_version) else api_version, api_base=api_base, api_key=api_key, model=model, @@ -84,10 +95,9 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - response: Final = azure_client.audio.transcriptions.create( - **data, - timeout=timeout, - ) + _, response = create_sdk_transcription(azure_client, sdk_data, timeout) + if isinstance(response, Stream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() @@ -114,7 +124,7 @@ class AzureAudioTranscription(AzureChatCompletion): self, audio_file: FileTypes, model: str, - data: dict, + data: Mapping[str, object], model_response: TranscriptionResponse, timeout: float, logging_obj: "LiteLLMLoggingObj", @@ -125,7 +135,7 @@ class AzureAudioTranscription(AzureChatCompletion): max_retries=None, litellm_params: dict | None = None, custom_llm_provider: str = "azure", - ) -> TranscriptionResponse: + ) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: response = None try: async_azure_client: Final = self.get_azure_openai_client( @@ -155,12 +165,9 @@ class AzureAudioTranscription(AzureChatCompletion): }, ) - raw_response: Final = await async_azure_client.audio.transcriptions.with_raw_response.create( - **data, timeout=timeout - ) - - headers: Final = dict(raw_response.headers) - response = raw_response.parse() + headers, response = await acreate_sdk_transcription(async_azure_client, data, timeout) + if isinstance(response, AsyncStream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 3d4576fd7ac..16ec4249244 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -5,9 +5,11 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. """ from collections.abc import Mapping +from dataclasses import dataclass from types import MappingProxyType from typing import Final, Protocol, cast +import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.url_utils import join_api_base @@ -48,17 +50,55 @@ async def forward_messages(client_ws: _RealtimeTextSink, backend_ws: _RealtimeTe pass +def azure_realtime_requires_ga(model: str) -> bool: + try: + azure_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + try: + openai_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="openai") + except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly + return False + openai_entry: Final = openai_model_info.get("provider_specific_entry") + return openai_entry is not None and openai_entry.get("realtime_ga_only") == 1 + azure_entry: Final = azure_model_info.get("provider_specific_entry") + return azure_entry is not None and azure_entry.get("realtime_ga_only") == 1 + + +def _is_ga_protocol(protocol: str | None) -> bool: + return protocol is not None and protocol.upper() in ("GA", "V1") + + +@dataclass(frozen=True, slots=True) +class AzureRealtimeProtocol: + name: str + bridges_beta_client_to_ga: bool = False + + def azure_realtime_protocol_for_client( configured_protocol: object, *, + model: str, + base_model: str | None = None, + realtime_mode: str, query_params: RealtimeQueryParams | None, websocket: ScopedWebSocket, -) -> str: +) -> AzureRealtimeProtocol: + serves_a_ga_only_session: Final = ( + realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription" + ) + client_wants_beta: Final = client_sent_openai_beta_realtime_header(websocket) if isinstance(configured_protocol, str) and configured_protocol: - return configured_protocol - if (query_params or {}).get("intent") == "transcription": - return "GA" - return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA" + return AzureRealtimeProtocol( + name=configured_protocol, + bridges_beta_client_to_ga=client_wants_beta + and not serves_a_ga_only_session + and _is_ga_protocol(configured_protocol), + ) + if serves_a_ga_only_session: + return AzureRealtimeProtocol(name="GA") + if azure_realtime_requires_ga(base_model or model): + return AzureRealtimeProtocol(name="GA", bridges_beta_client_to_ga=client_wants_beta) + return AzureRealtimeProtocol(name="beta" if client_wants_beta else "GA") class _ProxyClientWebSocket(Protocol): @@ -92,6 +132,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version: str | None, realtime_protocol: str | None = None, query_params: RealtimeQueryParams | None = None, + realtime_mode: str = "realtime", ) -> str: """ Construct Azure realtime WebSocket URL. @@ -104,6 +145,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): - "GA" or "v1": Uses /openai/v1/realtime (GA path) - "beta" or None: Uses /openai/realtime (beta path, default) query_params: Extra query params to forward (e.g. intent=transcription). + realtime_mode: Realtime surface to target, "realtime" or "translation". Returns: WebSocket URL string @@ -117,24 +159,29 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_base = api_base.replace("https://", "wss://") # Determine path based on realtime_protocol (case-insensitive) - _is_ga: Final = realtime_protocol is not None and realtime_protocol.upper() in ( - "GA", - "V1", - ) + _is_ga: Final = _is_ga_protocol(realtime_protocol) intent: Final = (query_params or {}).get("intent") - if _is_ga: - path = "/openai/v1/realtime" - query_parts = [] - if intent != "transcription" and (query_params is None or "model" in query_params): - query_parts.append(urlencode({"model": model})) - else: - # Default to beta path for backwards compatibility - path = "/openai/realtime" - query_parts = [urlencode({"api-version": api_version, "deployment": model})] + path: Final = ( + "/openai/v1/realtime/translations" + if realtime_mode == "translation" + else "/openai/v1/realtime" + if _is_ga + else "/openai/realtime" + ) + base_query_parts: Final = ( + (urlencode((("model", model),)),) + if realtime_mode == "translation" + else ( + (urlencode((("model", model),)),) + if intent != "transcription" and (query_params is None or "model" in query_params) + else () + ) + if _is_ga + else (urlencode((("api-version", api_version), ("deployment", model))),) + ) - if intent: - query_parts.append(urlencode({"intent": intent})) + query_parts: Final = (*base_query_parts, urlencode((("intent", intent),))) if intent else base_query_parts qs: Final = "&".join(query_parts) endpoint_url: Final = join_api_base(api_base, path) @@ -165,13 +212,16 @@ class AzureOpenAIRealtime(AzureChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", + bridges_beta_client_to_ga: bool = False, + drop_params: bool = False, ): import websockets from websockets.asyncio.client import ClientConnection if api_base is None: raise ValueError("api_base is required for Azure OpenAI calls") - backend_uses_beta_protocol: Final = realtime_protocol is None or realtime_protocol.upper() not in ("GA", "V1") + backend_uses_beta_protocol: Final = not _is_ga_protocol(realtime_protocol) if api_version is None and backend_uses_beta_protocol: raise ValueError("api_version is required for Azure OpenAI calls") @@ -181,6 +231,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): api_version, realtime_protocol=realtime_protocol, query_params=query_params, + realtime_mode=realtime_mode, ) auth_headers: Final = self.get_auth_headers(api_key=api_key, azure_ad_token=azure_ad_token) @@ -204,6 +255,9 @@ class AzureOpenAIRealtime(AzureChatCompletion): force_transcription_model=( model if (query_params or {}).get("intent") == "transcription" else None ), + translation_session=realtime_mode == "translation", + bridges_beta_client_to_ga=bridges_beta_client_to_ga, + drop_params=drop_params, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 545c3348182..64051b2c3c9 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -1,11 +1,20 @@ """Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" +from collections.abc import Mapping from typing import Final import litellm from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.azure.common_utils import get_azure_ad_token from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams + +_CALLER_AUTH_HEADERS: Final = frozenset({"authorization", "api-key"}) + + +def _azure_resource_base(api_base: str) -> str: + return api_base.rstrip("/").removesuffix("/openai/v1").removesuffix("/openai") def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str: @@ -21,6 +30,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_api_key(self, api_key: str | None, **kwargs: object) -> str: return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or "" + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + if api_key or any(name.lower() in _CALLER_AUTH_HEADERS for name in extra_headers or {}): + return extra_headers + azure_ad_token: Final = get_azure_ad_token(litellm_params) + if azure_ad_token is None: + return extra_headers + return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"} + def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets") @@ -30,20 +52,30 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): model: str, api_key: str | None = None, ) -> dict: - return { + validated_headers: Final = { **headers, - "api-key": api_key or "", "Content-Type": "application/json", } + if api_key: + validated_headers["api-key"] = api_key + return validated_headers def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: return _azure_realtime_v1_url(self.get_api_base(api_base), "calls") def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: - base: Final = self.get_api_base(api_base) + base: Final = _azure_resource_base(self.get_api_base(api_base)) version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}" + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/client_secrets") + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + return _azure_realtime_v1_url(self.get_api_base(api_base), "translations/calls") + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]: return { "Authorization": f"Bearer {ephemeral_key}", diff --git a/litellm/llms/base_llm/audio_transcription/transformation.py b/litellm/llms/base_llm/audio_transcription/transformation.py index e4bf148abf3..68365372e7e 100644 --- a/litellm/llms/base_llm/audio_transcription/transformation.py +++ b/litellm/llms/base_llm/audio_transcription/transformation.py @@ -1,9 +1,30 @@ from abc import ABC, abstractmethod +from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Final +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Final, + cast, # noqa: TID251 # SDK file handles have no runtime validation schema +) import httpx +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream +from openai.types.audio import ( + Transcription, + TranscriptionDiarized, + TranscriptionStreamEvent, + TranscriptionVerbose, +) +from openai.types.audio.transcription_create_params import ( + TranscriptionCreateParamsBase, + TranscriptionCreateParamsStreaming, +) +from pydantic import BaseModel, TypeAdapter +from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.types.llms.openai import ( AllMessageValues, @@ -11,6 +32,31 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse +_TRANSCRIPTION_REQUEST: Final = TypeAdapter(Mapping[str, object]) +_TRANSCRIPTION_TEXT: Final = TypeAdapter(str) +_TRANSCRIPTION_BODY: Final = TypeAdapter(dict[str, object]) +_TRANSCRIPTION_TYPES_BY_FORMAT: Final = MappingProxyType( + {"verbose_json": TranscriptionVerbose, "diarized_json": TranscriptionDiarized} +) + + +def sdk_compatible_transcription_request_data(data: object) -> Mapping[str, object]: + request_data: Final = _TRANSCRIPTION_REQUEST.validate_python(data) + extension_keys: Final = ("keywords", "languages") + extension_body: Final = {key: request_data[key] for key in extension_keys if key in request_data} + if not extension_body: + return request_data + + existing_extra_body: Final = request_data.get("extra_body") + return { + **{key: value for key, value in request_data.items() if key not in extension_keys}, + "extra_body": { + **(as_str_mapping(existing_extra_body) or {}), + **extension_body, + }, + } + + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer @@ -20,6 +66,77 @@ else: LiteLLMLoggingObj = Any +def sdk_compatible_transcription_response(response: object) -> BaseModel | str: + if isinstance(response, BaseModel): + return response + return _TRANSCRIPTION_TEXT.validate_python(response, strict=True) + + +def _content_type(http_response: httpx.Response) -> str: + return next(iter(http_response.headers.get_list("content-type")), "") + + +def _is_event_stream(http_response: httpx.Response) -> bool: + return _content_type(http_response).startswith("text/event-stream") + + +def _unstreamed_transcription_response(http_response: httpx.Response, response_format: object) -> BaseModel | str: + if not _content_type(http_response).startswith("application/json"): + return http_response.text + transcription_type: Final = _TRANSCRIPTION_TYPES_BY_FORMAT.get(str(response_format), Transcription) + return transcription_type.model_construct(None, **_TRANSCRIPTION_BODY.validate_json(http_response.content)) + + +def _non_streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsBase: + return cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsBase, {key: value for key, value in data.items() if key != "stream"} + ) + + +def _streaming_params(data: Mapping[str, object]) -> TranscriptionCreateParamsStreaming: + return cast( # cast-ok: SDK file handles have no runtime validation schema + TranscriptionCreateParamsStreaming, data + ) + + +def create_sdk_transcription( + client: OpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout +) -> tuple[dict[str, str], Stream[TranscriptionStreamEvent] | BaseModel | str]: + if not transcription_stream_requested(data.get("stream")): + raw_response: Final = client.audio.transcriptions.with_raw_response.create( + **_non_streaming_params(data), timeout=timeout + ) + return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) + raw_stream: Final = client.audio.transcriptions.with_raw_response.create( + **_streaming_params({**data, "stream": True}), timeout=timeout + ) + if _is_event_stream(raw_stream.http_response): + return dict(raw_stream.headers), raw_stream.parse(to=Stream[TranscriptionStreamEvent]) + raw_stream.http_response.read() + return dict(raw_stream.headers), _unstreamed_transcription_response( + raw_stream.http_response, data.get("response_format") + ) + + +async def acreate_sdk_transcription( + client: AsyncOpenAI, data: Mapping[str, object], timeout: float | httpx.Timeout +) -> tuple[dict[str, str], AsyncStream[TranscriptionStreamEvent] | BaseModel | str]: + if not transcription_stream_requested(data.get("stream")): + raw_response: Final = await client.audio.transcriptions.with_raw_response.create( + **_non_streaming_params(data), timeout=timeout + ) + return dict(raw_response.headers), sdk_compatible_transcription_response(raw_response.parse()) + raw_stream: Final = await client.audio.transcriptions.with_raw_response.create( + **_streaming_params({**data, "stream": True}), timeout=timeout + ) + if _is_event_stream(raw_stream.http_response): + return dict(raw_stream.headers), raw_stream.parse(to=AsyncStream[TranscriptionStreamEvent]) + await raw_stream.http_response.aread() + return dict(raw_stream.headers), _unstreamed_transcription_response( + raw_stream.http_response, data.get("response_format") + ) + + @dataclass class AudioTranscriptionRequestData: """ diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index a02054ec8cb..c1c943c2cd6 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow: """ from abc import ABC, abstractmethod +from collections.abc import Mapping +from typing import Final import httpx from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.types.router import GenericLiteLLMParams class BaseRealtimeHTTPConfig(ABC): @@ -50,6 +53,14 @@ class BaseRealtimeHTTPConfig(ABC): explicit api_key → litellm.api_key → env var → "" """ + def get_extra_headers( + self, + litellm_params: GenericLiteLLMParams, + api_key: str, + extra_headers: Mapping[str, object] | None, + ) -> Mapping[str, object] | None: + return extra_headers + # ------------------------------------------------------------------ # # client_secrets endpoint # # ------------------------------------------------------------------ # @@ -64,6 +75,12 @@ class BaseRealtimeHTTPConfig(ABC): base = base.removesuffix("/v1") return join_api_base(base, "v1/realtime/transcription_sessions") + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/client_secrets" + @abstractmethod def validate_environment( self, @@ -86,6 +103,10 @@ class BaseRealtimeHTTPConfig(ABC): """Return the full URL for POST /realtime/calls (SDP exchange).""" return join_api_base(api_base or "", "v1/realtime/calls") + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base: Final = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/translations/calls" + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: """ Build headers for the realtime_calls POST. diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index d247c5169bd..2f17dc6dd10 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -7097,7 +7097,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: @@ -7130,7 +7130,7 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: @@ -7149,9 +7149,8 @@ class BaseLLMHTTPHandler: api_version=api_version, ) - async def _async_realtime_session_post( + async def async_realtime_translation_client_secret_handler( self, - endpoint: Literal["client_secrets", "transcription_sessions"], api_base: str, api_key: str, request_data: dict[str, object], @@ -7159,7 +7158,35 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - extra_headers: dict[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, + client: HTTPHandler | AsyncHTTPHandler | None = None, + api_version: str | None = None, + ) -> httpx.Response: + return await self._async_realtime_session_post( + endpoint="translation_client_secrets", + api_base=api_base, + api_key=api_key, + request_data=request_data, + logging_obj=logging_obj, + timeout=timeout, + provider_config=provider_config, + model=model, + extra_headers=extra_headers, + client=client, + api_version=api_version, + ) + + async def _async_realtime_session_post( + self, + endpoint: Literal["client_secrets", "transcription_sessions", "translation_client_secrets"], + api_base: str, + api_key: str, + request_data: dict[str, object], + logging_obj: LiteLLMLoggingObj, + timeout: float | httpx.Timeout, + provider_config: BaseRealtimeHTTPConfig | None = None, + model: str | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, ) -> httpx.Response: @@ -7182,13 +7209,20 @@ class BaseLLMHTTPHandler: url = provider_config.get_transcription_session_url( api_base=api_base, model=model or "", api_version=api_version ) + elif endpoint == "translation_client_secrets": + url = provider_config.get_translation_client_secret_url( + api_base=api_base, model=model or "", api_version=api_version + ) else: url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) headers: dict[str, object] = provider_config.validate_environment( headers={}, model=model or "", api_key=api_key ) else: - url = join_api_base(api_base, f"v1/realtime/{endpoint}") + endpoint_path: Final = ( + "translations/client_secrets" if endpoint == "translation_client_secrets" else endpoint + ) + url = join_api_base(api_base, f"v1/realtime/{endpoint_path}") headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -7222,6 +7256,12 @@ class BaseLLMHTTPHandler: ) raise + @staticmethod + def _get_realtime_async_http_client(client: object | None) -> AsyncHTTPHandler: + if isinstance(client, AsyncHTTPHandler): + return client + return get_async_httpx_client(llm_provider=litellm.LlmProviders.OPENAI) + async def async_realtime_calls_handler( self, api_base: str, @@ -7231,10 +7271,11 @@ class BaseLLMHTTPHandler: timeout: float | httpx.Timeout, provider_config: BaseRealtimeHTTPConfig | None = None, model: str | None = None, - session_config: dict[str, object] | None = None, - extra_headers: dict[str, object] | None = None, + session_config: Mapping[str, object] | None = None, + extra_headers: Mapping[str, object] | None = None, client: HTTPHandler | AsyncHTTPHandler | None = None, api_version: str | None = None, + translation: bool = False, ) -> httpx.Response: """ Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. @@ -7246,18 +7287,30 @@ class BaseLLMHTTPHandler: - sdp: the SDP offer (text) - session: JSON string with {"type": "realtime", "model": "...", ...} """ - if client is None or not isinstance(client, AsyncHTTPHandler): - async_httpx_client = get_async_httpx_client( - llm_provider=litellm.LlmProviders.OPENAI, - ) - else: - async_httpx_client = client + session_data: Final[dict[str, object]] = { # mutable-ok: model and session type are resolved locally + **(session_config or {}) + } + if "type" not in session_data: + session_data["type"] = "translation" if translation else "realtime" + if "model" not in session_data and model: + session_data["model"] = model + + sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + + async_httpx_client: Final = self._get_realtime_async_http_client(client) if provider_config is not None: - url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) + url = ( + provider_config.get_translation_calls_url(api_base=api_base, model=model or "", api_version=api_version) + if translation + else provider_config.get_realtime_calls_url( + api_base=api_base, model=model or "", api_version=api_version + ) + ) headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key) else: - url = join_api_base(api_base, "v1/realtime/calls") + path: Final = "translations/calls" if translation else "calls" + url = join_api_base(api_base, f"v1/realtime/{path}") headers = { "Authorization": f"Bearer {openai_ephemeral_key}", } @@ -7265,14 +7318,8 @@ class BaseLLMHTTPHandler: if extra_headers: headers.update(extra_headers) - # Build multipart form data: sdp + session JSON - session_data: Final = session_config or {} - if "type" not in session_data: - session_data["type"] = "realtime" - if "model" not in session_data and model: - session_data["model"] = model - - sdp_text: Final = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + if translation: + headers["Content-Type"] = "application/sdp" files: Final = { "sdp": (None, sdp_text, "text/plain"), @@ -7290,12 +7337,14 @@ class BaseLLMHTTPHandler: ) try: - return await async_httpx_client.post( - url=url, - headers=headers, - files=files, - timeout=timeout, - ) + if translation: + return await async_httpx_client.post( + url=url, + headers=headers, + content=sdp_text, + timeout=timeout, + ) + return await async_httpx_client.post(url=url, headers=headers, files=files, timeout=timeout) except Exception as e: if provider_config is not None: raise self._handle_error( diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index 5be488f0f6e..40ead28d952 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -141,7 +141,7 @@ class OpenAIRealtime(OpenAIChatCompletion): return ssl_config - def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str: + def _construct_url(self, api_base: str, query_params: RealtimeQueryParams, realtime_mode: str = "realtime") -> str: """ Construct the backend websocket URL with the client's query parameters. @@ -158,7 +158,8 @@ class OpenAIRealtime(OpenAIChatCompletion): api_base = api_base.replace("http://", "ws://") url = URL(api_base) # Set the correct path - url = url.copy_with(path="/v1/realtime") + path: Final = "/v1/realtime/translations" if realtime_mode == "translation" else "/v1/realtime" + url = url.copy_with(path=path) upstream_params: Final = tuple( (key, value) for key, value in query_params.items() if not (drops_model and key == "model") ) @@ -197,6 +198,8 @@ class OpenAIRealtime(OpenAIChatCompletion): query_params: RealtimeQueryParams | None = None, user_api_key_dict: object | None = None, litellm_metadata: dict | None = None, + realtime_mode: str = "realtime", + drop_params: bool = False, **kwargs: object, ): import websockets @@ -210,13 +213,15 @@ class OpenAIRealtime(OpenAIChatCompletion): # Use all query params if provided, else fallback to just model if query_params is None: query_params = {"model": model} - url: Final = self._construct_url(api_base, query_params) + url: Final = self._construct_url(api_base, query_params, realtime_mode=realtime_mode) try: # Get provider-specific SSL configuration ssl_config: Final = self._get_ssl_config(url) - openai_beta_realtime: Final = client_sent_openai_beta_realtime_header(websocket) + openai_beta_realtime: Final = realtime_mode != "translation" and client_sent_openai_beta_realtime_header( + websocket + ) if not openai_beta_realtime: verbose_logger.debug( "OpenAI Realtime: connecting with GA protocol (no OpenAI-Beta header). " @@ -253,6 +258,8 @@ class OpenAIRealtime(OpenAIChatCompletion): ), event_normalizer=self._make_event_normalizer(), usage_meter=self._make_usage_meter(), + translation_session=realtime_mode == "translation", + drop_params=drop_params, ) await realtime_streaming.bidirectional_forward() diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 49d33582c6f..ab32ae6085e 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -28,6 +28,18 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): base = base.removesuffix("/v1") return join_api_base(base, "v1/realtime/transcription_sessions") + def get_translation_client_secret_url( + self, api_base: str | None, model: str, api_version: str | None = None + ) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/client_secrets" + + def get_translation_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + base = base.removesuffix("/v1") + return f"{base}/v1/realtime/translations/calls" + def validate_environment( self, headers: dict, diff --git a/litellm/llms/openai/transcriptions/gpt_transformation.py b/litellm/llms/openai/transcriptions/gpt_transformation.py index e7f8dd04516..b9eaa204567 100644 --- a/litellm/llms/openai/transcriptions/gpt_transformation.py +++ b/litellm/llms/openai/transcriptions/gpt_transformation.py @@ -26,8 +26,8 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): self, model: str, audio_file: FileTypes, - optional_params: dict, - litellm_params: dict, + optional_params: dict[str, object], + litellm_params: dict[str, object], ) -> AudioTranscriptionRequestData: """ Transform the audio transcription request @@ -37,3 +37,14 @@ class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): return AudioTranscriptionRequestData( data=data, ) + + +class OpenAIGPTTranscribeAudioTranscriptionConfig(OpenAIGPTAudioTranscriptionConfig): + def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: + return [ + "prompt", + "response_format", + "keywords", + "languages", + "stream", + ] diff --git a/litellm/llms/openai/transcriptions/handler.py b/litellm/llms/openai/transcriptions/handler.py index 48a44575989..23c5e70a3cc 100644 --- a/litellm/llms/openai/transcriptions/handler.py +++ b/litellm/llms/openai/transcriptions/handler.py @@ -1,8 +1,9 @@ -from collections.abc import Mapping +from collections.abc import Coroutine, Mapping from typing import TYPE_CHECKING, Final, Optional, cast import httpx -from openai import AsyncOpenAI, OpenAI +from openai import AsyncOpenAI, AsyncStream, OpenAI, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel if TYPE_CHECKING: @@ -12,6 +13,9 @@ from litellm.litellm_core_utils.core_helpers import set_provider_response_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, + acreate_sdk_transcription, + create_sdk_transcription, + sdk_compatible_transcription_request_data, ) from litellm.types.utils import FileTypes from litellm.utils import ( @@ -28,31 +32,18 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): async def make_openai_audio_transcriptions_request( self, openai_aclient: AsyncOpenAI, - data: dict, + data: Mapping[str, object], timeout: float | httpx.Timeout, ): - try: - raw_response = await openai_aclient.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) - headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() - - return headers, response - except Exception as e: - raise e + return await acreate_sdk_transcription(openai_aclient, sdk_compatible_transcription_request_data(data), timeout) def make_sync_openai_audio_transcriptions_request( self, openai_client: OpenAI, - data: dict, + data: Mapping[str, object], timeout: float | httpx.Timeout, ): - try: - raw_response: Final = openai_client.audio.transcriptions.with_raw_response.create(**data, timeout=timeout) - headers: Final = dict(raw_response.headers) - response: Final = raw_response.parse() - return headers, response - except Exception as e: - raise e + return create_sdk_transcription(openai_client, sdk_compatible_transcription_request_data(data), timeout) def audio_transcriptions( self, @@ -70,7 +61,11 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): atranscription: bool = False, provider_config: BaseAudioTranscriptionConfig | None = None, shared_session: Optional["ClientSession"] = None, - ) -> TranscriptionResponse: + ) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[object, object, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ): """ Handle audio transcription request """ @@ -128,6 +123,9 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): ) logging_obj.model_call_details["response_headers"] = headers + if isinstance(response, Stream): + return response + if isinstance(response, BaseModel): stringified_response = response.model_dump() else: @@ -153,7 +151,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): async def async_audio_transcriptions( self, audio_file: FileTypes, - data: dict, + data: Mapping[str, object], model_response: TranscriptionResponse, timeout: float, logging_obj: LiteLLMLoggingObj, @@ -192,6 +190,8 @@ class OpenAIAudioTranscription(OpenAIChatCompletion): timeout=timeout, ) logging_obj.model_call_details["response_headers"] = headers + if isinstance(response, AsyncStream): + return response if isinstance(response, BaseModel): stringified_response = response.model_dump() else: diff --git a/litellm/main.py b/litellm/main.py index b9fd86274b4..fc18aaa356d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -37,6 +37,8 @@ if TYPE_CHECKING: import dotenv import httpx import openai +from openai import AsyncStream, Stream +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, TypeAdapter, ValidationError from typing_extensions import assert_never, overload @@ -1001,7 +1003,7 @@ def _model_response_from_cached_dict(cached_response_dict: Mapping[str, object]) def _transcription_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TranscriptionResponse: - return TranscriptionResponse(**cached_response_dict) + return TranscriptionResponse.model_validate(cached_response_dict) def _text_completion_response_from_cached_dict(cached_response_dict: Mapping[str, object]) -> TextCompletionResponse: @@ -8501,7 +8503,10 @@ async def amoderation( @client -async def atranscription(*args, **kwargs) -> TranscriptionResponse: +async def atranscription( + *args, # noqa: ANN002 # public SDK wrapper preserves positional call compatibility + **kwargs, # noqa: ANN003 # kwargs-ok: public SDK wrapper preserves keyword call compatibility +) -> TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]: """ Calls openai + azure whisper endpoints. @@ -8545,6 +8550,14 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: else: # Call the synchronous function using run_in_executor response = await loop.run_in_executor(None, func_with_context) + if isinstance(response, AsyncStream): + if file is not None: + calculated_duration = calculate_request_duration(file) + if calculated_duration is not None: + response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it + calculated_duration + ) + return response if not isinstance(response, TranscriptionResponse): raise ValueError( f"Invalid response from transcription provider, expected TranscriptionResponse, but got {type(response)}" @@ -8557,9 +8570,9 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: if response is not None and not isinstance(response, Coroutine) and file is not None: existing_duration: Final = getattr(response, "duration", None) if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) - if calculated_duration is not None: - response.hidden_params["audio_transcription_duration"] = calculated_duration + sync_calculated_duration: Final = calculate_request_duration(file) + if sync_calculated_duration is not None: + response.set_audio_transcription_duration(sync_calculated_duration) return response except Exception as e: @@ -8573,6 +8586,42 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: ) +def _validate_gpt_transcription_request( + model: str, + pricing_model: str, + custom_llm_provider: str, + response_format: str | None, + drop_params: object, +) -> str | None: + model_cost_key: Final = next( + ( + key + for key in (f"{custom_llm_provider}/{pricing_model}", pricing_model) + if (as_str_mapping(litellm.model_cost.get(key)) or {}).get("litellm_provider") == custom_llm_provider + ), + None, + ) + model_info: Final = ( + get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider) + if model_cost_key is not None + else None + ) + provider_specific_entry: Final = model_info.get("provider_specific_entry") if model_info is not None else None + if ( + provider_specific_entry is None + or provider_specific_entry.get("transcription_json_or_text_only") != 1 + or response_format in (None, "json", "text") + ): + return response_format + if litellm.drop_params is True or normalize_drop_params(drop_params) is True: + return None + raise litellm.UnsupportedParamsError( + message=f"{model} only supports response_format 'json' or 'text'", + model=model, + llm_provider=custom_llm_provider, + ) + + @client def transcription( model: str, @@ -8592,8 +8641,15 @@ def transcription( max_retries: int | None = None, custom_llm_provider=None, base_url: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, + stream: bool | None = None, **kwargs, -) -> TranscriptionResponse | Coroutine[object, object, TranscriptionResponse]: +) -> ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] +): """ Calls openai + azure whisper endpoints. @@ -8631,19 +8687,36 @@ def transcription( api_key = dynamic_api_key if dynamic_api_key is not None else api_key + litellm_params_dict: Final = get_litellm_params(**kwargs) + configured_base_model: Final = litellm_params_dict.get("base_model") + pricing_model: Final = ( + configured_base_model.removeprefix(f"{custom_llm_provider}/") + if isinstance(configured_base_model, str) + else model + ) + + supported_response_format: Final = _validate_gpt_transcription_request( + model=model, + pricing_model=pricing_model, + custom_llm_provider=custom_llm_provider, + response_format=response_format, + drop_params=kwargs.get("drop_params"), + ) + optional_params: Final = get_optional_params_transcription( model=model, language=language, + languages=languages, + keywords=keywords, prompt=prompt, - response_format=response_format, + response_format=supported_response_format, timestamp_granularities=timestamp_granularities, temperature=temperature, + stream=stream, custom_llm_provider=custom_llm_provider, **non_default_params, ) - litellm_params_dict: Final = get_litellm_params(**kwargs) - litellm_logging_obj.update_environment_variables( model=model, user=user, @@ -8660,11 +8733,17 @@ def transcription( custom_llm_provider=custom_llm_provider, ) - response: TranscriptionResponse | Coroutine[object, object, TranscriptionResponse] + response: ( + TranscriptionResponse + | Stream[TranscriptionStreamEvent] + | AsyncStream[TranscriptionStreamEvent] + | Coroutine[Any, Any, TranscriptionResponse | AsyncStream[TranscriptionStreamEvent]] + ) provider_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( model=model, provider=LlmProviders(custom_llm_provider), + base_model=pricing_model, ) uses_openai_transport: Final = custom_llm_provider in OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS and not ( @@ -8675,7 +8754,7 @@ def transcription( # azure configs api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") - api_version = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + azure_api_version: Final = api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") azure_ad_token: Final = kwargs.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN") @@ -8694,7 +8773,7 @@ def transcription( logging_obj=litellm_logging_obj, api_base=api_base, api_key=api_key, - api_version=api_version, + api_version=azure_api_version, azure_ad_token=azure_ad_token, max_retries=max_retries, litellm_params=litellm_params_dict, @@ -8831,15 +8910,17 @@ def transcription( # Store duration in _hidden_params for cost calculation without # exposing it in the response body (see sync path comment above). - if not isinstance(response, Coroutine): - existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: - calculated_duration: Final = calculate_request_duration(file) + if not isinstance(response, Coroutine) and ( + isinstance(response, (Stream, AsyncStream)) or getattr(response, "duration", None) is None + ): + calculated_duration: Final = calculate_request_duration(file) + if isinstance(response, (Stream, AsyncStream)): if calculated_duration is not None: - response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + response._litellm_audio_duration = ( # pyright: ignore[reportAttributeAccessIssue] # SDK omits it + calculated_duration ) - response_hidden_params["audio_transcription_duration"] = calculated_duration + elif calculated_duration is not None: + response.set_audio_transcription_duration(calculated_duration) return response diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 376cd816c66..d7c829f6e2d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1,4 +1,168 @@ { + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -6424,6 +6588,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6458,6 +6625,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6491,6 +6661,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6525,6 +6698,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6547,6 +6723,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 3.2e-05, @@ -6556,6 +6733,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6575,12 +6755,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 1e-05, @@ -6590,6 +6773,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6609,6 +6795,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -6626,6 +6814,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6659,6 +6850,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6791,6 +6985,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -29238,9 +29435,9 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -39069,6 +39266,9 @@ "max_input_tokens": 128000, "max_output_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -39124,6 +39324,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -39160,6 +39362,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -52587,9 +52791,9 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -63405,6 +63609,10 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -63417,7 +63625,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -63435,7 +63644,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -63469,7 +63679,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, @@ -71340,6 +71551,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/realtime" @@ -71397,7 +71611,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-transcribe": { "deprecation_date": "2028-02-01", @@ -71407,7 +71625,8 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" + "/v1/realtime/transcription_sessions", + "/v1/realtime" ], "supported_modalities": [ "text", @@ -71416,7 +71635,13 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, + "supports_native_streaming": true }, "azure/gpt-realtime-translate": { "deprecation_date": "2027-11-09", @@ -71434,7 +71659,11 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index b14f43635b2..2f613396413 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -504,10 +504,13 @@ def _lazy_lock(app: "FastAPI", module_path: str) -> asyncio.Lock: def _register_feature(app: "FastAPI", feat: LazyFeature, module: object, features: tuple[LazyFeature, ...]) -> None: - before: Final = len(app.router.routes) + routes_before: Final = frozenset(id(route) for route in app.router.routes) feat.register_fn(app, module) lazy_routes: Final[Mapping[str, tuple[BaseRoute, ...]]] = MappingProxyType( - {**_lazy_routes(app), feat.module_path: tuple(app.router.routes[before:])} + { + **_lazy_routes(app), + feat.module_path: tuple(route for route in app.router.routes if id(route) not in routes_before), + } ) app.state.lazy_routes = lazy_routes # rebind-ok: the app owns the record of which routes each feature added app.router.routes[:] = _route_table( # rebind-ok: the app owns its route table diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index ba6494c3bda..5cc1c53d396 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -34367,6 +34367,7 @@ }, "/openai/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post", "responses": { "200": { @@ -34386,6 +34387,7 @@ }, "/openai/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post", "responses": { "200": { @@ -54572,6 +54574,7 @@ "paths": { "/openai/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_openai_v1_realtime_calls_post_2", "responses": { "200": { @@ -54591,6 +54594,7 @@ }, "/openai/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_openai_v1_realtime_client_secrets_post_2", "responses": { "200": { @@ -54644,6 +54648,7 @@ }, "/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_realtime_calls_post", "responses": { "200": { @@ -54663,6 +54668,7 @@ }, "/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_realtime_client_secrets_post", "responses": { "200": { @@ -54714,8 +54720,489 @@ ] } }, + "/realtime/translations/calls": { + "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", + "operationId": "proxy_realtime_calls_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "WebRTC SDP answer" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/realtime/translations/client_secrets": { + "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", + "operationId": "create_realtime_client_secret_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } + }, "/v1/realtime/calls": { "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", "operationId": "proxy_realtime_calls_v1_realtime_calls_post", "responses": { "200": { @@ -54735,6 +55222,7 @@ }, "/v1/realtime/client_secrets": { "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", "operationId": "create_realtime_client_secret_v1_realtime_client_secrets_post", "responses": { "200": { @@ -54785,6 +55273,486 @@ "realtime" ] } + }, + "/v1/realtime/translations/calls": { + "post": { + "description": "Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets", + "operationId": "proxy_realtime_calls_v1_realtime_translations_calls_post", + "requestBody": { + "content": { + "application/sdp": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "WebRTC SDP answer" + } + }, + "summary": "Proxy Realtime Calls", + "tags": [ + "realtime" + ] + } + }, + "/v1/realtime/translations/client_secrets": { + "post": { + "description": "Create a short-lived client token for a Realtime conversation, transcription, or translation session", + "operationId": "create_realtime_client_secret_v1_realtime_translations_client_secrets_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "gpt-realtime-translate", + "session": { + "model": "gpt-realtime-translate", + "type": "translation" + } + }, + "schema": { + "description": "Request body for POST /v1/realtime/client_secrets.\n\nLiteLLM also accepts a top-level `model` field for routing when\nsession.model is absent (LiteLLM extension, not forwarded to OpenAI).", + "properties": { + "expires_after": { + "anyOf": [ + { + "description": "Expiration config for a client secret.", + "properties": { + "anchor": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "created_at", + "title": "Anchor" + }, + "seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Seconds" + } + }, + "title": "RealtimeExpiresAfter", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "session": { + "anyOf": [ + { + "additionalProperties": true, + "description": "Session configuration nested inside the client_secrets request body.\n\nMirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and\nRealtimeTranscriptionSessionCreateRequest (type=transcription).\nExtra/unknown fields are passed through unchanged.", + "properties": { + "audio": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "noise_reduction": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Noise Reduction" + }, + "transcription": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "delay": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Delay" + }, + "keywords": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Keywords" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "languages": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Languages" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "prompt": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Prompt" + } + }, + "title": "RealtimeAudioTranscriptionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "turn_detection": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Turn Detection" + } + }, + "title": "RealtimeAudioInputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "output": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "format": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Format" + }, + "language": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Language" + }, + "voice": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Voice" + } + }, + "title": "RealtimeAudioOutputConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeSessionAudioConfig", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "include": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Include" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Instructions" + }, + "max_output_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Max Output Tokens" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "output_modalities": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Output Modalities" + }, + "prompt": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "Prompt" + }, + "tool_choice": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tool Choice" + }, + "tools": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Tools" + }, + "tracing": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Tracing" + }, + "truncation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Truncation" + }, + "type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Type" + } + }, + "title": "RealtimeSessionConfig", + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "title": "RealtimeClientSecretRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RealtimeClientSecretResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Create Realtime Client Secret", + "tags": [ + "realtime" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c9960d794c6..fc1e7128e10 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -332,6 +332,8 @@ class LiteLLMRoutes(enum.Enum): "/realtime?{model}", "/v1/realtime?{model}", "/openai/v1/realtime?{model}", + "/realtime/translations", + "/v1/realtime/translations", # realtime (GA WebRTC HTTP routes) "/realtime/client_secrets", "/v1/realtime/client_secrets", @@ -342,6 +344,10 @@ class LiteLLMRoutes(enum.Enum): "/realtime/transcription_sessions", "/v1/realtime/transcription_sessions", "/openai/v1/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", # responses API "/responses", "/v1/responses", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index cdbbd6029fa..57e99edecba 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -11,7 +11,15 @@ from litellm.proxy.agent_endpoints.managed_identity import raise_identity_failur from litellm.types.agents import AgentResponse from litellm.types.proxy.agent_identity import AgentIdentityFailure, ManagedAgentContext -_MANAGED_REALTIME_ROUTES: Final = frozenset(("/realtime", "/v1/realtime", "/openai/v1/realtime")) +_MANAGED_REALTIME_ROUTES: Final = frozenset( + ( + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + "/realtime/translations", + "/v1/realtime/translations", + ) +) _MANAGED_LIVE_ROUTES: Final = frozenset(("/live/sessions", "/v1/live/sessions", "/openai/v1/live/sessions")) _MANAGED_MODEL_ROUTES: Final = frozenset( f"{prefix}/{operation}" @@ -112,7 +120,11 @@ def managed_inference_request( if route in _MANAGED_LIVE_ROUTES: return dict(body) if route in _MANAGED_REALTIME_ROUTES: - model: Final = query_model or body.get("model") + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + model: Final = resolve_realtime_route_model( + query_model if isinstance(query_model, str) else None, None, route.endswith("/realtime/translations") + ) if not isinstance(model, str) or not model: raise_identity_failure( AgentIdentityFailure(message="Managed inference requires an explicit or configured model") diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9350c9c935c..161ff2fdfcd 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -1971,6 +1971,7 @@ _MODEL_ROUTING_COMPLETION_MODEL_ROUTE_MARKERS: Final = ("/evals",) # can_key_call_model() validates the real target model. _MODEL_ROUTING_SESSION_MODEL_ROUTE_MARKERS: Final = ( "/realtime/client_secrets", + "/realtime/translations/client_secrets", "/realtime/calls", ) _MODEL_ROUTING_ID_FIELDS: Final = ( diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a863de8b0f8..1dd0647d25f 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -148,6 +148,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # leg populate_request_with_path_params, read_raw_json_body, read_request_body, + resolve_realtime_route_model, rewrite_request_model, safe_get_request_headers, safe_get_request_query_params, @@ -1246,11 +1247,21 @@ def _apply_budget_limits_to_end_user_params( async def user_api_key_auth_websocket(websocket: WebSocket) -> UserAPIKeyAuth: - return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model")) + return await user_api_key_auth_websocket_for_model(websocket, model=websocket.query_params.get("model", "")) + + +async def user_api_key_auth_realtime_websocket(websocket: WebSocket) -> UserAPIKeyAuth: + route_path: Final[object] = websocket.scope.get("path") + model: Final = resolve_realtime_route_model( + model=websocket.query_params.get("model"), + intent=websocket.query_params.get("intent"), + is_translation=isinstance(route_path, str) and route_path.endswith("/realtime/translations"), + ) + return await user_api_key_auth_websocket_for_model(websocket, model=model or "") async def user_api_key_auth_live_websocket(websocket: WebSocket) -> UserAPIKeyAuth: - return await user_api_key_auth_websocket_for_model(websocket, model=None) + return await user_api_key_auth_websocket_for_model(websocket, model="") def _has_required_baggage_tag_mapping() -> bool: @@ -1266,7 +1277,7 @@ def _has_required_baggage_tag_mapping() -> bool: async def user_api_key_auth_websocket_for_model( - websocket: WebSocket, model: str | None, *, close_on_auth_error: bool = True + websocket: WebSocket, model: str, *, close_on_auth_error: bool = True ) -> UserAPIKeyAuth: ws_scope: Final = websocket.scope or {} scope_headers: Final = list(ws_scope.get("headers") or []) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 994b3a96799..5c538c0f6bd 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -171,7 +171,9 @@ ProxyRouteType: TypeAlias = Literal[ "_arealtime", "_aresponses_websocket", "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "aget_responses", "adelete_responses", "acancel_responses", diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 81ff460613c..baf586acef6 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -37,6 +37,16 @@ _DECODED_JSON: Final = TypeAdapter(object) _JSON_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object]) +def resolve_realtime_route_model(model: str | None, intent: str | None, is_translation: bool) -> str | None: + if model is not None: + return model + if is_translation: + return "gpt-realtime-translate" + if intent == "transcription": + return "gpt-realtime-whisper" + return None + + def resolve_inference_model( body_model: object, settings: Mapping[str, object], @@ -572,14 +582,16 @@ async def get_form_data(request: Request) -> dict[str, Any]: Handles when OpenAI SDKs pass form keys as `timestamp_granularities[]="word"` instead of `timestamp_granularities=["word", "sentence"]` """ form: Final = await request.form() - parsed_form_data: Final[dict[str, Any]] = {} - for key, value in form.multi_items(): # not dict(form), which keeps only the last repeat - if key.endswith("[]"): - clean_key = key[:-2] - parsed_form_data.setdefault(clean_key, []).append(value) - else: - parsed_form_data[key] = value - return parsed_form_data + form_items: Final = tuple(form.multi_items() if hasattr(form, "multi_items") else form.items()) + array_keys: Final = frozenset(key[:-2] for key, _ in form_items if key.endswith("[]")) + normalized_items: Final = tuple((key.removesuffix("[]"), value) for key, value in form_items) + normalized_keys: Final = tuple(dict.fromkeys(key for key, _ in normalized_items)) + return { + key: [value for item_key, value in normalized_items if item_key == key] + if key in array_keys + else next(value for item_key, value in reversed(normalized_items) if item_key == key) + for key in normalized_keys + } async def convert_upload_files_to_file_data( diff --git a/litellm/proxy/common_utils/realtime_utils.py b/litellm/proxy/common_utils/realtime_utils.py index e96685bdeae..d4fd4556ed4 100644 --- a/litellm/proxy/common_utils/realtime_utils.py +++ b/litellm/proxy/common_utils/realtime_utils.py @@ -11,7 +11,7 @@ def realtime_request_body(model: str | None) -> bytes: Generate the realtime websocket request body. Cached with LRU semantics to avoid repeated string formatting work while keeping memory usage bounded. """ - return json.dumps({"model": model or ""}).encode() + return json.dumps({"model": model}).encode() _realtime_request_body: Final = realtime_request_body diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ae4f5072a26..424e68ae3bc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -60,6 +60,7 @@ import anyio import anyio.lowlevel import websockets import websockets.exceptions +from openai.types.audio import TranscriptionStreamEvent from pydantic import BaseModel, ConfigDict, Json, JsonValue, TypeAdapter, ValidationError, with_config from pydantic.fields import FieldInfo, PydanticUndefined from pydantic.main import IncEx @@ -215,6 +216,7 @@ from litellm.types.utils import ( TextCompletionResponse, TokenCountMessagesFormat, TokenCountResponse, + TranscriptionResponse, echoed_cost_map_pricing_fields, is_server_derived_pricing_key, pricing_override_fields, @@ -324,6 +326,7 @@ from litellm._internal_context import service_target, with_service_target from litellm._logging import ( STARTUP_OUTPUT_SETTINGS, print_startup_message, + redact_internal_details_from_client_message, redact_string, verbose_proxy_logger, verbose_router_logger, @@ -373,6 +376,11 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + TRANSCRIPTION_STREAM_DONE_FRAME, + GuardedTranscriptionReplay, + guarded_transcription_events, +) from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, @@ -471,6 +479,7 @@ from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # lega user_api_key_auth, user_api_key_auth_accepted_websocket_for_model, user_api_key_auth_live_websocket, + user_api_key_auth_realtime_websocket, user_api_key_auth_websocket, ) from litellm.proxy.batches_endpoints.endpoints import router as batches_router @@ -536,6 +545,9 @@ from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401, RUF10 resolve_inference_model, safe_get_request_headers, ) +from litellm.proxy.common_utils.http_parsing_utils import ( + resolve_realtime_route_model as _resolve_realtime_route_model, +) from litellm.proxy.common_utils.load_config_utils import get_config_from_bucket from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations from litellm.proxy.common_utils.model_listing_utils import ( @@ -555,9 +567,11 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( from litellm.proxy.common_utils.openai_error_payload import ( LITELLM_CALL_ID_HEADER, attribute_of, + error_status_code, headers_with_litellm_call_id, litellm_call_id_headers, openai_error_param, + openai_error_type, proxy_exception_for, route_failure_proxy_exception, with_litellm_call_id, @@ -10715,6 +10729,26 @@ def _make_keepalive_resolver(request_data: Mapping[str, object]) -> Callable[[ob return _resolve +def _transcription_sse_error_frame(client_error: Exception) -> str: + status_code: Final = error_status_code(client_error, 500) + error_returned: Final = json.dumps( + { + "error": sse_error_payload(client_error)[1] + if isinstance(client_error, HTTPException) + else ProxyException( + message=redact_internal_details_from_client_message( + str(getattr(client_error, "message", client_error)) + ), + type=openai_error_type(client_error, status_code), + param=openai_error_param(client_error), + code=status_code, + ).to_dict() + }, + default=str, + ) + return f"data: {error_returned}\n\n" + + async def async_data_generator( response, user_api_key_dict: UserAPIKeyAuth, @@ -13751,11 +13785,13 @@ def _form_fields_without_file(form_data: Mapping[str, object]) -> Mapping[str, o "/v1/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], tags=["audio"], + responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}}, ) @router.post( "/audio/transcriptions", dependencies=[Depends(user_api_key_auth)], tags=["audio"], + responses={200: {"model": TranscriptionResponse, "content": {"text/event-stream": {"schema": {"type": "string"}}}}}, ) async def audio_transcriptions( request: Request, @@ -13843,34 +13879,72 @@ async def audio_transcriptions( cache_key: Final = hidden_params.get("cache_key", None) or "" api_base: Final = hidden_params.get("api_base", None) or "" response_cost: Final = hidden_params.get("response_cost", None) or "" - response_call_id: Final = hidden_params.get("litellm_call_id", None) or "" + response_call_id: Final = hidden_params.get("litellm_call_id", None) or litellm_call_id additional_headers: Final[dict] = hidden_params.get("additional_headers", {}) or {} + stream_logging_obj: Final = getattr(response, "logging_obj", None) - fastapi_response.headers.update( - ProxyBaseLLMRequestProcessing.get_custom_headers( - user_api_key_dict=user_api_key_dict, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - response_cost=response_cost, - model_region=getattr(user_api_key_dict, "allowed_model_region", ""), - call_id=response_call_id, - request_data=data, - hidden_params=hidden_params, - **additional_headers, - ) + custom_headers: Final = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + response_cost=response_cost, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + call_id=response_call_id, + request_data=data, + hidden_params=hidden_params, + litellm_logging_obj=stream_logging_obj if isinstance(stream_logging_obj, LiteLLMLoggingObj) else None, + **additional_headers, ) # Call response headers hook (matches base_process_llm_request behavior) callback_headers: Final = await proxy_logging_obj.post_call_response_headers_hook( data=data, user_api_key_dict=user_api_key_dict, - response=response, + response=response.response if isinstance(response, GuardedTranscriptionReplay) else response, request_headers=dict(request.headers), ) - if callback_headers: - fastapi_response.headers.update(callback_headers) + response_headers: Final = {**custom_headers, **(callback_headers or {})} + + if hasattr(response, "__aiter__"): + + async def transcription_event_stream( + stream: AsyncIterator[TranscriptionStreamEvent], + ) -> AsyncGenerator[str, None]: + try: + async for event in stream: + yield f"data: {event.model_dump_json(exclude_unset=True)}\n\n" + yield TRANSCRIPTION_STREAM_DONE_FRAME + except Exception as stream_error: # noqa: BLE001 # stream boundary: a mid-stream failure ends the SSE body as an error event + replacement: Final = await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=stream_error, + request_data=( + {**data, "litellm_logging_obj": stream_logging_obj} + if isinstance(stream_logging_obj, LiteLLMLoggingObj) + else data + ), + ) + yield _transcription_sse_error_frame(stream_error if replacement is None else replacement) + yield TRANSCRIPTION_STREAM_DONE_FRAME + finally: + with anyio.CancelScope(shield=True): + close: Final = getattr(stream, "aclose", None) or getattr(stream, "close", None) + if callable(close): + close_result: Final = close() + if inspect.isawaitable(close_result): + await close_result + + return StreamingResponse( + guarded_transcription_events(response.response) + if isinstance(response, GuardedTranscriptionReplay) + else transcription_event_stream(cast(AsyncIterator[TranscriptionStreamEvent], response)), + media_type="text/event-stream", + headers=response_headers, + ) + + fastapi_response.headers.update(response_headers) return response except Exception as e: @@ -14019,6 +14093,8 @@ async def _reject_realtime_session( @app.websocket("/openai/v1/realtime") @app.websocket("/v1/realtime") @app.websocket("/realtime") +@app.websocket("/v1/realtime/translations") +@app.websocket("/realtime/translations") async def realtime_websocket_endpoint( websocket: WebSocket, model: str | None = fastapi.Query(None, description="The model to use for the websocket connection."), @@ -14027,7 +14103,7 @@ async def realtime_websocket_endpoint( None, description="Comma-separated list of guardrail names to apply to this request.", ), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_realtime_websocket), ): requested_protocols: Final = tuple( protocol.strip() @@ -14035,15 +14111,13 @@ async def realtime_websocket_endpoint( if protocol.strip() ) - route_model = model + is_translation: Final = websocket.url.path.endswith("/realtime/translations") + route_model: Final = _resolve_realtime_route_model(model, intent, is_translation) if route_model is None: - if intent == "transcription": - route_model = "gpt-realtime-whisper" - else: - await _reject_realtime_session( - websocket, user_api_key_dict, code=1008, reason="model query parameter is required" - ) - return + await _reject_realtime_session( + websocket, user_api_key_dict, code=1008, reason="model query parameter is required" + ) + return assert route_model is not None try: await can_key_call_resolved_model( @@ -14064,6 +14138,7 @@ async def realtime_websocket_endpoint( model=model, intent=intent, guardrails=guardrails, + is_translation=is_translation, ) @@ -14077,13 +14152,20 @@ async def _route_realtime_websocket_session( guardrails: str | None, request_path: str | None = None, live_session_start: Mapping[str, object] | None = None, + is_translation: bool = False, ) -> None: # Only use explicit parameters, not all query params - query_params: Final = cast(RealtimeQueryParams, dict(_realtime_query_params_template(model, intent))) + query_model: Final = route_model if is_translation else model + query_params: Final = cast( # cast-ok: cached tuples contain only the declared realtime query keys + RealtimeQueryParams, + dict(_realtime_query_params_template(query_model, intent)), + ) + data: dict[str, object] = { "model": route_model, "websocket": websocket, "query_params": query_params, # Only explicit params + "realtime_mode": "translation" if is_translation else "realtime", } # Pass guardrails into data so pre-call guardrail processing picks them up if guardrails: diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py index 9dbf6bcc43d..17c52fc54bd 100644 --- a/litellm/proxy/realtime_endpoints/endpoints.py +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -2,19 +2,24 @@ import json import time -from collections.abc import Mapping +from collections.abc import Generator, Mapping +from contextlib import contextmanager from typing import TYPE_CHECKING, Final import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi import status as http_status -from pydantic import ConfigDict, TypeAdapter, with_config +from fastapi.responses import PlainTextResponse +from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict +import litellm from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_checks import can_key_call_resolved_model from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body from litellm.proxy.common_utils.encrypt_decrypt_utils import ( decrypt_value_helper, encrypt_value_helper, @@ -43,7 +48,26 @@ router: Final = APIRouter() _REALTIME_TOKEN_VERSION: Final = "realtime_v1" _DEFAULT_REALTIME_MODEL: Final = "gpt-4o-realtime-preview" _DEFAULT_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" -_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription") +_ALLOWED_SESSION_TYPES: Final = ("realtime", "transcription", "translation") + + +@contextmanager +def _invalid_request_body_answers_400() -> Generator[None, None, None]: + try: + yield + except ValidationError as error: + raise ProxyException(message=str(error), type="invalid_request_error", param=None, code=400) from error + + +_TRANSLATION_SECRET_BODY: Final = { + "requestBody": inline_request_body( + RealtimeClientSecretRequest, + {"model": "gpt-realtime-translate", "session": {"type": "translation", "model": "gpt-realtime-translate"}}, + ) +} +_TRANSLATION_SDP_BODY: Final = { + "requestBody": {"required": True, "content": {"application/sdp": {"schema": {"type": "string"}}}} +} @with_config(ConfigDict(strict=True, extra="allow")) @@ -140,19 +164,60 @@ def _set_transcription_model_on_session( } +def nested_transcription_upstream_model(model: str, team_id: str | None, llm_router: "Router | None") -> str: + deployment_model: Final = ( + llm_router.registry().resolve(model, kind="upstream", team_id=team_id) if llm_router is not None else None + ) + if deployment_model is None: + return model + try: + return litellm.get_llm_provider(model=deployment_model)[0] + except litellm.BadRequestError: + return deployment_model + + +async def _authorize_and_bind_nested_transcription_models( + session_data: dict, # mutable-ok: session payload is rewritten in place for provider serialization + user_api_key_dict: UserAPIKeyAuth, + llm_model_list: list | None, # mutable-ok: inherited auth helper accepts the proxy model list + llm_router: "Router | None", +) -> None: + nested_models: Final = tuple(_transcription_model_candidates_from_session(session_data)) + for nested_model in nested_models: + await can_key_call_resolved_model( + model=nested_model, + valid_token=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) + if nested_models: + _set_transcription_model_on_session( + session=session_data, + model=nested_transcription_upstream_model(nested_models[0], user_api_key_dict.team_id, llm_router), + ) + + async def _prepare_client_secret_session( req: RealtimeClientSecretRequest, user_api_key_dict: UserAPIKeyAuth, llm_model_list: list | None, llm_router: "Router | None", + forced_session_type: str | None = None, ) -> tuple[str, dict | None, str]: - session_type: Final = _coerce_realtime_session_type(req.session.type if req.session else None) - session_data: Final[dict | None] = req.session.model_dump(exclude_none=True) if req.session else None + requested_session_type: Final = req.session.type if req.session else None + if forced_session_type is None and requested_session_type == "translation": + raise HTTPException(status_code=400, detail="Translation sessions require the translations endpoint") + session_type: Final = forced_session_type or _coerce_realtime_session_type(requested_session_type) + session_data: Final[dict | None] = ( + req.session.model_dump(exclude_unset=True) if req.session else ({} if session_type == "translation" else None) + ) if session_data is not None: session_data["type"] = session_type session_model: Final = req.session.model if req.session else None - model: str = session_model or req.model or _DEFAULT_REALTIME_MODEL + requested_model: Final = session_model or req.model + default_model: Final = "gpt-realtime-translate" if session_type == "translation" else _DEFAULT_REALTIME_MODEL + model: str = requested_model or default_model if session_type != "transcription": await can_key_call_resolved_model( model=model, @@ -160,6 +225,15 @@ async def _prepare_client_secret_session( llm_model_list=llm_model_list, llm_router=llm_router, ) + if session_data is not None and (requested_model or session_type == "translation"): + session_data["model"] = model + if session_type == "translation": + await _authorize_and_bind_nested_transcription_models( + session_data=session_data, + user_api_key_dict=user_api_key_dict, + llm_model_list=llm_model_list, + llm_router=llm_router, + ) return model, session_data, session_type transcription_model_candidates: Final = _transcription_model_candidates_from_session(session_data or {}) @@ -194,12 +268,13 @@ def _encode_realtime_token_payload( team_id: str | None, expires_at: int | None, session_type: str = "realtime", + caller_auth: Mapping[str, object] | None = None, ) -> str: """ Encode metadata with the upstream ephemeral key so /realtime/calls can route without requiring model as a query param. """ - payload: Final[dict[str, str | int | None]] = { + payload: Final[Mapping[str, object]] = { "v": _REALTIME_TOKEN_VERSION, "ephemeral_key": ephemeral_key, "model_id": model_id, @@ -207,10 +282,24 @@ def _encode_realtime_token_payload( "team_id": team_id or "", "expires_at": expires_at, "session_type": session_type, + **({} if caller_auth is None else {"caller_auth": caller_auth}), } return json.dumps(payload, separators=(",", ":")) +def _caller_auth_for_token(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]: + return user_api_key_dict.model_dump(mode="json", exclude_none=True, exclude={"api_key", "parent_otel_span"}) + + +def _caller_auth_from_token( + decoded_payload: Mapping[str, object] | None, user_id: str | None, team_id: str | None +) -> UserAPIKeyAuth: + caller_auth: Final = None if decoded_payload is None else as_str_mapping(decoded_payload.get("caller_auth")) + if caller_auth is None: + return UserAPIKeyAuth(user_id=user_id, team_id=team_id) + return UserAPIKeyAuth.model_validate({**caller_auth, "api_key": caller_auth.get("token")}) + + def _header_value(headers: Mapping[str, str], name: str, default: str) -> str: return headers.get(name, default) @@ -249,11 +338,24 @@ def _decode_realtime_token_payload( dependencies=[Depends(user_api_key_auth)], tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) +@router.post( + "/realtime/translations/client_secrets", + openapi_extra=_TRANSLATION_SECRET_BODY, + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) async def create_realtime_client_secret( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ) -> RealtimeClientSecretResponse: + """Create a short-lived client token for a Realtime conversation, transcription, or translation session""" from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, @@ -269,13 +371,16 @@ async def create_realtime_client_secret( data: dict = {} try: body: Final = await read_request_body(request=request) - req: Final = RealtimeClientSecretRequest(**body) + with _invalid_request_body_answers_400(): + req: Final = RealtimeClientSecretRequest.model_validate(body) + is_translation_request: Final = "/realtime/translations/client_secrets" in request.url.path model, session_data, session_type = await _prepare_client_secret_session( req=req, user_api_key_dict=user_api_key_dict, llm_model_list=llm_model_list, llm_router=llm_router, + forced_session_type="translation" if is_translation_request else None, ) data = {"model": model} @@ -299,17 +404,20 @@ async def create_realtime_client_secret( proxy_config=proxy_config, ) + call_type: Final = ( + "acreate_realtime_translation_client_secret" if is_translation_request else "acreate_realtime_client_secret" + ) data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=data, - call_type="acreate_realtime_client_secret", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/client_secrets (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="acreate_realtime_client_secret", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -366,6 +474,7 @@ async def create_realtime_client_secret( team_id=getattr(user_api_key_dict, "team_id", None), expires_at=expires_at if isinstance(expires_at, int) else None, session_type=session_type, + caller_auth=_caller_auth_for_token(user_api_key_dict) if session_type == "translation" else None, ) encrypted_token: Final[str] = encrypt_value_helper(token_payload) upstream_json["value"] = encrypted_token @@ -392,10 +501,27 @@ async def create_realtime_client_secret( "/openai/v1/realtime/calls", tags=["realtime"], ) +@router.post( + "/v1/realtime/translations/calls", + status_code=http_status.HTTP_201_CREATED, + response_class=PlainTextResponse, + response_description="WebRTC SDP answer", + openapi_extra=_TRANSLATION_SDP_BODY, + tags=["realtime"], +) +@router.post( + "/realtime/translations/calls", + status_code=http_status.HTTP_201_CREATED, + response_class=PlainTextResponse, + response_description="WebRTC SDP answer", + openapi_extra=_TRANSLATION_SDP_BODY, + tags=["realtime"], +) async def proxy_realtime_calls( request: Request, fastapi_response: Response, ) -> Response: + """Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets""" from litellm.proxy.proxy_server import ( add_litellm_data_to_request, general_settings, @@ -417,6 +543,7 @@ async def proxy_realtime_calls( media_type="application/json", ) + is_translation_request: Final = "/realtime/translations/calls" in request.url.path encrypted_token: Final = auth_header.removeprefix("Bearer ").strip() decrypted_token_value: Final = decrypt_value_helper( value=encrypted_token, @@ -434,33 +561,39 @@ async def proxy_realtime_calls( if decoded_payload is not None: # Check token expiry expires_at: Final = decoded_payload.get("expires_at") - if expires_at is not None and isinstance(expires_at, int): - if time.time() > expires_at: - return Response( - content=json.dumps({"error": "Token has expired"}), - status_code=http_status.HTTP_401_UNAUTHORIZED, - media_type="application/json", - ) + if isinstance(expires_at, int) and time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") model = decoded_payload.get("model_id") or request.query_params.get("model") or _DEFAULT_REALTIME_MODEL user_id = decoded_payload.get("user_id") or None team_id = decoded_payload.get("team_id") or None - session_type = _coerce_realtime_session_type(decoded_payload.get("session_type")) + raw_session_type: Final = decoded_payload.get("session_type") + session_type = _coerce_realtime_session_type(raw_session_type) + if is_translation_request != (raw_session_type == "translation"): + return Response( + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) else: - # Backward compatibility: older tokens contained only encrypted upstream key. + if is_translation_request: + return Response( + content=json.dumps({"error": "Token is not valid for this Realtime endpoint"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) openai_ephemeral_key = decrypted_token_value model = request.query_params.get("model", _DEFAULT_REALTIME_MODEL) user_id = None team_id = None session_type = "realtime" - # Build a minimal UserAPIKeyAuth with user/team IDs from the token - # so spend tracking and budget enforcement work correctly. - minimal_auth: Final = UserAPIKeyAuth( - user_id=user_id, - team_id=team_id, - ) + token_caller_auth: Final = _caller_auth_from_token(decoded_payload, user_id, team_id) data: dict = {} try: @@ -487,22 +620,23 @@ async def proxy_realtime_calls( data=data, request=request, general_settings=general_settings, - user_api_key_dict=minimal_auth, + user_api_key_dict=token_caller_auth, version=version, proxy_config=proxy_config, ) + call_type: Final = "arealtime_translation_calls" if is_translation_request else "arealtime_calls" data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=minimal_auth, + user_api_key_dict=token_caller_auth, data=data, - call_type="arealtime_calls", + call_type=call_type, ) verbose_proxy_logger.debug("WebRTC: /v1/realtime/calls (model=%s)", model) llm_call: Final = await route_request( data=data, - route_type="arealtime_calls", + route_type=call_type, llm_router=llm_router, user_model=user_model, ) @@ -510,7 +644,7 @@ async def proxy_realtime_calls( except Exception as e: await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=minimal_auth, + user_api_key_dict=token_caller_auth, original_exception=e, request_data=data, ) @@ -581,7 +715,8 @@ async def create_realtime_transcription_session( data: dict = {} try: body: Final = await read_request_body(request=request) - req: Final = RealtimeTranscriptionSessionRequest(**body) + with _invalid_request_body_answers_400(): + req: Final = RealtimeTranscriptionSessionRequest.model_validate(body) model: Final[str] = req.resolved_model() or "gpt-realtime-whisper" await can_key_call_resolved_model( @@ -592,6 +727,7 @@ async def create_realtime_transcription_session( ) transcription_session: Final = {k: v for k, v in body.items() if k != "model"} + _set_transcription_model_on_session(session=transcription_session, model=model, create_if_missing=True) data = {"model": model, "transcription_session": transcription_session} data = await add_litellm_data_to_request( diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index ecc31f3b0a7..148d465099f 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -621,7 +621,9 @@ RouteType = Literal[ "alist_input_items", "_arealtime", # private function for realtime API "acreate_realtime_client_secret", + "acreate_realtime_translation_client_secret", "arealtime_calls", + "arealtime_translation_calls", "acreate_realtime_transcription_session", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index e03cb44efda..be1df4aa0f6 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -2,11 +2,13 @@ import asyncio import os +import traceback from collections.abc import Mapping from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, cast +import httpx from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict, assert_never @@ -14,10 +16,14 @@ import litellm from litellm.constants import ( AZURE_OPENAI_AUDIO_PROVIDERS, REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS, + REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout, ) +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.xai.common_utils import XAIModelInfo @@ -36,18 +42,24 @@ from litellm.utils import ( ModelInfoLookupMiss, ProviderConfigManager, aresolve_api_key_secret_reference, + get_base_model_from_metadata, lookup_model_info_for_registration, ) -from ..litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params +from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor from ..litellm_core_utils.get_litellm_params import get_litellm_params from ..litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from ..litellm_core_utils.realtime_streaming import client_websocket_scope +from ..litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason +from ..litellm_core_utils.realtime_streaming import ( + TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, + client_websocket_scope, + translation_guardrails_apply, +) from ..llms.azure.common_utils import get_azure_ad_token from ..llms.azure.realtime.handler import AzureOpenAIRealtime, azure_realtime_protocol_for_client from ..llms.bedrock.realtime.handler import BedrockRealtime, raise_if_session_committed -from ..llms.custom_httpx.http_handler import realtime_ssl_for_url +from ..llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, realtime_ssl_for_url from ..llms.openai.common_utils import is_openai_backed_api_base from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import ( @@ -66,8 +78,6 @@ from ..utils import client as wrapper_client if TYPE_CHECKING: from fastapi import WebSocket - from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig - azure_realtime: Final = AzureOpenAIRealtime() openai_realtime: Final = OpenAIRealtime() openai_live_sessions: Final = OpenAILiveSessions() @@ -77,7 +87,7 @@ vertex_llm_base: Final = VertexBase() base_llm_http_handler = BaseLLMHTTPHandler() _EMPTY_MODEL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _EMPTY_AUTH_HEADERS: Final[Mapping[str, str]] = MappingProxyType({}) -_DEPLOYMENT_PARAMS: Final = TypeAdapter(Mapping[str, object]) +_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> Mapping[str, object]: @@ -90,26 +100,145 @@ def _model_params_with_stored_credentials(model_params: Mapping[str, object]) -> return MappingProxyType({**credential_values, **model_params}) -def _with_resolved_session_model(session: dict[str, object], model_name: str) -> dict[str, object]: - if "model" not in session: - return session - return {**session, "model": model_name} +def _transcription_model_overrides(session: Mapping[str, object], model_name: str) -> Mapping[str, object]: + flat_transcription: Final = as_str_mapping(session.get("input_audio_transcription")) + audio: Final = as_str_mapping(session.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + nested_transcription: Final = as_str_mapping(audio_input.get("transcription")) + flat_part: Final = ( + {} if flat_transcription is None else {"input_audio_transcription": {**flat_transcription, "model": model_name}} + ) + nested_part: Final = ( + {} + if nested_transcription is None and flat_transcription is not None + else { + "audio": { + **audio, + "input": {**audio_input, "transcription": {**(nested_transcription or {}), "model": model_name}}, + } + } + ) + return MappingProxyType({**flat_part, **nested_part}) -def _resolved_request_session(request_data: Mapping[str, object], model_name: str) -> Mapping[str, object] | None: - session: Final = as_str_mapping(request_data.get("session")) - if not isinstance(session, dict): - return None - return _with_resolved_session_model(session, model_name) +def with_resolved_session_model(session: Mapping[str, object], model_name: str) -> dict[str, object]: + top_level_model: Final = {} if "model" not in session else {"model": model_name} + transcription_models: Final = ( + _transcription_model_overrides(session, model_name) if session.get("type") == "transcription" else {} + ) + return {**session, **top_level_model, **transcription_models} -def _build_litellm_metadata(kwargs: dict) -> dict[str, object]: - """Build the litellm_metadata dict for guardrail checking (internal only, not forwarded to provider).""" - metadata: Final[dict[str, object]] = {**(kwargs.get("litellm_metadata") or {})} - guardrails: Final = (kwargs.get("metadata") or {}).get("guardrails") or kwargs.get("guardrails") or [] - if guardrails: - metadata["guardrails"] = guardrails - return metadata +def _without_top_level_nulls(fields: Mapping[str, object]) -> dict[str, object]: + return {key: value for key, value in fields.items() if value is not None} + + +_POLICY_PIPELINE_METADATA_KEYS: Final = ("_guardrail_pipelines", "_pipeline_managed_guardrails") + + +def _build_litellm_metadata(kwargs: Mapping[str, object]) -> dict[str, object]: + request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {}) + internal_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("litellm_metadata") or {}) + policy_pipeline_state: Final = { + key: request_metadata[key] for key in _POLICY_PIPELINE_METADATA_KEYS if key in request_metadata + } + guardrails: Final = request_metadata.get("guardrails") or kwargs.get("guardrails") + return {**policy_pipeline_state, **internal_metadata, **({"guardrails": guardrails} if guardrails else {})} + + +_GUARDRAIL_OPT_OUT_METADATA_KEYS: Final = ("user_api_key_metadata", "user_api_key_team_metadata") + + +def _translation_litellm_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + request_metadata: Final = _OBJECT_MAPPING.validate_python(kwargs.get("metadata") or {}) + guardrail_opt_outs: Final = { + key: request_metadata[key] for key in _GUARDRAIL_OPT_OUT_METADATA_KEYS if key in request_metadata + } + return MappingProxyType({**guardrail_opt_outs, **_build_litellm_metadata(kwargs)}) + + +_HTTP_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, LlmProviders.AZURE.value}) +_WEBSOCKET_TRANSLATION_PROVIDERS: Final = frozenset({LlmProviders.OPENAI.value, *AZURE_OPENAI_AUDIO_PROVIDERS}) + + +@dataclass(frozen=True, slots=True) +class _TranslationProviderUnsupported: + provider: str + supported_providers: frozenset[str] + + +@dataclass(frozen=True, slots=True) +class _TranslationGuardrailsUnsupported: + pass + + +_TranslationRejection = _TranslationProviderUnsupported | _TranslationGuardrailsUnsupported + + +def _translation_rejection( + custom_llm_provider: str, supported_providers: frozenset[str], litellm_metadata: Mapping[str, object] +) -> _TranslationRejection | None: + if custom_llm_provider not in supported_providers: + return _TranslationProviderUnsupported(provider=custom_llm_provider, supported_providers=supported_providers) + if translation_guardrails_apply({"litellm_metadata": dict(litellm_metadata)}): + return _TranslationGuardrailsUnsupported() + return None + + +def _translation_rejection_message(rejection: _TranslationRejection) -> str: + match rejection: + case _TranslationProviderUnsupported(provider=provider, supported_providers=supported_providers): + return ( + f"Realtime translation supports only the {', '.join(sorted(supported_providers))} providers, " + f"got {provider}" + ) + case _TranslationGuardrailsUnsupported(): + return TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE + case _: + assert_never(rejection) + + +def _translation_rejection_error_type(rejection: _TranslationRejection) -> str: + match rejection: + case _TranslationProviderUnsupported(): + return "invalid_request_error" + case _TranslationGuardrailsUnsupported(): + return "guardrail_error" + case _: + assert_never(rejection) + + +def _translation_rejection_error( + rejection: _TranslationRejection, model: str, custom_llm_provider: str +) -> BadRequestError: + return BadRequestError( + message=_translation_rejection_message(rejection), model=model, llm_provider=custom_llm_provider + ) + + +def _raise_if_http_translation_rejected(custom_llm_provider: str, model: str, kwargs: Mapping[str, object]) -> None: + rejection: Final = _translation_rejection( + custom_llm_provider, _HTTP_TRANSLATION_PROVIDERS, _translation_litellm_metadata(kwargs) + ) + if rejection is None: + return + raise _translation_rejection_error(rejection, model, custom_llm_provider) + + +async def _refuse_translation_session( + websocket: "WebSocket", + rejection: _TranslationRejection, + logging_obj: LiteLLMLogging, + model: str, + custom_llm_provider: str, +) -> None: + message: Final = _translation_rejection_message(rejection) + await websocket.send_text(realtime_error_event(message, error_type=_translation_rejection_error_type(rejection))) + await websocket.close(code=1008, reason=websocket_close_reason(message, fallback="Translation session refused")) + error: Final = _translation_rejection_error(rejection, model, custom_llm_provider) + await logging_obj.dispatch_failure_handlers( + error, "".join(traceback.format_exception(error)), prefer_async_handlers=True + ) def _get_realtime_http_provider_config( @@ -125,10 +254,6 @@ def _get_realtime_http_provider_config( Uses ProviderConfigManager so each provider keeps its credential-resolution and URL-construction logic in its own transformation class. """ - from litellm.llms.base_llm.realtime.http_transformation import ( - BaseRealtimeHTTPConfig, - ) - provider_config: BaseRealtimeHTTPConfig | None = None if custom_llm_provider in LlmProviders._member_map_.values(): provider_config = ProviderConfigManager.get_provider_realtime_http_config( @@ -163,6 +288,32 @@ async def _realtime_http_bearer_token( return bearer_token or "" +async def _get_realtime_http_extra_headers( + provider_config: BaseRealtimeHTTPConfig | None, + litellm_params: GenericLiteLLMParams, + resolved_api_key: str, + extra_headers: Mapping[str, object] | None, +) -> Mapping[str, object] | None: + if provider_config is None: + return extra_headers + return await asyncio.to_thread( + provider_config.get_extra_headers, + litellm_params=litellm_params, + api_key=resolved_api_key, + extra_headers=extra_headers, + ) + + +def _client_secret_session_body(req: RealtimeClientSecretRequest, model_name: str) -> Mapping[str, object] | None: + if req.session is not None: + return with_resolved_session_model( + _without_top_level_nulls(req.session.model_dump(exclude_unset=True)), model_name + ) + if req.model is not None: + return {"type": "realtime", "model": model_name} + return None + + @wrapper_client async def acreate_realtime_client_secret( model: str | None = None, @@ -176,31 +327,42 @@ async def acreate_realtime_client_secret( session=RealtimeSessionConfig.model_validate(session) if session else None, expires_after=RealtimeExpiresAfter.model_validate(expires_after) if expires_after else None, ) - model_name = (req.session.model if req.session is not None else None) or req.model or "gpt-4o-realtime-preview" + transcription_model: Final = ( + req.session.audio.input.transcription.model + if req.session is not None + and req.session.type == "transcription" + and req.session.audio is not None + and req.session.audio.input is not None + and req.session.audio.input.transcription is not None + else None + ) + requested_model_name: Final = ( + transcription_model + or (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) litellm_logging_obj: Final[LiteLLMLogging] = kwargs.get("litellm_logging_obj") litellm_params: Final = GenericLiteLLMParams(**kwargs) - ( - model_name, - custom_llm_provider, - dynamic_api_key, - dynamic_api_base, - ) = get_llm_provider( - model=model_name, + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, custom_llm_provider=litellm_params.custom_llm_provider, api_base=litellm_params.api_base, api_key=litellm_params.api_key, ) - ( - provider_config, - resolved_api_base, - resolved_api_key, - ) = _get_realtime_http_provider_config( + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( custom_llm_provider=custom_llm_provider, dynamic_api_base=dynamic_api_base, dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = await _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -208,26 +370,124 @@ async def acreate_realtime_client_secret( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - resolved_session: Final = _resolved_request_session(request_data, model_name) - if resolved_session is not None: - request_data["session"] = resolved_session + session_body: Final = _client_secret_session_body(req, model_name) return await base_llm_http_handler.async_realtime_client_secret_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params ), - request_data=request_data, + request_data={ + **req.model_dump(exclude_none=True, exclude={"model", "session"}), + **({} if session_body is None else {"session": session_body}), + }, logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) +@wrapper_client +async def acreate_realtime_translation_client_secret( + model: str | None = None, + session: Mapping[str, Any] | None = None, + expires_after: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs: object, # kwargs-ok: public API preserves provider-specific request options +) -> httpx.Response: + requested_model_name: Final = model or (session or {}).get("model") or "gpt-realtime-translate" + session_config: Final = RealtimeSessionConfig.model_validate( + { + **(session or {}), + "type": "translation", + "model": requested_model_name, + } + ) + req: Final = RealtimeClientSecretRequest( + model=requested_model_name, + session=session_config, + expires_after=RealtimeExpiresAfter(**expires_after) if expires_after else None, + ) + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + custom_llm_provider=litellm_params.custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + _raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + resolved_extra_headers: Final = await _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=as_str_mapping(kwargs.get("extra_headers")), + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ + "expires_after": expires_after, + "session": session, + }, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + translation_session: Final = with_resolved_session_model( + _without_top_level_nulls(session_config.model_dump(exclude_unset=True)), model_name + ) + return await base_llm_http_handler.async_realtime_translation_client_secret_handler( + api_base=resolved_api_base, + api_key=await _realtime_http_bearer_token( + custom_llm_provider, resolved_api_key, resolved_api_base, litellm_params + ), + request_data={ + **req.model_dump(exclude_none=True, exclude={"model", "session"}), + "session": {key: value for key, value in translation_session.items() if key != "type"}, + }, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=resolved_extra_headers, + client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None, + api_version=litellm_params.api_version, + ) + + +def _with_upstream_transcription_model(request_data: Mapping[str, object], model_name: str) -> dict[str, object]: + flat: Final = as_str_mapping(request_data.get("input_audio_transcription")) + audio: Final = as_str_mapping(request_data.get("audio")) or {} + audio_input: Final = as_str_mapping(audio.get("input")) or {} + nested: Final = as_str_mapping(audio_input.get("transcription")) + return { + **request_data, + **({} if flat is None else {"input_audio_transcription": {**flat, "model": model_name}}), + **( + {} + if nested is None + else {"audio": {**audio, "input": {**audio_input, "transcription": {**nested, "model": model_name}}}} + ), + } + + +def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: str) -> str | None: + base_model: Final = get_base_model_from_metadata(model_call_details={"litellm_params": kwargs}) + return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/") + + @wrapper_client async def acreate_realtime_transcription_session( model: str | None = None, @@ -273,6 +533,12 @@ async def acreate_realtime_transcription_session( dynamic_api_key=dynamic_api_key, litellm_params=litellm_params, ) + resolved_extra_headers: Final = await _get_realtime_http_extra_headers( + provider_config=provider_config, + litellm_params=litellm_params, + resolved_api_key=resolved_api_key, + extra_headers=kwargs.get("extra_headers"), + ) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -280,13 +546,11 @@ async def acreate_realtime_transcription_session( litellm_params={"api_base": resolved_api_base}, custom_llm_provider=custom_llm_provider, ) - request_data: Final = req.model_dump(exclude_none=True, exclude={"model"}) - # Ensure the upstream body's input_audio_transcription.model matches the - # authorized routing model. This prevents a caller from supplying an allowed - # top-level model for auth while sneaking a different model into the nested - # transcription config that gets forwarded to the provider. - if isinstance(request_data.get("input_audio_transcription"), dict): - request_data["input_audio_transcription"]["model"] = model_name + # The upstream transcription model must match the authorized routing model, so a + # caller cannot pass auth with an allowed top-level model and forward a different one. + request_data: Final = _with_upstream_transcription_model( + _without_top_level_nulls(req.model_dump(exclude_unset=True, exclude={"model"})), model_name + ) return await base_llm_http_handler.async_realtime_transcription_session_handler( api_base=resolved_api_base, api_key=await _realtime_http_bearer_token( @@ -297,7 +561,7 @@ async def acreate_realtime_transcription_session( timeout=timeout or request_timeout, provider_config=provider_config, model=model_name, - extra_headers=kwargs.get("extra_headers"), + extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, ) @@ -334,7 +598,7 @@ async def arealtime_calls( litellm_params=litellm_params, ) if session is not None: - session = _with_resolved_session_model(session, model_name) + session = with_resolved_session_model(session, model_name) litellm_logging_obj.update_from_kwargs( kwargs=kwargs, model=model_name, @@ -357,6 +621,68 @@ async def arealtime_calls( ) +@wrapper_client +async def arealtime_translation_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: str | None = None, + session: Mapping[str, Any] | None = None, + timeout: float | None = None, + **kwargs: object, # kwargs-ok: public API preserves provider-specific request options +) -> httpx.Response: + requested_model_name: Final = model or "gpt-realtime-translate" + litellm_logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(litellm_logging_obj, LiteLLMLogging): + raise TypeError("litellm_logging_obj must be a LiteLLM Logging instance") + litellm_params: Final = GenericLiteLLMParams.model_validate(kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=requested_model_name, + custom_llm_provider=litellm_params.custom_llm_provider, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + _raise_if_http_translation_rejected(custom_llm_provider, model_name, kwargs) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + session_config: Final = with_resolved_session_model( + { + **(session or {}), + "type": "translation", + "model": model_name, + }, + model_name, + ) + litellm_logging_obj.update_from_kwargs( + kwargs=kwargs, + model=model_name, + optional_params={ + "realtime_translation_calls": True, + "session": session_config, + }, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session_config, + extra_headers=as_str_mapping(kwargs.get("extra_headers")), + client=client if isinstance(client := kwargs.get("client"), (HTTPHandler, AsyncHTTPHandler)) else None, + api_version=litellm_params.api_version, + translation=True, + ) + + async def vertex_access_token_resolver( credentials: VERTEX_CREDENTIALS_TYPES | None, project_id: str | None, @@ -486,6 +812,7 @@ async def _arealtime( timeout: float | None = None, query_params: RealtimeQueryParams | None = None, live_session_start: Mapping[str, object] | None = None, + realtime_mode: str = "realtime", **kwargs, ) -> None: """ @@ -530,7 +857,24 @@ async def _arealtime( custom_llm_provider=_custom_llm_provider, ) - deployment_params: Final = _DEPLOYMENT_PARAMS.validate_python(kwargs) + session_litellm_metadata: Final = ( + dict(_translation_litellm_metadata(kwargs)) + if realtime_mode == "translation" + else _build_litellm_metadata(kwargs) + ) + translation_rejection: Final = ( + _translation_rejection(_custom_llm_provider, _WEBSOCKET_TRANSLATION_PROVIDERS, session_litellm_metadata) + if realtime_mode == "translation" + else None + ) + if translation_rejection is not None: + await _refuse_translation_session( + websocket, translation_rejection, litellm_logging_obj, model, _custom_llm_provider + ) + litellm_logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] = True + return + + deployment_params: Final = _OBJECT_MAPPING.validate_python(kwargs) if live_session_start is not None: await _alive_session( model=model, @@ -540,7 +884,7 @@ async def _arealtime( session_start=live_session_start, api_base=dynamic_api_base or litellm_params.api_base, api_key=dynamic_api_key, - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, litellm_params=deployment_params, ) return @@ -566,7 +910,7 @@ async def _arealtime( timeout=timeout, headers=headers, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, query_params=query_params, ) elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS: @@ -581,9 +925,6 @@ async def _arealtime( or litellm_params.get("realtime_protocol") or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL") ) - realtime_protocol: Final = azure_realtime_protocol_for_client( - configured_realtime_protocol, query_params=query_params, websocket=websocket - ) resolved_azure_ad_token: Final = ( None if api_key @@ -592,6 +933,14 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) + realtime_protocol: Final = azure_realtime_protocol_for_client( + configured_realtime_protocol, + model=model, + base_model=_deployment_base_model(kwargs, _custom_llm_provider), + query_params=query_params, + realtime_mode=realtime_mode, + websocket=websocket, + ) await azure_realtime.async_realtime( model=model, websocket=websocket, @@ -602,10 +951,13 @@ async def _arealtime( client=None, timeout=timeout, logging_obj=litellm_logging_obj, - realtime_protocol=realtime_protocol, + realtime_protocol=realtime_protocol.name, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, + bridges_beta_client_to_ga=realtime_protocol.bridges_beta_client_to_ga, + drop_params=drop_params, ) elif _custom_llm_provider == "openai": api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or "https://api.openai.com/" @@ -622,8 +974,10 @@ async def _arealtime( client=None, timeout=timeout, query_params=query_params, + realtime_mode=realtime_mode, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, + drop_params=drop_params, ) elif _custom_llm_provider == "bedrock": # Extract AWS parameters from kwargs @@ -678,7 +1032,7 @@ async def _arealtime( timeout=timeout, query_params=query_params, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, ) elif _custom_llm_provider == "vertex_ai": vertex_credentials: Final = ( @@ -738,7 +1092,7 @@ async def _arealtime( timeout=timeout, headers=headers, user_api_key_dict=kwargs.get("user_api_key_dict"), - litellm_metadata=_build_litellm_metadata(kwargs), + litellm_metadata=session_litellm_metadata, query_params=query_params, ) else: diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index ee67d82abfa..17791019aab 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -847,9 +847,11 @@ def _with_router_resolved_session_model(session: object, model_name: str) -> Map typed_session: Final = _SESSION_ADAPTER.validate_python(session) except ValidationError: return _NO_SESSION_KWARGS - if "model" not in typed_session: + if "model" not in typed_session and typed_session.get("type") != "transcription": return _NO_SESSION_KWARGS - return MappingProxyType({"session": {**typed_session, "model": model_name}}) + from litellm.realtime_api.main import with_resolved_session_model + + return MappingProxyType({"session": with_resolved_session_model(dict(typed_session), model_name)}) # Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks @@ -2865,6 +2867,14 @@ class PythonCore(DeprecatedRouterMembers): self._host.invoke("acreate_realtime_transcription_session"), call_type="acreate_realtime_transcription_session", ) + self.acreate_realtime_translation_client_secret = self.factory_function( + self._host.invoke("acreate_realtime_translation_client_secret"), + call_type="acreate_realtime_translation_client_secret", + ) + self.arealtime_translation_calls = self.factory_function( + self._host.invoke("arealtime_translation_calls"), + call_type="arealtime_translation_calls", + ) self._aresponses_websocket = self.factory_function( self._host.invoke("_aresponses_websocket"), call_type="_aresponses_websocket" ) @@ -8433,6 +8443,8 @@ class PythonCore(DeprecatedRouterMembers): "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", "_aresponses_websocket", "acreate_fine_tuning_job", "acancel_fine_tuning_job", @@ -8703,6 +8715,8 @@ class PythonCore(DeprecatedRouterMembers): "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c803df04911..c9739eee91d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1247,9 +1247,12 @@ AllEmbeddingInputValues = str | list[str] | list[int] | list[list[int]] OpenAIAudioTranscriptionOptionalParams = Literal[ "language", + "languages", + "keywords", "prompt", "temperature", "response_format", + "stream", "timestamp_granularities", "include", ] @@ -2545,6 +2548,17 @@ class OpenAIRealtimeResponseUsage(TypedDict): output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] +class OpenAIRealtimeTranslationDurationUsage(TypedDict): + type: ReadOnly[Literal["duration"]] + output_seconds: NotRequired[ReadOnly[float]] + input_seconds: NotRequired[ReadOnly[float]] + + +class OpenAIRealtimeTranslationClosedEvent(TypedDict): + type: ReadOnly[Literal["session.closed"]] + usage: ReadOnly[OpenAIRealtimeTranslationDurationUsage] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2588,6 +2602,7 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeInputAudioTranscriptionCompleted | OpenAIRealtimeTranscriptionSessionCreated | OpenAIRealtimeErrorEvent + | OpenAIRealtimeTranslationClosedEvent ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 6da73982291..4a9b4228bc8 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping, Sequence from typing import Any, Literal from pydantic import ConfigDict @@ -66,6 +67,41 @@ class RealtimeExpiresAfter(LiteLLMBaseModel): seconds: int | None = None +class RealtimeAudioTranscriptionConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + model: str | None = None + delay: str | None = None + keywords: Sequence[str] | None = None + language: str | None = None + languages: Sequence[str] | None = None + prompt: str | None = None + + +class RealtimeAudioInputConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + format: str | Mapping[str, object] | None = None + noise_reduction: Mapping[str, object] | None = None + transcription: RealtimeAudioTranscriptionConfig | None = None + turn_detection: Mapping[str, object] | None = None + + +class RealtimeAudioOutputConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + format: str | Mapping[str, object] | None = None + language: str | None = None + voice: str | Mapping[str, object] | None = None + + +class RealtimeSessionAudioConfig(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", frozen=True) + + input: RealtimeAudioInputConfig | None = None + output: RealtimeAudioOutputConfig | None = None + + class RealtimeSessionConfig(LiteLLMBaseModel): """ Session configuration nested inside the client_secrets request body. @@ -75,12 +111,12 @@ class RealtimeSessionConfig(LiteLLMBaseModel): Extra/unknown fields are passed through unchanged. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) type: str | None = None model: str | None = None instructions: str | None = None - audio: dict[str, object] | None = None + audio: RealtimeSessionAudioConfig | None = None include: list[str] | None = None max_output_tokens: int | str | None = None output_modalities: list[str] | None = None @@ -129,17 +165,20 @@ class RealtimeTranscriptionSessionRequest(LiteLLMBaseModel): unchanged to the provider. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) # LiteLLM-only routing hint — stripped before forwarding upstream. model: str | None = None input_audio_transcription: dict[str, Any] | None = None + audio: RealtimeSessionAudioConfig | None = None def resolved_model(self) -> str | None: if self.model: return self.model if self.input_audio_transcription: return self.input_audio_transcription.get("model") + if self.audio and self.audio.input and self.audio.input.transcription: + return self.audio.input.transcription.model return None @@ -151,7 +190,7 @@ class RealtimeTranscriptionSessionResponse(LiteLLMBaseModel): ephemeral key. Unknown fields pass through unchanged. """ - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", frozen=True) client_secret: dict[str, object] | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e633e557bb2..57f4d971eb3 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -554,6 +554,11 @@ class CallTypes(str, Enum): aextraction = "aextraction" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" + acreate_realtime_client_secret = "acreate_realtime_client_secret" + arealtime_calls = "arealtime_calls" + acreate_realtime_transcription_session = "acreate_realtime_transcription_session" + acreate_realtime_translation_client_secret = "acreate_realtime_translation_client_secret" + arealtime_translation_calls = "arealtime_translation_calls" create_batch = "create_batch" acreate_batch = "acreate_batch" aretrieve_batch = "aretrieve_batch" @@ -794,6 +799,8 @@ CallTypesLiteral = Literal[ "acreate_realtime_client_secret", "arealtime_calls", "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_translation_calls", ] MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( @@ -802,6 +809,10 @@ MCP_GUARDRAIL_CALL_TYPES: Final[frozenset[str]] = frozenset( # Mapping of API routes to their corresponding call types API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { + "/v1/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/realtime/translations/client_secrets": (CallTypes.acreate_realtime_translation_client_secret,), + "/v1/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), + "/realtime/translations/calls": (CallTypes.arealtime_translation_calls,), # Chat Completions "/chat/completions": [CallTypes.acompletion, CallTypes.completion], "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], @@ -1120,9 +1131,11 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), "/openai/v1/responses/{response_id}/input_items": (CallTypes.alist_input_items,), # Realtime API - "/realtime": [CallTypes.arealtime], - "/v1/realtime": [CallTypes.arealtime], - "/openai/v1/realtime": [CallTypes.arealtime], + "/realtime": (CallTypes.arealtime,), + "/v1/realtime": (CallTypes.arealtime,), + "/openai/v1/realtime": (CallTypes.arealtime,), + "/realtime/translations": (CallTypes.arealtime,), + "/v1/realtime/translations": (CallTypes.arealtime,), "/live/sessions": (CallTypes.arealtime,), "/v1/live/sessions": (CallTypes.arealtime,), "/openai/v1/live/sessions": (CallTypes.arealtime,), @@ -2951,9 +2964,19 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None +class TranscriptionDetectedLanguage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + code: str + + +_DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, ...]) + + class TranscriptionResponse(OpenAIObject): text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None + languages: Sequence[TranscriptionDetectedLanguage] | None = None _hidden_params: dict = {} _response_headers: dict | None = None @@ -2966,8 +2989,28 @@ class TranscriptionResponse(OpenAIObject): def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params - def __init__(self, text=None) -> None: - super().__init__(text=text) + def __init__( + self, + text: str | None = None, + usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | Mapping[str, object] | None = None, + languages: Sequence[TranscriptionDetectedLanguage | Mapping[str, object]] | None = None, + **kwargs: object, # kwargs-ok: OpenAI-compatible responses preserve provider extension fields + ) -> None: + super().__init__( + text=text, usage=usage, languages=TranscriptionResponse.detected_languages(languages), **kwargs + ) + if self.languages is None: + del self.languages + + @staticmethod + def detected_languages(value: object) -> tuple[TranscriptionDetectedLanguage, ...] | None: + try: + return _DETECTED_LANGUAGES.validate_python(value, from_attributes=True) + except ValidationError: + return None + + def set_audio_transcription_duration(self, duration: float) -> None: + self._hidden_params["audio_transcription_duration"] = duration def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/litellm/utils.py b/litellm/utils.py index 85e669843fe..645947dcdba 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1124,6 +1124,38 @@ def _is_streaming_response_for_correlation(result: object) -> bool: return isinstance(result, CustomStreamWrapper) +def _returns_requested_stream(kwargs: dict[str, object], call_type: CallTypes | str, result: object) -> bool: + if call_type in (CallTypes.transcription, CallTypes.atranscription): + return isinstance(result, (openai.Stream, openai.AsyncStream)) + return _is_streaming_request(kwargs=kwargs, call_type=call_type) + + +def _transcription_runs_unstreamed( + call_type: str, requested_kwargs: Mapping[str, object], call_kwargs: Mapping[str, object] +) -> bool: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested + + return ( + call_type == CallTypes.atranscription.value + and transcription_stream_requested(requested_kwargs.get("stream")) + and call_kwargs.get("stream") is False + ) + + +def _guardrail_view_of_requested_stream(call_kwargs: Mapping[str, object]) -> dict[str, object]: + from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming + + return guardrail_request_data_with_streaming(call_kwargs, is_streaming=True) + + +def _replayed_when_run_unstreamed(result: object, runs_unstreamed: bool) -> object: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import GuardedTranscriptionReplay + + if not runs_unstreamed or not isinstance(result, TranscriptionResponse): + return result + return GuardedTranscriptionReplay(result) + + def _is_converted_stream_result(result: object) -> bool: from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator @@ -1559,7 +1591,6 @@ def function_setup( applied_guardrails=applied_guardrails, supports_correlation_logging=is_async_call, ) - ## check if metadata is passed in litellm_params: Final[dict[str, object]] = {"api_base": ""} if "metadata" in kwargs: @@ -1807,7 +1838,28 @@ async def async_pre_call_deployment_hook(kwargs: Mapping[str, object], call_type if result is not None: modified_kwargs = result - return modified_kwargs + return _unstreamed_when_post_call_guardrails_apply(modified_kwargs, typed_call_type) + + +def _unstreamed_when_post_call_guardrails_apply( + kwargs: dict[str, object], call_type: CallTypes | None +) -> dict[str, object]: + from litellm.litellm_core_utils.audio_utils.transcription_streaming import transcription_stream_requested + + if ( + call_type is not CallTypes.atranscription + or not transcription_stream_requested(kwargs.get("stream")) + or _deployment_provider(kwargs) not in (LlmProviders.OPENAI.value, LlmProviders.AZURE.value) + ): + return kwargs + streamed_request: Final = _guardrail_view_of_requested_stream(kwargs) + custom_guardrail_class: Final = _get_cached_custom_guardrail() + if not any( + isinstance(callback, custom_guardrail_class) and callback.runs_post_call_deployment_hook(streamed_request) + for callback in litellm.callbacks + ): + return kwargs + return {**kwargs, "stream": False} async def async_post_call_success_deployment_hook( @@ -2155,7 +2207,7 @@ def client(original_function): # MODEL CALL result = original_function(*args, **kwargs) end_time = datetime.datetime.now() - if _is_streaming_request(kwargs=kwargs, call_type=call_type) or _is_converted_stream_result(result): + if _returns_requested_stream(kwargs, call_type, result) or _is_converted_stream_result(result): if "complete_response" in kwargs and kwargs["complete_response"] is True: chunks = [] for idx, chunk in enumerate(result): @@ -2164,6 +2216,15 @@ def client(original_function): else: logging_obj.stream = True logging_obj.model_call_details["stream"] = True + if call_type == CallTypes.transcription.value and isinstance(result, openai.Stream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, + ) + + result = wrap_transcription_stream( + result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details) + ) # RETURN RESULT update_response_metadata: _ResponseMetadataUpdater = litellm_utils.update_response_metadata update_response_metadata( @@ -2339,6 +2400,7 @@ def client(original_function): kwargs["litellm_logging_obj"] = logging_obj modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type) + transcription_runs_unstreamed: Final = _transcription_runs_unstreamed(call_type, kwargs, modified_kwargs) if modified_kwargs is not None: kwargs = modified_kwargs @@ -2389,7 +2451,9 @@ def client(original_function): if _is_converted_stream_result(_caching_handler_response.cached_result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True - return _caching_handler_response.cached_result + return _replayed_when_run_unstreamed( + _caching_handler_response.cached_result, transcription_runs_unstreamed + ) elif _caching_handler_response.embedding_all_elements_cache_hit is True: return _caching_handler_response.final_embedding_cached_response @@ -2452,7 +2516,7 @@ def client(original_function): raise end_time = datetime.datetime.now() - streaming_requested: Final = _is_streaming_request(kwargs=kwargs, call_type=call_type) + streaming_requested: Final = _returns_requested_stream(kwargs, call_type, result) if streaming_requested or _is_converted_stream_result(result): logging_obj.stream = True logging_obj.model_call_details["stream"] = True @@ -2466,6 +2530,15 @@ def client(original_function): chunks.append(chunk) return litellm.stream_chunk_builder(chunks, messages=kwargs.get("messages", None)) else: + if call_type == CallTypes.atranscription.value and isinstance(result, openai.AsyncStream): + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, + ) + + result = wrap_transcription_stream( + result, logging_obj, start_time, litellm_exception_mapper(logging_obj.model_call_details) + ) _update_response_metadata( result=result, logging_obj=logging_obj, @@ -2492,7 +2565,9 @@ def client(original_function): _call_type_enum: Final = _CALL_TYPE_ENUM_MAP.get(call_type) if _call_type_enum is not None: result = await async_post_call_success_deployment_hook( - request_data=call_kwargs, + request_data=_guardrail_view_of_requested_stream(call_kwargs) + if transcription_runs_unstreamed + else call_kwargs, response=result, call_type=_call_type_enum, ) @@ -2543,7 +2618,7 @@ def client(original_function): is_litellm_internal_call=_is_litellm_internal_call, ) - return result + return _replayed_when_run_unstreamed(result, transcription_runs_unstreamed) except Exception as e: traceback_exception: Final = traceback.format_exc() # Reuse the timestamp taken right when the deployment call itself failed, before @@ -4333,40 +4408,61 @@ def _get_non_default_params(passed_params: dict, default_params: dict, additiona return non_default_params +_OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS: Final = frozenset(("languages", "keywords", "stream")) + + def get_optional_params_transcription( model: str, custom_llm_provider: str, language: str | None = None, + languages: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, prompt: str | None = None, response_format: str | None = None, temperature: int | None = None, timestamp_granularities: list[Literal["word", "segment"]] | None = None, + stream: bool | None = None, drop_params: bool | None = None, **kwargs, ): from litellm.constants import OPENAI_TRANSCRIPTION_PARAMS # retrieve all parameters passed to the function - passed_params: Final = locals() + passed_params: Final = locals().copy() passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS") - passed_params.pop("custom_llm_provider") - passed_params.pop("drop_params") - drop_params = normalize_drop_params(drop_params) + passed_params.pop("model") + custom_llm_provider = passed_params.pop("custom_llm_provider") + drop_params = normalize_drop_params(passed_params.pop("drop_params")) special_params: Final[Mapping[str, object]] = passed_params.pop("kwargs") + from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + transcription_stream_flag, + transcription_stream_unset, + ) + for k, v in special_params.items(): passed_params[k] = v - default_params: Final = { - "language": None, - "prompt": None, - "response_format": None, - "temperature": None, # openai defaults this to 0 - "timestamp_granularities": None, - } + sends_through_openai_sdk: Final = custom_llm_provider in ("openai", "azure") + drops_unsupported_params: Final = drop_params is True or litellm.drop_params is True + transcription_params: Final = ( + OPENAI_TRANSCRIPTION_PARAMS + if sends_through_openai_sdk or drops_unsupported_params + else [param for param in OPENAI_TRANSCRIPTION_PARAMS if param not in _OPENAI_SDK_ONLY_TRANSCRIPTION_PARAMS] + ) + default_params: Final = {param: None for param in (*transcription_params, "prompt", "temperature")} + stream_unset: Final = transcription_stream_unset(stream) + malformed_stream: Final = not stream_unset and transcription_stream_flag(stream) is None + if malformed_stream and not drops_unsupported_params: + raise UnsupportedParamsError( + status_code=400, + message="`stream` must be a boolean. To drop it from the call, set `litellm.drop_params = True`.", + ) non_default_params: Final = { - k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k]) + k: v + for k, v in passed_params.items() + if k in default_params and v != default_params[k] and not (k == "stream" and (stream_unset or malformed_stream)) } optional_params = {} @@ -4375,9 +4471,7 @@ def get_optional_params_transcription( if len(non_default_params.keys()) > 0: keys: Final = list(non_default_params.keys()) for k in keys: - if ( - drop_params is True or litellm.drop_params is True - ) and k not in supported_params: # drop the unsupported non-default values + if drops_unsupported_params and k not in supported_params: non_default_params.pop(k, None) elif k not in supported_params: raise UnsupportedParamsError( @@ -4431,9 +4525,12 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS, + openai_params=list(default_params) if custom_llm_provider == "bedrock" else transcription_params, additional_drop_params=kwargs.get("additional_drop_params", None), ) + extra_body: Final = optional_params.get("extra_body") + if isinstance(extra_body, dict) and not extra_body: + optional_params.pop("extra_body") return optional_params @@ -7182,6 +7279,7 @@ def get_model_info_helper( "cache_read_input_audio_token_cost_batches", None ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), + cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None @@ -10239,6 +10337,7 @@ class ProviderConfigManager: def get_provider_audio_transcription_config( model: str, provider: LlmProviders, + base_model: str | None = None, ) -> BaseAudioTranscriptionConfig | None: model_cost_entry: Final = _get_model_cost_entry_for_provider_config( model=model, @@ -10266,7 +10365,13 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): + if "gpt-transcribe" in (model, base_model): + from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, + ) + + return OpenAIGPTTranscribeAudioTranscriptionConfig() + elif "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 376cd816c66..d7c829f6e2d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1,4 +1,168 @@ { + "azure/gpt-realtime-2.1-2026-07-07": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image_token": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 2.4e-05, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 5e-07, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-2.1-mini-2026-07-07": { + "cache_creation_input_audio_token_cost": 3e-07, + "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_token_cost": 6e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_image_token": 8e-07, + "input_cost_per_token": 6e-07, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2.4e-06, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "cache_read_input_image_token_cost": 8e-08, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-realtime-translate-2026-05-06": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-translate-2026-05-07": { + "input_cost_per_second": 0.0005666666666666667, + "litellm_provider": "azure", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://ai.azure.com/catalog/models/gpt-realtime-translate", + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "audio", + "text" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_native_streaming": true + }, + "azure/gpt-realtime-whisper-2026-05-06": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, + "azure/gpt-realtime-whisper-2026-05-07": { + "input_cost_per_second": 0.0002833333333333333, + "litellm_provider": "azure", + "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", + "supported_endpoints": [ + "/v1/realtime", + "/v1/realtime/transcription_sessions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "max_input_tokens": 32000, + "max_output_tokens": 4096 + }, "sample_spec": { "code_interpreter_cost_per_session": 0.0, "computer_use_input_cost_per_1k_tokens": 0.0, @@ -6424,6 +6588,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6458,6 +6625,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6491,6 +6661,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6525,6 +6698,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 1.6e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6547,6 +6723,7 @@ "azure/gpt-realtime-2.1": { "cache_creation_input_audio_token_cost": 4e-07, "cache_read_input_audio_token_cost": 4e-07, + "cache_read_input_image_token_cost": 5e-07, "cache_read_input_token_cost": 4e-07, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 3.2e-05, @@ -6556,6 +6733,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6575,12 +6755,15 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, "azure/gpt-realtime-2.1-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, + "cache_read_input_image_token_cost": 8e-08, "cache_read_input_token_cost": 6e-08, "deprecation_date": "2027-06-25", "input_cost_per_audio_token": 1e-05, @@ -6590,6 +6773,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/model-retirement-schedule", @@ -6609,6 +6795,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -6626,6 +6814,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6659,6 +6850,9 @@ "mode": "realtime", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2.4e-06, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "supported_endpoints": [ "/v1/realtime" ], @@ -6791,6 +6985,9 @@ "max_input_tokens": 32000, "max_output_tokens": 4096, "mode": "audio_transcription", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper", "supported_endpoints": [ "/v1/realtime", @@ -29238,9 +29435,9 @@ }, "gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -39069,6 +39266,9 @@ "max_input_tokens": 128000, "max_output_tokens": 32000, "mode": "realtime", + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, "source": "https://developers.openai.com/api/docs/pricing", @@ -39124,6 +39324,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -39160,6 +39362,8 @@ "supports_audio_output": true, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true }, @@ -52587,9 +52791,9 @@ }, "vertex_ai/gemini-3-pro-image": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_above_200k_tokens": 4e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, - "cache_read_input_token_cost_batches": 1e-07, "cache_read_input_token_cost_flex": 1e-07, "cache_read_input_token_cost_priority": 3.6e-07, "deprecation_date": "2027-05-28", @@ -63405,6 +63609,10 @@ "input_cost_per_second": 7.5e-05, "litellm_provider": "openai", "mode": "audio_transcription", + "provider_specific_entry": { + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, "source": "https://developers.openai.com/api/docs/pricing", "supported_endpoints": [ "/v1/audio/transcriptions", @@ -63417,7 +63625,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-transcribe": { "input_cost_per_second": 0.000283333333333, @@ -63435,7 +63644,8 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "supports_native_streaming": true }, "gpt-live-1": { "input_cost_per_second": 0.000833333333333, @@ -63469,7 +63679,8 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "supports_native_streaming": true }, "claude-mythos-5": { "supports_anthropic_compaction": true, @@ -71340,6 +71551,9 @@ "mode": "realtime", "output_cost_per_audio_token": 6.4e-05, "output_cost_per_token": 2.4e-05, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/realtime" @@ -71397,7 +71611,11 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "azure/gpt-transcribe": { "deprecation_date": "2028-02-01", @@ -71407,7 +71625,8 @@ "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ "/v1/audio/transcriptions", - "/v1/realtime/transcription_sessions" + "/v1/realtime/transcription_sessions", + "/v1/realtime" ], "supported_modalities": [ "text", @@ -71416,7 +71635,13 @@ "supported_output_modalities": [ "text" ], - "supports_audio_input": true + "supports_audio_input": true, + "provider_specific_entry": { + "realtime_ga_only": 1, + "transcription_json_or_text_only": 1, + "transcription_usage_duration": 1 + }, + "supports_native_streaming": true }, "azure/gpt-realtime-translate": { "deprecation_date": "2027-11-09", @@ -71434,7 +71659,11 @@ "audio" ], "supports_audio_input": true, - "supports_audio_output": true + "supports_audio_output": true, + "provider_specific_entry": { + "realtime_ga_only": 1 + }, + "supports_native_streaming": true }, "aihubmix/agnes-2.5-flash": { "input_cost_per_token": 3e-08, diff --git a/scripts/check_openapi_docs.py b/scripts/check_openapi_docs.py index 26bb034bfa6..e29cac29212 100755 --- a/scripts/check_openapi_docs.py +++ b/scripts/check_openapi_docs.py @@ -18,6 +18,7 @@ from collections.abc import Iterator, Mapping from typing import Final, NamedTuple GATED_METHODS: Final = frozenset({"post"}) +SCHEMA_MEDIA_TYPES: Final = ("application/json", "application/sdp") class Violation(NamedTuple): @@ -30,18 +31,21 @@ def _as_mapping(node: object) -> Mapping[str, object]: return node if isinstance(node, Mapping) else {} -def _json_schema(container: object) -> object: +def _content_schema(container: object) -> object: content: Final = _as_mapping(_as_mapping(container).get("content")) - return _as_mapping(content.get("application/json")).get("schema") + return next( + (schema for media_type in SCHEMA_MEDIA_TYPES if (schema := _as_mapping(content.get(media_type)).get("schema"))), + None, + ) def _operation_codes(operation: Mapping[str, object]) -> Iterator[str]: - if not _json_schema(operation.get("requestBody")): + if not _content_schema(operation.get("requestBody")): yield "OA001" description: Final = operation.get("description") if not isinstance(description, str) or not description.strip(): yield "OA002" - if not _json_schema(_as_mapping(operation.get("responses")).get("200")): + if not _content_schema(_as_mapping(operation.get("responses")).get("200")): yield "OA003" diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 367135eae26..d99bcff46e0 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -77,6 +77,10 @@ - {id: llm.bedrock_native.bedrock_invoke.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native invoke stream"} - {id: llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_invoke, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock invoke missing fields and invalid temperature"} - {id: llm.ocr.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: ocr, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.13 / LIT-4778", rationale: "OCR missing document rejected"} +- {id: llm.realtime.openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: translation, streaming: stream, assertions: [works], source: "realtime_endpoints/endpoints.py", rationale: "Dedicated translation client-secret, raw SDP, and WebSocket paths emit translated audio and transcript deltas"} +- {id: llm.realtime.azure_openai.translation.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: translation, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA translation session emits translated audio and transcript deltas"} +- {id: llm.realtime.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/realtime/handler.py", rationale: "gpt-live-transcribe and gpt-realtime-whisper emit live transcript deltas"} +- {id: llm.realtime.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/realtime/handler.py", rationale: "Azure GA live transcription emits transcript deltas"} - {id: llm.rerank.bedrock.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "llms/bedrock/rerank/handler.py", rationale: "Bedrock rerank"} - {id: llm.rerank.together_ai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: together_ai, capability: basic, streaming: nonstream, assertions: [works], source: "llms/together_ai/rerank/handler.py", rationale: "Together rerank"} - {id: llm.images_generations.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: images_generations, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_image_generation_e2e.py:22", rationale: "OpenAI image gen, b64/url"} @@ -94,6 +98,8 @@ - {id: llm.audio_speech.vertex.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_speech, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "vertex_ai/text_to_speech/text_to_speech_handler.py", rationale: "Vertex TTS"} - {id: llm.audio_transcriptions.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "OpenAI Whisper"} - {id: llm.audio_transcriptions.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.7 / LIT-4778", rationale: "Transcription empty file and missing model are rejected"} +- {id: llm.audio_transcriptions.openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: openai, capability: transcription, streaming: stream, assertions: [works], source: "openai/transcriptions/handler.py", rationale: "gpt-transcribe streams typed transcript delta and done events"} +- {id: llm.audio_transcriptions.azure_openai.transcription.stream.works, module: llm, tier: P0, subject_endpoint: audio_transcriptions, route: azure_openai, capability: transcription, streaming: stream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure gpt-transcribe streams typed transcript delta and done events over the v1 API"} - {id: llm.audio_transcriptions.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: audio_transcriptions, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "azure/audio_transcriptions.py", rationale: "Azure STT"} - {id: llm.audio_transcriptions.soniox.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "soniox/audio_transcription/handler.py", rationale: "Soniox via OpenAI-compat (smoke)"} - {id: llm.audio_transcriptions.nvidia_riva.basic.nonstream.works, module: llm, tier: P2, subject_endpoint: audio_transcriptions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "nvidia_riva/audio_transcription/handler.py", rationale: "NVIDIA Riva (smoke)"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index d2ba2c325df..7d9bc380bd0 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -93,6 +93,8 @@ LlmCapability = Literal[ "tool_search_history", "tool_use", "upstream_stream_failure", + "transcription", + "translation", "vision", "web_search", "web_search_server_tool", @@ -162,14 +164,7 @@ class OtherCell(_Base): Cell = Annotated[ - LlmCell - | MgmtCell - | McpCell - | ReliabilityCell - | QuotaCell - | LoggingCell - | GuardrailCell - | OtherCell, + LlmCell | MgmtCell | McpCell | ReliabilityCell | QuotaCell | LoggingCell | GuardrailCell | OtherCell, Field(discriminator="module"), ] diff --git a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md index c85471da90d..49b543bf666 100644 --- a/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md +++ b/tests/e2e/llm_translation/realtime/REALTIME_COVERAGE_MATRIX.md @@ -19,6 +19,15 @@ a matching `function_call` output item, the test sends a `function_call_output` back, and the follow-up response incorporates the result (the temperature 72 appears). That raw-websocket tool path is the source of truth for tool calling. +The audio surfaces provision their own OpenAI and Azure deployments. +`TestRealtimeTranslation` streams the spoken weather question into +`/v1/realtime/translations` with French as the target and asserts translated +audio, a French transcript, and a billed spend row. `TestRealtimeLiveTranscription` +streams it into `/v1/realtime?intent=transcription` on gpt-live-transcribe and +asserts the deltas add up to the completed transcript and the session is billed. +`test_azure_ga_text_turn_is_answered_and_billed` guards a plain text turn over +the Azure GA deployment, spend row included + `test_pipecat_tool_smoke` is a realism layer through pipecat for openai, azure, and gemini only (not vertex_ai: native-audio live is flaky under pipecat tool calling while raw-ws tools pass; see pipecat-ai/pipecat#2544). Assertions are diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 6bd5d61d721..7e7f9f424a2 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -29,9 +29,13 @@ from models import LiteLLMParamsBody _M = TypeVar("_M", bound=BaseModel) +REALTIME_ROUTE: Final = "/v1/realtime" +TRANSLATIONS_ROUTE: Final = "/v1/realtime/translations" -def realtime_ws_url(model: str) -> str: - return f"{ws_base_url()}/v1/realtime?{urlencode({'model': model})}" + +def realtime_ws_url(model: str, *, route: str = REALTIME_ROUTE, intent: str | None = None) -> str: + query: Final = {"model": model} if intent is None else {"model": model, "intent": intent} + return f"{ws_base_url()}{route}?{urlencode(query)}" @dataclass(frozen=True, slots=True) @@ -183,6 +187,75 @@ class InputAudioBufferAppend(BaseModel): audio: str +class InputAudioBufferCommit(BaseModel): + type: str = "input_audio_buffer.commit" + + +class SessionInputAudioBufferAppend(BaseModel): + type: str = "session.input_audio_buffer.append" + audio: str + + +AudioAppend = InputAudioBufferAppend | SessionInputAudioBufferAppend + + +class TranslationOutput(BaseModel): + language: str + + +class TranslationAudio(BaseModel): + output: TranslationOutput + + +class TranslationSessionConfig(BaseModel): + audio: TranslationAudio + + +class TranslationSessionUpdate(BaseModel): + type: str = "session.update" + session: TranslationSessionConfig + + +class PcmFormat(BaseModel): + type: str = "audio/pcm" + rate: int = 24000 + + +class InputTranscription(BaseModel): + model: str + + +class TranscriptionAudioInput(BaseModel): + format: PcmFormat = PcmFormat() + transcription: InputTranscription + + +class TranscriptionAudio(BaseModel): + input: TranscriptionAudioInput + + +class TranscriptionSessionConfig(BaseModel): + type: str = "transcription" + audio: TranscriptionAudio + + +class TranscriptionSessionUpdate(BaseModel): + type: str = "session.update" + session: TranscriptionSessionConfig + + +RealtimeClientEvent = ( + SessionUpdate + | TranslationSessionUpdate + | TranscriptionSessionUpdate + | ConversationItemCreate + | ResponseCreate + | InputAudioBufferAppend + | InputAudioBufferCommit + | SessionInputAudioBufferAppend +) + + def user_message(text: str) -> ConversationItemCreate: return ConversationItemCreate( item=MessageItem(content=[InputTextContent(text=text)]) @@ -320,7 +393,7 @@ class RealtimeSession: connection: Connection @step("Send the realtime event {event.type} over the websocket") - def send(self, event: SessionUpdate | ConversationItemCreate | ResponseCreate | InputAudioBufferAppend) -> None: + def send(self, event: RealtimeClientEvent) -> None: self.connection.send(event.model_dump_json(by_alias=True, exclude_none=True)) @step("Wait for a {stop_type} event on the realtime websocket") @@ -350,9 +423,9 @@ class RealtimeSession: @step("Stream the spoken question as mic chunks, then silence until the turn goes idle") def stream_and_collect( self, - chunks: Sequence[InputAudioBufferAppend], + chunks: Sequence[AudioAppend], *, - tail: InputAudioBufferAppend, + tail: AudioAppend, interval: float, idle: float, timeout: float, @@ -363,9 +436,9 @@ class RealtimeSession: def _live_mic_events( self, - chunks: Sequence[InputAudioBufferAppend], + chunks: Sequence[AudioAppend], *, - tail: InputAudioBufferAppend, + tail: AudioAppend, interval: float, idle: float, timeout: float, @@ -408,13 +481,19 @@ class RealtimeClient: ) return model_name, model_id - @step("Open a /v1/realtime websocket session to {model}") + @step("Open a {route} websocket session to {model}") @contextmanager def connect( - self, *, key: str, model: str, timeout: float = 15.0 + self, + *, + key: str, + model: str, + route: str = REALTIME_ROUTE, + intent: str | None = None, + timeout: float = 15.0, ) -> Generator[RealtimeSession, None, None]: with connect( - realtime_ws_url(model), + realtime_ws_url(model, route=route, intent=intent), additional_headers={"Authorization": f"Bearer {key}"}, open_timeout=timeout, ) as connection: diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index 3b7ba3a2430..9645e31f50f 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -4,7 +4,10 @@ Each test opens a websocket through the proxy, speaks the OpenAI GA realtime event schema, and asserts the proxy normalizes the provider's stream into that schema: the session lifecycle, the canonical response event sequence with a reconstructed transcript and usage, and a full tool-call round-trip (call -> -tool result -> a follow-up response that uses the result). +tool result -> a follow-up response that uses the result). The audio tests +stream the spoken weather question into /v1/realtime/translations and into a +/v1/realtime?intent=transcription session, and assert the translated or +transcribed output and the session's spend row. One GA-speaking client validates every provider; only the model alias changes. Every provider is provisioned at session start, so a missing realtime alias is a @@ -12,19 +15,28 @@ hard failure, not a skip; once configured, a protocol failure is likewise a hard failure. See REALTIME_COVERAGE_MATRIX.md. """ +import base64 +import wave +from pathlib import Path +from types import MappingProxyType from typing import Final import pytest from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import LiteLLMParamsBody, SpendLogRow from pydantic import BaseModel from realtime_client import ( PROVIDERS, + TRANSLATIONS_ROUTE, ConversationItemCreate, + DeltaEvent, FunctionCallArgumentsDone, FunctionCallOutputItem, FunctionTool, + InputAudioBufferAppend, + InputAudioBufferCommit, + InputTranscription, JsonSchema, JsonSchemaProperty, RealtimeClient, @@ -35,8 +47,18 @@ from realtime_client import ( ResponseDone, ServerEnvelope, SessionConfig, + SessionInputAudioBufferAppend, SessionUpdate, + TranscriptionAudio, + TranscriptionAudioInput, + TranscriptionSessionConfig, + TranscriptionSessionUpdate, + TranslationAudio, + TranslationOutput, + TranslationSessionConfig, + TranslationSessionUpdate, as_text, + events_of_type, parse_last, realtime_model, transcript, @@ -264,3 +286,204 @@ def test_upstream_handshake_refusal_is_an_error_event_and_policy_close( assert closed.value.rcvd is not None assert closed.value.rcvd.code == 1008, closed.value + + +WEATHER_WAV: Final = Path(__file__).parent / "fixtures" / "weather_question_24k.wav" +PCM_RATE: Final = 24000 +CHUNK_SECONDS: Final = 0.1 +CHUNK_BYTES: Final = int(PCM_RATE * CHUNK_SECONDS) * 2 +SILENCE_B64: Final = base64.b64encode(bytes(CHUNK_BYTES)).decode() +REGISTRY_ROUTES: Final = MappingProxyType({"openai": "openai", "azure": "azure_openai"}) + + +def weather_pcm_chunks() -> tuple[str, ...]: + with wave.open(str(WEATHER_WAV), "rb") as wav: + assert (wav.getnchannels(), wav.getsampwidth(), wav.getframerate()) == (1, 2, PCM_RATE), ( + "fixture must be mono PCM16 at 24 kHz" + ) + pcm: Final = wav.readframes(wav.getnframes()) + return tuple(base64.b64encode(pcm[i : i + CHUNK_BYTES]).decode() for i in range(0, len(pcm), CHUNK_BYTES)) + + +def has_billed_row(rows: list[SpendLogRow]) -> bool: + return any((row.spend or 0) > 0 for row in rows) + + +def audio_marks(provider: RealtimeProvider, capability: str) -> tuple[pytest.MarkDecorator, ...]: + return ( + pytest.mark.covers(f"llm.realtime.{REGISTRY_ROUTES[provider.id]}.{capability}.stream.works"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider(provider.id),), + models=(provider.litellm_params.model,), + capabilities=(Capability.AUDIO_INPUT,), + mode=Mode.WEBSOCKET, + ) + ), + ) + + +TRANSLATION_PROVIDERS: Final = ( + RealtimeProvider( + "openai", + "openai-translate", + LiteLLMParamsBody(model="openai/gpt-realtime-translate", api_key="os.environ/OPENAI_API_KEY"), + ), + RealtimeProvider( + "azure", + "azure-translate", + LiteLLMParamsBody( + model="azure/gpt-realtime-translate", + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + ), + ), +) + +LIVE_TRANSCRIPTION_PROVIDERS: Final = ( + RealtimeProvider( + "openai", + "openai-live-transcribe", + LiteLLMParamsBody(model="openai/gpt-live-transcribe", api_key="os.environ/OPENAI_API_KEY"), + ), + RealtimeProvider( + "azure", + "azure-live-transcribe", + LiteLLMParamsBody( + model="azure/gpt-live-transcribe", + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + ), + ), +) + +TRANSLATION_PARAMS: Final = tuple( + pytest.param(p, id=p.id, marks=audio_marks(p, "translation")) for p in TRANSLATION_PROVIDERS +) +LIVE_TRANSCRIPTION_PARAMS: Final = tuple( + pytest.param(p, id=p.id, marks=audio_marks(p, "transcription")) for p in LIVE_TRANSCRIPTION_PROVIDERS +) +AZURE_GA_PROVIDER: Final = next(p for p in PROVIDERS if p.id == "azure") + + +class TranscriptionCompleted(BaseModel): + type: str + transcript: str + + +class TestRealtimeTranslation: + @pytest.mark.parametrize("provider", TRANSLATION_PARAMS) + def test_spoken_english_comes_back_as_french_audio_and_transcript_and_is_billed( + self, + client: RealtimeClient, + resources: ResourceManager, + scoped_key: str, + provider: RealtimeProvider, + ) -> None: + model, model_id = client.provision(provider) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model, route=TRANSLATIONS_ROUTE) as session: + session.send( + TranslationSessionUpdate( + session=TranslationSessionConfig(audio=TranslationAudio(output=TranslationOutput(language="fr"))) + ) + ) + session.collect_until("session.updated", timeout=30) + events: Final = session.stream_and_collect( + tuple(SessionInputAudioBufferAppend(audio=chunk) for chunk in weather_pcm_chunks()), + tail=SessionInputAudioBufferAppend(audio=SILENCE_B64), + interval=CHUNK_SECONDS, + idle=8, + timeout=60, + ) + + types: Final = tuple(e.type for e in events) + audio_bytes: Final = sum( + len(base64.b64decode(DeltaEvent.model_validate_json(e.payload).delta)) + for e in events_of_type(events, "session.output_audio.delta") + ) + assert audio_bytes > 0, f"translation returned no translated audio; types={types}" + translated: Final = "".join( + DeltaEvent.model_validate_json(e.payload).delta + for e in events_of_type(events, "session.output_transcript.delta") + ).lower() + assert translated.strip(), f"translation returned no translated text; types={types}" + assert "weather" not in translated, f"the spoken weather question came back untranslated: {translated!r}" + + rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row) + assert has_billed_row(rows), f"the translation session was not billed: {rows}" + + +class TestRealtimeLiveTranscription: + @pytest.mark.parametrize("provider", LIVE_TRANSCRIPTION_PARAMS) + def test_spoken_question_streams_transcript_deltas_that_match_the_completed_transcript( + self, + client: RealtimeClient, + resources: ResourceManager, + scoped_key: str, + provider: RealtimeProvider, + ) -> None: + model, model_id = client.provision(provider) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model, intent="transcription") as session: + session.send( + TranscriptionSessionUpdate( + session=TranscriptionSessionConfig( + audio=TranscriptionAudio( + input=TranscriptionAudioInput(transcription=InputTranscription(model=model)) + ) + ) + ) + ) + session.collect_until("session.updated", timeout=30) + for chunk in weather_pcm_chunks(): + session.send(InputAudioBufferAppend(audio=chunk)) + session.send(InputAudioBufferCommit()) + events: Final = session.collect_until("conversation.item.input_audio_transcription.completed", timeout=60) + + completed: Final = TranscriptionCompleted.model_validate_json(events[-1].payload) + assert "weather" in completed.transcript.lower(), ( + f"transcript of a spoken weather question does not mention weather: {completed.transcript!r}" + ) + streamed: Final = "".join( + DeltaEvent.model_validate_json(e.payload).delta + for e in events_of_type(events, "conversation.item.input_audio_transcription.delta") + ) + assert streamed.strip() == completed.transcript.strip(), ( + f"streamed deltas {streamed!r} do not add up to the completed transcript {completed.transcript!r}" + ) + + rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row) + assert has_billed_row(rows), f"the live transcription session was not billed: {rows}" + + +@meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.AZURE,), + models=(AZURE_GA_PROVIDER.litellm_params.model,), + mode=Mode.WEBSOCKET, + ) +) +def test_azure_ga_text_turn_is_answered_and_billed( + client: RealtimeClient, resources: ResourceManager, scoped_key: str +) -> None: + model, model_id = client.provision(AZURE_GA_PROVIDER) + resources.defer(lambda: client.proxy.delete_model(model_id)) + + with client.connect(key=scoped_key, model=model) as session: + session.collect_until("session.created", timeout=20) + session.send(SessionUpdate(session=SessionConfig(instructions="Reply with the single word hello."))) + session.collect_until("session.updated", timeout=20) + session.send(user_message("Say the single word hello.")) + session.send(ResponseCreate()) + events: Final = session.collect_until("response.done", timeout=60) + + assert "hello" in transcript(events).lower(), f"Azure GA realtime did not answer the turn: {transcript(events)!r}" + rows: Final = client.proxy.poll_logs_for_key(scoped_key, predicate=has_billed_row) + assert has_billed_row(rows), f"the Azure GA realtime turn was not billed: {rows}" diff --git a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py index cd1ee8fb03c..43c276dfdf0 100644 --- a/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py +++ b/tests/e2e/llm_translation/test_audio_transcriptions_e2e.py @@ -8,10 +8,13 @@ the shared multipart transport, since the SDK refuses to send them. A model-less request comes back as one of two 400s depending on whether any wildcard deployment happens to be registered on the shared proxy, so the assertion accepts either phrasing and holds both to naming the model as the problem. +Streamed gpt-transcribe (stream=true) on OpenAI and Azure must send transcript +deltas, a done event, and data: [DONE], and bill the call. """ from __future__ import annotations +from dataclasses import dataclass from pathlib import Path from typing import Final @@ -20,10 +23,10 @@ from e2e_config import unique_marker from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from e2e_http import UnknownApiError, unwrap from lifecycle import ResourceManager -from models import LiteLLMParamsBody +from models import LiteLLMParamsBody, SpendLogRow from proxy_client import ProxyClient from pydantic import BaseModel, Field -from sdk_clients import SdkClients +from sdk_clients import SdkClients, response_header pytestmark = pytest.mark.e2e @@ -221,3 +224,102 @@ class TestWhisperTranscriptionFormats: assert all(word.start <= word.end for word in transcript.words), ( f"word timings out of order: {transcript.words}" ) + + +STREAM_DONE: Final = "[DONE]" + + +@dataclass(frozen=True, slots=True) +class _StreamingTranscriber: + id: str + registry_route: str + provider: Provider + litellm_params: LiteLLMParamsBody + + +STREAMING_TRANSCRIBERS: Final = ( + _StreamingTranscriber( + "openai", + "openai", + Provider.OPENAI, + LiteLLMParamsBody(model="openai/gpt-transcribe", api_key="os.environ/OPENAI_API_KEY"), + ), + _StreamingTranscriber( + "azure", + "azure_openai", + Provider.AZURE, + LiteLLMParamsBody( + model="azure/gpt-transcribe", + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version="v1", + ), + ), +) + +STREAMING_PARAMS: Final = tuple( + pytest.param( + t, + id=t.id, + marks=( + pytest.mark.covers(f"llm.audio_transcriptions.{t.registry_route}.transcription.stream.works"), + meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.AUDIO, + providers=(t.provider,), + models=(t.litellm_params.model,), + mode=Mode.STREAM, + ) + ), + ), + ) + for t in STREAMING_TRANSCRIBERS +) + + +class _TranscriptStreamEvent(BaseModel): + type: str + delta: str = "" + text: str = "" + + +def _has_billed_row(rows: list[SpendLogRow]) -> bool: + return any((row.spend or 0) > 0 for row in rows) + + +class TestStreamedAudioTranscriptions: + @pytest.mark.parametrize("transcriber", STREAMING_PARAMS) + def test_stream_sends_transcript_deltas_then_done_and_bills_the_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients, transcriber: _StreamingTranscriber + ) -> None: + model: Final = f"e2e-transcribe-stream-{transcriber.id}-{unique_marker()}" + model_id: Final = proxy.create_model(model, transcriber.litellm_params) + resources.defer(lambda: proxy.delete_model(model_id)) + client: Final = sdk.openai(resources.key()) + + with client.audio.transcriptions.with_streaming_response.create( + model=model, file=(WEATHER_WAV.name, WEATHER_WAV.read_bytes(), "audio/wav"), stream=True + ) as response: + content_type: Final = response_header(response.headers, "content-type") + call_id: Final = response_header(response.headers, "x-litellm-call-id") + payloads: Final = tuple( + line.removeprefix("data: ") for line in response.iter_lines() if line.startswith("data: ") + ) + + assert "text/event-stream" in (content_type or ""), f"stream=true did not stream: {content_type!r}" + assert payloads and payloads[-1] == STREAM_DONE, f"the stream did not end with data: [DONE]: {payloads[-3:]}" + assert payloads.count(STREAM_DONE) == 1, f"data: [DONE] was sent more than once: {payloads}" + events: Final = tuple(_TranscriptStreamEvent.model_validate_json(payload) for payload in payloads[:-1]) + deltas: Final = tuple(e.delta for e in events if e.type == "transcript.text.delta") + assert len(deltas) > 1, f"the transcript arrived in {len(deltas)} delta events, not as a stream: {events}" + done: Final = events[-1] + assert done.type == "transcript.text.done", f"the last event before [DONE] is not the done event: {done}" + assert "weather" in done.text.lower(), f"transcript of a spoken weather question lost the word: {done.text!r}" + assert "".join(deltas).strip() == done.text.strip(), ( + f"the deltas {deltas} do not add up to the final transcript {done.text!r}" + ) + + assert call_id, "the streamed response carries no x-litellm-call-id to find its spend log" + rows: Final = proxy.poll_logs_for_request_id(call_id, predicate=_has_billed_row) + assert _has_billed_row(rows), f"the streamed transcription was not billed: {rows}" diff --git a/tests/e2e/llm_translation/test_realtime_http_e2e.py b/tests/e2e/llm_translation/test_realtime_http_e2e.py index ee3ce3893c2..3abb74308e8 100644 --- a/tests/e2e/llm_translation/test_realtime_http_e2e.py +++ b/tests/e2e/llm_translation/test_realtime_http_e2e.py @@ -1,14 +1,18 @@ """Vendor §9.19: realtime client_secrets + calls HTTP surface (LIT-4778). Websocket coverage already lives under realtime/; this file pins the HTTP -client-secret mint and the missing-auth contract. +client-secret mint (realtime, translation and transcription sessions), the +translation token scope, and the missing-auth contract. """ from __future__ import annotations +from dataclasses import dataclass +from typing import Final + import pytest from e2e_config import unique_marker -from e2e_http import NoBody, assert_auth_denied, unwrap +from e2e_http import NoBody, PartialBody, UnknownApiError, assert_auth_denied, unwrap from e2e_metadata import Domain, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody @@ -18,6 +22,7 @@ from pydantic import BaseModel pytestmark = pytest.mark.e2e REALTIME_BACKEND = "openai/gpt-realtime" +TRANSLATIONS_CLIENT_SECRETS: Final = "/v1/realtime/translations/client_secrets" class RealtimeSession(BaseModel): @@ -38,8 +43,65 @@ class RealtimeClientSecretRequest(BaseModel): session: RealtimeSession | None = None +class TurnDetection(BaseModel): + type: str + + +class SessionTranscription(BaseModel): + model: str | None = None + + +class SessionAudioInput(BaseModel): + transcription: SessionTranscription | None = None + turn_detection: TurnDetection | None = None + + +class SessionAudioOutput(BaseModel): + language: str | None = None + + +class SessionAudio(BaseModel): + input: SessionAudioInput | None = None + output: SessionAudioOutput | None = None + + +class TranslationSession(BaseModel): + model: str + audio: SessionAudio + + +class TranscriptionSession(BaseModel): + type: str = "transcription" + audio: SessionAudio + + +class SessionClientSecretRequest(BaseModel): + session: TranslationSession | TranscriptionSession + + +class NullableAudioInput(PartialBody): + turn_detection: TurnDetection | None = None + + +class NullableAudio(PartialBody): + input: NullableAudioInput + + +class NullableRealtimeSession(PartialBody): + type: str + model: str + instructions: str | None = None + audio: NullableAudio + + +class NullableClientSecretRequest(PartialBody): + model: str + session: NullableRealtimeSession + + class RealtimeClientSecretSession(BaseModel): type: str | None = None + audio: SessionAudio | None = None class RealtimeClientSecretResponse(BaseModel): @@ -49,15 +111,85 @@ class RealtimeClientSecretResponse(BaseModel): def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]: - model = f"e2e-realtime-http-{unique_marker()}" - model_id = proxy.create_model( - model, - LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY"), + return _register_backend( + proxy, resources, LiteLLMParamsBody(model=REALTIME_BACKEND, api_key="os.environ/OPENAI_API_KEY") ) + + +def _register_backend( + proxy: ProxyClient, resources: ResourceManager, litellm_params: LiteLLMParamsBody +) -> tuple[str, str]: + model: Final = f"e2e-realtime-http-{unique_marker()}" + model_id: Final = proxy.create_model(model, litellm_params) resources.defer(lambda: proxy.delete_model(model_id)) return model, resources.key() +@dataclass(frozen=True, slots=True) +class _AudioBackend: + id: str + registry_route: str + provider: Provider + litellm_params: LiteLLMParamsBody + + @property + def upstream_model(self) -> str: + return self.litellm_params.model.split("/", 1)[1] + + +def _openai_backend(model: str) -> _AudioBackend: + return _AudioBackend( + "openai", "openai", Provider.OPENAI, LiteLLMParamsBody(model=model, api_key="os.environ/OPENAI_API_KEY") + ) + + +def _azure_backend(model: str) -> _AudioBackend: + return _AudioBackend( + "azure", + "azure_openai", + Provider.AZURE, + LiteLLMParamsBody(model=model, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"), + ) + + +TRANSLATION_BACKENDS: Final = ( + _openai_backend("openai/gpt-realtime-translate"), + _azure_backend("azure/gpt-realtime-translate"), +) +TRANSCRIPTION_BACKENDS: Final = ( + _openai_backend("openai/gpt-live-transcribe"), + _azure_backend("azure/gpt-live-transcribe"), +) + + +def _subject(backend: _AudioBackend) -> pytest.MarkDecorator: + return meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(backend.provider,), + models=(backend.litellm_params.model,), + ) + ) + + +TRANSLATION_PARAMS: Final = tuple( + pytest.param( + b, + id=b.id, + marks=(pytest.mark.covers(f"llm.realtime.{b.registry_route}.translation.stream.works"), _subject(b)), + ) + for b in TRANSLATION_BACKENDS +) +TRANSCRIPTION_PARAMS: Final = tuple(pytest.param(b, id=b.id, marks=_subject(b)) for b in TRANSCRIPTION_BACKENDS) + + +def _french_translation_request(model: str) -> SessionClientSecretRequest: + return SessionClientSecretRequest( + session=TranslationSession(model=model, audio=SessionAudio(output=SessionAudioOutput(language="fr"))) + ) + + class TestRealtimeHttp: @pytest.mark.covers("llm.realtime.openai.basic.nonstream.works") @meta( @@ -110,3 +242,150 @@ class TestRealtimeHttp: json=NoBody(), ) assert_auth_denied(result, "realtime calls missing auth") + + +class TestRealtimeTranslationClientSecrets: + @pytest.mark.parametrize("backend", TRANSLATION_PARAMS) + def test_translations_route_mints_a_french_translation_session( + self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend + ) -> None: + model, key = _register_backend(proxy, resources, backend.litellm_params) + secret: Final = unwrap( + proxy.transport.post( + TRANSLATIONS_CLIENT_SECRETS, + headers=proxy.transport.bearer(key), + json=_french_translation_request(model), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"client secret value missing: {secret}" + assert secret.session is not None and secret.session.type == "translation", ( + f"the translations route did not mint a translation session: {secret.session}" + ) + output: Final = secret.session.audio.output if secret.session.audio else None + assert output is not None and output.language == "fr", ( + f"the requested target language did not reach the session: {secret.session}" + ) + + @meta( + Subject( + domain=Domain.PROXY_AUTH, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(TRANSLATION_BACKENDS[0].litellm_params.model,), + ) + ) + def test_translation_token_is_refused_by_the_realtime_calls_route( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params) + secret: Final = unwrap( + proxy.transport.post( + TRANSLATIONS_CLIENT_SECRETS, + headers=proxy.transport.bearer(key), + json=_french_translation_request(model), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"client secret value missing: {secret}" + result: Final = proxy.transport.send( + "/v1/realtime/calls", headers=proxy.transport.bearer(secret.value), json=NoBody() + ) + assert result.status_code == 401, ( + f"a translation token opened a plain realtime call: {result.status_code} {result.body[:300]}" + ) + assert "not valid for this Realtime endpoint" in result.body, ( + f"the refusal does not say the token belongs to another endpoint: {result.body[:300]}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(TRANSLATION_BACKENDS[0].litellm_params.model,), + ) + ) + def test_translation_session_on_the_realtime_route_is_rejected( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model, key = _register_backend(proxy, resources, TRANSLATION_BACKENDS[0].litellm_params) + result: Final = proxy.transport.post( + "/v1/realtime/client_secrets", + headers=proxy.transport.bearer(key), + json=RealtimeClientSecretRequest(model=model, session=RealtimeSession(type="translation", model=model)), + response_type=RealtimeClientSecretResponse, + ) + match result: + case UnknownApiError(status_code=400, body=body): + assert "translations endpoint" in body, f"the 400 does not point at the translations route: {body}" + case other: + pytest.fail(f"a translation session on /v1/realtime/client_secrets expected a 400, got {other!r}") + + +class TestRealtimeTranscriptionClientSecrets: + @pytest.mark.parametrize("backend", TRANSCRIPTION_PARAMS) + def test_transcription_session_runs_on_the_deployment_model( + self, proxy: ProxyClient, resources: ResourceManager, backend: _AudioBackend + ) -> None: + model, key = _register_backend(proxy, resources, backend.litellm_params) + secret: Final = unwrap( + proxy.transport.post( + "/v1/realtime/client_secrets", + headers=proxy.transport.bearer(key), + json=SessionClientSecretRequest( + session=TranscriptionSession( + audio=SessionAudio(input=SessionAudioInput(transcription=SessionTranscription(model=model))) + ) + ), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"client secret value missing: {secret}" + assert secret.session is not None and secret.session.type == "transcription", ( + f"the client secret did not mint a transcription session: {secret.session}" + ) + audio_input: Final = secret.session.audio.input if secret.session.audio else None + transcription: Final = audio_input.transcription if audio_input else None + assert transcription is not None and transcription.model == backend.upstream_model, ( + f"the session does not transcribe with the deployment's model {backend.upstream_model}: {secret.session}" + ) + + +class TestRealtimeClientSecretNulls: + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.REALTIME, + providers=(Provider.OPENAI,), + models=(REALTIME_BACKEND,), + ) + ) + def test_top_level_null_is_dropped_and_nested_null_is_kept( + self, proxy: ProxyClient, resources: ResourceManager + ) -> None: + model, key = _register(proxy, resources) + secret: Final = unwrap( + proxy.transport.post( + "/v1/realtime/client_secrets", + headers=proxy.transport.bearer(key), + json=NullableClientSecretRequest( + model=model, + session=NullableRealtimeSession( + type="realtime", + model=model, + instructions=None, + audio=NullableAudio(input=NullableAudioInput(turn_detection=None)), + ), + ), + response_type=RealtimeClientSecretResponse, + ) + ) + assert secret.value, f"a session with instructions: null was not minted: {secret}" + audio_input: Final = secret.session.audio.input if secret.session and secret.session.audio else None + assert audio_input is not None and "turn_detection" in audio_input.model_fields_set, ( + f"the session does not report turn detection: {secret.session}" + ) + assert audio_input.turn_detection is None, ( + f"turn_detection: null was dropped, so the session fell back to {audio_input.turn_detection}" + ) diff --git a/tests/integration/compatibility/test_audio_transcriptions_wire.py b/tests/integration/compatibility/test_audio_transcriptions_wire.py index 3deeaebaf7d..faebfda715b 100644 --- a/tests/integration/compatibility/test_audio_transcriptions_wire.py +++ b/tests/integration/compatibility/test_audio_transcriptions_wire.py @@ -345,10 +345,6 @@ def test_audio_transcription_forwards_nested_and_repeated_bracket_fields_verbati def test_audio_transcription_stream_returns_server_sent_events(gateway: Gateway) -> None: - pytest.skip( - "BUG: stream=true transcriptions reach the upstream but its SSE body comes back as application/json " - '{"text": "data: ..."}, so OpenAI SDK stream=True callers get no transcript events' - ) wav: Final = _WAV_BYTES + uuid.uuid4().bytes def respond(request: Request) -> Reply: diff --git a/tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py b/tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py new file mode 100644 index 00000000000..358b36cd2b0 --- /dev/null +++ b/tests/integration/sdk/test_openai_realtime_http_ca_bundle_wire.py @@ -0,0 +1,95 @@ +import json +from collections.abc import Awaitable +from datetime import datetime, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.wire import Reply, Request, wire_server + +import litellm +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.realtime_api import main as realtime_main + +UPSTREAM_PATHS: Final = MappingProxyType( + { + "client_secrets": "/v1/realtime/client_secrets", + "translation_client_secrets": "/v1/realtime/translations/client_secrets", + "calls": "/v1/realtime/calls", + "translation_calls": "/v1/realtime/translations/calls", + } +) + + +def _logging(route: str) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id=f"test-realtime-{route}", + function_id=f"test-realtime-{route}", + ) + + +def _route_call(route: str, api_base: str) -> Awaitable[httpx.Response]: + match route: + case "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + custom_llm_provider="openai", + api_key="sk-test", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + case "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + custom_llm_provider="openai", + api_key="sk-test", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + case "calls": + return realtime_main.arealtime_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime", + custom_llm_provider="openai", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + case _: + return realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime-translate", + custom_llm_provider="openai", + api_base=api_base, + litellm_logging_obj=_logging(route), + ) + + +def _ephemeral_key(_: Request) -> Reply: + return Reply(body=json.dumps({"value": "ek_test"}).encode()) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", tuple(UPSTREAM_PATHS)) +async def test_openai_realtime_http_routes_trust_the_ca_bundle_set_in_litellm_ssl_verify( + route: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + certificate, key = write_self_signed_cert(tmp_path) + monkeypatch.setattr(litellm, "ssl_verify", str(certificate)) + + with wire_server(_ephemeral_key, tls=server_context(certificate, key)) as upstream: + response: Final = await _route_call(route, upstream.url) + targets: Final = tuple(request.target for request in upstream.drain()) + + assert response.status_code == 200 + assert response.json() == {"value": "ek_test"} + assert targets == (UPSTREAM_PATHS[route],) diff --git a/tests/unit/cookbook/__init__.py b/tests/unit/cookbook/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/cookbook/test_gpt_realtime_translate.py b/tests/unit/cookbook/test_gpt_realtime_translate.py new file mode 100644 index 00000000000..a40a3265600 --- /dev/null +++ b/tests/unit/cookbook/test_gpt_realtime_translate.py @@ -0,0 +1,141 @@ +import asyncio +import base64 +import json +import wave +from pathlib import Path +from types import SimpleNamespace +from typing import Final, cast + +import pytest +from websockets.asyncio.client import ClientConnection + +from cookbook import gpt_realtime_translate as translate + + +@pytest.mark.asyncio +async def test_short_upload_waits_for_first_translated_audio(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.1) + audio: Final = bytes(480) + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()}, + {"type": "error", "error": {"message": "session closed"}}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event["type"] == "session.output_audio.delta": + await asyncio.sleep(0.03) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result == 'Realtime API error: {"message": "session closed"}' + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio + + +@pytest.mark.asyncio +async def test_long_upload_keeps_waiting_for_first_translated_audio_while_still_sending( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05) + audio: Final = bytes(480) + pending: Final[asyncio.Queue[str]] = asyncio.Queue() + sender_finished: Final = asyncio.Event() + + async def speak_after_a_long_pause() -> None: + await asyncio.sleep(0.2) + await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()})) + await asyncio.sleep(0.02) + sender_finished.set() + await pending.put(json.dumps({"type": "session.closed"})) + + output: Final = tmp_path / "translation.wav" + speaker: Final = asyncio.create_task(speak_after_a_long_pause()) + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=pending.get)), output, sender_finished + ) + await speaker + + assert result is None + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(240) == audio + + +@pytest.mark.asyncio +async def test_a_quiet_gap_after_the_sender_stops_keeps_the_rest_of_the_translation( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 1.0) + first: Final = bytes(480) + second: Final = bytes(range(240)) * 2 + events: Final = iter( + ( + {"type": "session.output_audio.delta", "delta": base64.b64encode(first).decode()}, + {"type": "session.output_audio.delta", "delta": base64.b64encode(second).decode()}, + {"type": "session.closed"}, + ) + ) + + async def recv() -> str: + event: Final = next(events) + if event.get("delta") == base64.b64encode(second).decode(): + await asyncio.sleep(0.3) + return json.dumps(event) + + sender_finished: Final = asyncio.Event() + sender_finished.set() + output: Final = tmp_path / "translation.wav" + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=recv)), output, sender_finished + ) + + assert result is None + with wave.open(str(output), "rb") as rendered: + assert rendered.readframes(480) == first + second + + +@pytest.mark.asyncio +async def test_a_stream_that_never_closes_the_session_reports_an_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr(translate, "RESPONSE_TIMEOUT_SECONDS", 0.05) + audio: Final = bytes(480) + pending: Final[asyncio.Queue[str]] = asyncio.Queue() + await pending.put(json.dumps({"type": "session.output_audio.delta", "delta": base64.b64encode(audio).decode()})) + sender_finished: Final = asyncio.Event() + sender_finished.set() + + result: Final = await translate.receive_translation( + cast(ClientConnection, SimpleNamespace(recv=pending.get)), tmp_path / "translation.wav", sender_finished + ) + + assert result == "The translation stream went quiet for 0.05 s without closing the session" + + +@pytest.mark.asyncio +async def test_the_sender_closes_the_session_after_the_last_audio_chunk() -> None: + sent: Final[asyncio.Queue[str]] = asyncio.Queue() + finished: Final = asyncio.Event() + pcm: Final = bytes(translate.CHUNK_BYTES + 2) + + await translate.send_audio(cast(ClientConnection, SimpleNamespace(send=sent.put)), pcm, 0.0, finished) + + messages: Final = tuple(json.loads(sent.get_nowait()) for _ in range(sent.qsize())) + assert [message["type"] for message in messages] == [ + "session.input_audio_buffer.append", + "session.input_audio_buffer.append", + "session.close", + ] + assert b"".join(base64.b64decode(message["audio"]) for message in messages[:-1]) == pcm + assert finished.is_set() diff --git a/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py b/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py new file mode 100644 index 00000000000..5296ea3b15a --- /dev/null +++ b/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py @@ -0,0 +1,319 @@ +import datetime +import io +import math +import wave +from collections.abc import AsyncIterator, Iterator +from typing import Final + +import anyio +import anyio.lowlevel +import httpx +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +import litellm +from litellm import Router +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, +) +from litellm.types.utils import TranscriptionResponse + +_STARTED_AT: Final = datetime.datetime(2026, 10, 28, 12, 0, 0) # noqa: DTZ001 # callback timestamps use the legacy naive contract + + +class _FirstDeltaThenStall(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + await anyio.sleep_forever() + + +class _FirstDeltaThenFails(httpx.AsyncByteStream): + def __init__(self, failure: httpx.TransportError) -> None: + self._failure = failure + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise self._failure + + +def _transcription_router(failing_api_base: str, healthy_api_base: str) -> Router: + return Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": api_base, + "order": order, + }, + } + for order, api_base in ((1, failing_api_base), (2, healthy_api_base)) + ], + allowed_fails=0, + cooldown_time=60, + ) + + +class _RecordingLogging: + def __init__(self) -> None: + self.logged: tuple[TranscriptionResponse, ...] = () + + def success_handler( + self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime + ) -> None: + return None + + async def async_success_handler( + self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime + ) -> None: + await anyio.lowlevel.checkpoint() + self.logged = (*self.logged, result) + + def handle_sync_success_callbacks_for_async_calls( + self, result: TranscriptionResponse, start_time: datetime.datetime, end_time: datetime.datetime + ) -> None: + return None + + def failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + *, + for_async_request: bool = False, + ) -> None: + return None + + async def async_failure_handler( + self, + exception: Exception, + traceback_exception: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + return None + + +@pytest.mark.asyncio +async def test_a_stream_the_client_drops_mid_transcript_is_still_logged_as_a_success() -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenStall() + ) + ) + ), + ) + stream: Final = await upstream.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.wav", b"audio"), stream=True + ) + logging_obj: Final = _RecordingLogging() + wrapped: Final = wrap_transcription_stream(stream, logging_obj, _STARTED_AT, litellm_exception_mapper({})) + + with anyio.CancelScope() as client_connection: + async for _ in wrapped: + client_connection.cancel() + + assert [result.text for result in logging_obj.logged] == ["Good"] + + +@pytest.mark.asyncio +async def test_a_deployment_whose_stream_times_out_mid_transcript_goes_into_cooldown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = _transcription_router("https://stalls.example/v1", "https://healthy.example/v1") + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://stalls.example/v1/audio/transcriptions").mock( + side_effect=lambda request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadTimeout("read timed out")), + ) + ) + upstream.post("https://healthy.example/v1/audio/transcriptions").mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=b'data: {"type":"transcript.text.done","text":"Good morning"}\n\n', + ) + ) + stalled: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True) + with pytest.raises(litellm.Timeout): + async for _ in stalled: + pass + await anyio.sleep(0.1) + retried: Final = await router.atranscription(model="transcriber", file=("sample.wav", b"audio"), stream=True) + events: Final = [event async for event in retried] + + assert [getattr(event, "text", None) for event in events] == ["Good morning"] + + +@pytest.mark.asyncio +async def test_a_stream_the_upstream_drops_mid_transcript_raises_a_litellm_error_to_the_caller() -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.RemoteProtocolError("peer closed connection")), + ) + ) + ), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError, match="peer closed connection"): + async for _ in stream: + pass + + +class _FailureHookCounter(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[str, ...] = () + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.failures = (*self.failures, "sync_failure") + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures = (*self.failures, "async_failure") + + +@pytest.mark.asyncio +async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + counter: Final = _FailureHookCounter() + monkeypatch.setattr(litellm, "callbacks", [counter]) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")), + ) + ) + ), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", b"audio"), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert counter.failures == ("async_failure",) + + +class _FirstDeltaThenResetSync(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise httpx.ReadError("connection reset") + + +class _FailureCostRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.costs: tuple[object, ...] = () + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None) + + +def _silent_wav(seconds: int) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16_000) + wav.writeframes(b"\x00\x00" * 16_000 * seconds) + return buffer.getvalue() + + +def _reset_mid_transcript(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")), + ) + + +def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync()) + + +def _per_second_rate(model: str) -> float: + rate: Final = litellm.model_cost[model]["input_cost_per_second"] + assert isinstance(rate, float) + return rate + + +@pytest.mark.asyncio +async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _FailureCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert len(recorder.costs) == 1 + assert isinstance(recorder.costs[0], float) + assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe")) + + +def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None: + recorder: Final = _FailureCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + upstream: Final = OpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)), + ) + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + for _ in stream: + pass + + assert len(recorder.costs) == 1 + assert isinstance(recorder.costs[0], float) + assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe")) diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 21b527250c4..0a08142c66f 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -29,6 +29,7 @@ from litellm.llms.gemini.image_generation.cost_calculator import ( from litellm.llms.vertex_ai.image_generation.cost_calculator import ( cost_calculator as vertex_image_generation_cost_calculator, ) +from litellm.types.llms.base import CachedTokensDetails from litellm.types.utils import ( CacheCreationTokenDetails, CompletionTokensDetailsWrapper, @@ -42,6 +43,62 @@ from litellm.types.utils import ( ) +def test_realtime_cached_modality_breakdown_matches_prompt_cost(_local_model_cost_map): + model: Final = "gpt-realtime-2.1-mini" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=400, + audio_tokens=400, + image_tokens=200, + cached_tokens=300, + cached_tokens_details=CachedTokensDetails(text_tokens=100, audio_tokens=150, image_tokens=50), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + breakdown: Final = get_token_type_cost_breakdown(model=model, custom_llm_provider="openai", usage=usage) + cached_cost: Final = ( + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + ) + uncached_cost: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + ) + + assert breakdown.cache_read_cost == pytest.approx(cached_cost) + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + +def test_image_model_bills_cached_image_tokens_at_its_cached_image_rate(_local_model_cost_map): + model: Final = "gpt-image-2" + rates: Final = litellm.model_cost[model] + usage: Final = Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=200, + image_tokens=800, + cached_tokens=600, + cached_tokens_details=CachedTokensDetails(text_tokens=100, image_tokens=500), + ), + ) + + prompt_cost, _ = generic_cost_per_token(model=model, usage=usage, custom_llm_provider="openai") + cached_cost: Final = 100 * rates["cache_read_input_token_cost"] + 500 * rates["cache_read_input_image_token_cost"] + uncached_cost: Final = 100 * rates["input_cost_per_token"] + 300 * rates["input_cost_per_image_token"] + + assert rates["cache_read_input_image_token_cost"] != rates["cache_read_input_token_cost"] + assert prompt_cost == pytest.approx(uncached_cost + cached_cost) + + @pytest.fixture def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") @@ -2234,6 +2291,10 @@ def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): prompt_tokens_details: PromptTokensDetailsResult = { "cache_hit_tokens": 0, "cache_hit_audio_tokens": 0, + "cached_text_tokens": 0, + "cached_audio_tokens": 0, + "cached_image_tokens": 0, + "has_cached_tokens_details": False, "cache_creation_tokens": 0, "cache_creation_token_details": CacheCreationTokenDetails( ephemeral_5m_input_tokens=100, diff --git a/tests/unit/litellm_core_utils/test_model_param_helper.py b/tests/unit/litellm_core_utils/test_model_param_helper.py index 6293c748505..df37139bc7b 100644 --- a/tests/unit/litellm_core_utils/test_model_param_helper.py +++ b/tests/unit/litellm_core_utils/test_model_param_helper.py @@ -1,5 +1,9 @@ +from collections.abc import Sequence +from typing import Final +import pytest +from litellm.caching.caching import Cache from litellm.litellm_core_utils.model_param_helper import ModelParamHelper @@ -46,3 +50,18 @@ def test_langfuse_model_parameters_no_secret_leakage(): assert "secret_fields" not in sanitized assert "authorization" not in sanitized assert "headers" not in sanitized + + +@pytest.mark.parametrize( + "hint,first_value,second_value", + [("languages", ("en",), ("fr",)), ("keywords", ("LiteLLM",), ("Kubernetes",))], +) +def test_transcription_cache_key_separates_recognition_hints( + hint: str, first_value: Sequence[str], second_value: Sequence[str] +) -> None: + cache: Final = Cache() + request: Final = {"model": "gpt-4o-transcribe", "file": "meeting.wav", "response_format": "json"} + + assert cache.get_cache_key(**request, **{hint: first_value}) != cache.get_cache_key( + **request, **{hint: second_value} + ) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index ba76138a268..eec1dfcf459 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -30,6 +30,8 @@ from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTransc from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter + +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.realtime import RealtimeResponseTransformInput, RealtimeResponseTypedDict @@ -512,6 +514,76 @@ async def test_backend_to_client_beta_receives_normalized_events(): assert sent["output_index"] == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ("session.created", "session.updated")) +async def test_beta_client_bridged_to_a_ga_backend_receives_the_beta_session_shape(event_type: str) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + json.dumps( + { + "type": event_type, + "event_id": "e1", + "session": { + "type": "realtime", + "object": "realtime.session", + "id": "sess_1", + "model": "gpt-realtime-2.1-mini", + "instructions": "Answer briefly", + "output_modalities": ["audio"], + "max_output_tokens": "inf", + "tools": [], + "audio": { + "input": { + "format": {"type": "audio/pcmu"}, + "transcription": {"model": "gpt-transcribe"}, + "noise_reduction": None, + "turn_detection": {"type": "server_vad", "create_response": True}, + }, + "output": {"format": {"type": "audio/pcm", "rate": 24000}, "voice": "marin", "speed": 1.1}, + }, + }, + } + ).encode(), + ConnectionClosed(None, None), + ] + ) + logging_obj: Final = _mock_logging() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) + + await streaming.backend_to_client_send_messages() + + sent: Final = json.loads(client_ws.send_text.call_args_list[0].args[0]) + assert sent["type"] == event_type + assert sent["session"] == { + "object": "realtime.session", + "id": "sess_1", + "model": "gpt-realtime-2.1-mini", + "instructions": "Answer briefly", + "modalities": ["audio", "text"], + "max_response_output_tokens": "inf", + "tools": [], + "input_audio_format": "g711_ulaw", + "input_audio_transcription": {"model": "gpt-transcribe"}, + "input_audio_noise_reduction": None, + "turn_detection": {"type": "server_vad", "create_response": True}, + "output_audio_format": "pcm16", + "voice": "marin", + "speed": 1.1, + }, ( + "Beta and GA session fields, checked 2026-10-20: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/session.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_session_create_response.py" + ) + + @pytest.mark.asyncio async def test_backend_to_client_stores_normalized_events_for_logging(): client_ws = MagicMock() @@ -648,6 +720,50 @@ async def test_client_ack_messages_keeps_beta_session_shape_for_beta_backend(): assert "audio" not in session +@pytest.mark.asyncio +@pytest.mark.parametrize("beta_client", (False, True)) +async def test_translation_session_update_omits_session_type(beta_client: bool) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "input_audio_format": "g711_ulaw", + "output_audio_format": "pcm16", + "audio": {"output": {"language": "fr"}}, + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + translation_session=True, + ) + + await streaming.client_ack_messages() + + sent_to_backend = json.loads(backend_ws.send.call_args_list[0].args[0]) + assert "type" not in sent_to_backend["session"] + assert sent_to_backend["session"]["audio"]["output"]["language"] == "fr" + assert sent_to_backend["session"]["audio"]["input"]["format"]["type"] == "audio/pcmu", ( + "OpenAI Realtime audio formats, checked 2026-10-06: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_audio_formats.py" + ) + assert sent_to_backend["session"]["audio"]["output"]["format"]["type"] == "audio/pcm" + + def test_translate_event_to_beta_renames_delta_types(): ev = RealTimeStreaming._translate_event_to_beta( {"type": "response.output_audio.delta", "delta": "abc", "event_id": "e1"} @@ -1158,6 +1274,153 @@ async def test_transcription_session_update_enforces_authorized_nested_model(): assert streaming._is_transcription_session is True +@pytest.mark.asyncio +async def test_translation_session_update_rejects_disallowed_nested_transcription_model() -> None: + client_ws: Final = MagicMock(send_text=AsyncMock(), close=AsyncMock()) + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + translation_session=True, + ) + + sent: Final = await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "type": "translation", + "audio": { + "input": { + "transcription": {"model": "gpt-live-transcribe"}, + } + }, + }, + } + ) + ) + assert sent is False + backend_ws.send.assert_not_awaited() + assert streaming._is_transcription_session is False + error: Final = json.loads(client_ws.send_text.await_args.args[0]) + assert error["type"] == "error" + assert error["error"]["type"] == "permission_error" + assert "gpt-live-transcribe" in error["error"]["message"] + assert client_ws.close.await_args.kwargs["code"] == 1008 + + +def _translation_transcription_update(transcription: Mapping[str, object]) -> str: + return json.dumps( + { + "type": "session.update", + "session": {"type": "translation", "audio": {"input": {"transcription": transcription}}}, + } + ) + + +@pytest.mark.asyncio +async def test_translation_session_update_authorizes_each_new_nested_transcription_model() -> None: + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth( + models=["gpt-realtime-translate", "gpt-realtime-whisper", "gpt-live-transcribe"] + ), + translation_session=True, + ) + + for transcription in ( + {"model": "gpt-realtime-whisper", "language": "en"}, + {"model": "gpt-live-transcribe", "language": "fr"}, + {"language": "de"}, + ): + await streaming._send_to_backend(_translation_transcription_update(transcription)) + + assert [ + json.loads(call.args[0])["session"]["audio"]["input"]["transcription"] + for call in backend_ws.send.await_args_list + ] == [ + {"model": "gpt-realtime-whisper", "language": "en"}, + {"model": "gpt-live-transcribe", "language": "fr"}, + {"model": "gpt-live-transcribe", "language": "de"}, + ] + + +@pytest.mark.asyncio +async def test_translation_session_update_refuses_a_later_nested_transcription_model_the_key_cannot_call() -> None: + client_ws: Final = MagicMock(send_text=AsyncMock(), close=AsyncMock()) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + translation_session=True, + ) + + first_sent: Final = await streaming._send_to_backend( + _translation_transcription_update({"model": "gpt-realtime-whisper", "language": "en"}) + ) + second_sent: Final = await streaming._send_to_backend( + _translation_transcription_update({"model": "gpt-live-transcribe", "language": "fr"}) + ) + + assert (first_sent, second_sent) == (True, False) + assert backend_ws.send.await_count == 1 + error: Final = json.loads(client_ws.send_text.await_args.args[0]) + assert error["error"]["type"] == "permission_error" + assert "gpt-live-transcribe" in error["error"]["message"] + assert client_ws.close.await_args.kwargs["code"] == 1008 + + +@pytest.mark.asyncio +async def test_translation_session_update_sends_the_deployment_model_for_a_nested_transcription_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + + model_list: Final = [ + { + "model_name": "whisper-alias", + "litellm_params": { + "model": "azure/qa-whisper-deployment", + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + MagicMock(), + backend_ws, + MagicMock(), + model="gpt-realtime-translate", + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "whisper-alias"]), + translation_session=True, + ) + + await streaming._send_to_backend(_translation_transcription_update({"model": "whisper-alias", "language": "en"})) + await streaming._send_to_backend(_translation_transcription_update({"language": "fr"})) + + assert [ + json.loads(call.args[0])["session"]["audio"]["input"]["transcription"] + for call in backend_ws.send.await_args_list + ] == [ + {"model": "qa-whisper-deployment", "language": "en"}, + {"model": "qa-whisper-deployment", "language": "fr"}, + ] + + @pytest.mark.asyncio async def test_normal_realtime_session_keeps_nested_transcription_model(): backend_ws = MagicMock() @@ -1847,8 +2110,7 @@ async def test_realtime_function_call_output_guardrail_allows_clean_output(monke @pytest.mark.asyncio async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pytest.MonkeyPatch): """ - Test that _has_realtime_guardrails returns True for a guardrail configured with - pre_call mode (not just realtime_input_transcription). + A pre_call guardrail gates typed user messages and tool output, never audio VAD. """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -1870,10 +2132,6 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pyt logging_obj: Final = _mock_logging() streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) - assert streaming._has_realtime_guardrails() is True, ( - "pre_call guardrail should be recognized as a realtime guardrail" - ) - # pre_call-only guardrails gate typed user messages / tool output, not audio VAD. assert streaming._has_audio_transcription_guardrails() is False, ( "pre_call-only guardrail must not disable server_vad auto-response" ) @@ -2024,7 +2282,6 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monke }, ) - assert streaming._has_realtime_guardrails() is True assert streaming._has_audio_transcription_guardrails() is False @@ -3561,6 +3818,444 @@ def test_store_message_skips_pydantic_for_unlogged_audio_delta(): assert streaming.messages == [] +@pytest.mark.parametrize( + "event_type", ["session.output_audio.delta", "response.output_audio.delta", "response.audio.delta"] +) +def test_translation_audio_duration_is_finalized_once(event_type: str): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + payload = base64.b64encode(bytes(48000)).decode() + streaming._capture_translation_output_audio({"type": event_type, "delta": payload}) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("output_bytes", (0, 48000)) +@pytest.mark.parametrize("format_event_type", ("session.created", "session.updated")) +@pytest.mark.parametrize( + "audio_format,bytes_per_second", + ( + ("pcm16", 48000), + ("g711_ulaw", 8000), + ("g711_alaw", 8000), + ({"type": "audio/pcm", "rate": 24000}, 48000), + ({"type": "audio/pcmu", "rate": 8000}, 8000), + ({"type": "audio/pcma", "rate": 8000}, 8000), + ({"type": "audio/pcmu"}, 8000), + ({"type": "audio/pcma"}, 8000), + ), +) +async def test_translation_disconnect_bills_sent_input_audio( + output_bytes: int, format_event_type: str, audio_format: str | Mapping[str, object], bytes_per_second: int +) -> None: + import base64 + + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), + backend_ws=backend, + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + format_event: Final = { + "type": format_event_type, + "session": {"audio": {"input": {"format": audio_format}}}, + } + streaming._capture_translation_output_audio(format_event) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.input_audio_buffer.append", + "audio": base64.b64encode(bytes(bytes_per_second * 2)).decode(), + } + ) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_bytes)).decode()} + ) + streaming._finalize_translation_usage() + streaming._finalize_translation_usage() + + assert streaming.messages == [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": output_bytes / 48000}, + } + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("rate", (24000, 1, 0, -1, float("nan"), float("inf"))) +async def test_translation_client_format_updates_do_not_change_billed_duration(rate: int | float) -> None: + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"input": {"format": "g711_ulaw"}, "output": {"format": "g711_ulaw"}}}, + } + ) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(8000)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(8000)).decode()} + await streaming._send_to_backend(input_event) + streaming._capture_translation_output_audio(output_event) + await streaming._send_to_backend( + json.dumps( + { + "type": "session.update", + "session": { + "audio": { + "input": {"format": {"type": "audio/pcm", "rate": rate}}, + "output": {"format": {"type": "audio/pcm", "rate": rate}}, + } + }, + } + ) + ) + await streaming._send_to_backend(input_event) + streaming._capture_translation_output_audio(output_event) + streaming._finalize_translation_usage() + + assert streaming.messages == [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 2.0}} + ] + + +@pytest.mark.asyncio +async def test_translation_confirmed_format_changes_preserve_previous_audio_duration() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(48000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + { + "type": "session.updated", + "session": {"audio": {"input": {"format": "g711_alaw"}, "output": {"format": "g711_alaw"}}}, + } + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(8000)).decode()}) + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(8000)).decode()} + ) + streaming._finalize_translation_usage() + + assert streaming.messages == [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 2.0}} + ] + + +@pytest.mark.parametrize("rate", (1, 0, -1, float("nan"), float("inf"), 10**400)) +def test_translation_invalid_provider_format_preserves_audio_duration(rate: int | float) -> None: + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=MagicMock(), logging_obj=_mock_logging(), translation_session=True + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + streaming._capture_translation_output_audio(output_event) + streaming._capture_translation_output_audio( + {"type": "session.updated", "session": {"audio": {"output": {"format": {"type": "audio/pcm", "rate": rate}}}}} + ) + streaming._capture_translation_output_audio(output_event) + streaming._finalize_translation_usage() + + assert streaming.messages == [{"type": "session.closed", "usage": {"type": "duration", "output_seconds": 2.0}}] + + +@pytest.mark.asyncio +async def test_translation_failed_audio_send_is_not_billed() -> None: + backend: Final = MagicMock() + backend.send = AsyncMock(side_effect=RuntimeError("send failed")) + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + + with pytest.raises(RuntimeError, match="send failed"): + await streaming._send_to_backend(json.dumps({"type": "session.input_audio_buffer.append", "audio": "AAAA"})) + streaming._finalize_translation_usage() + + assert streaming.messages == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("default_on,requested", ((False, False), (True, False), (False, True))) +@pytest.mark.parametrize( + "event_hook", + (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call), +) +async def test_translation_rejects_applicable_guardrails_before_forwarding_audio( + monkeypatch: pytest.MonkeyPatch, default_on: bool, requested: bool, event_hook: GuardrailEventHooks +) -> None: + guardrail: Final = CustomGuardrail(guardrail_name="speech-policy", event_hook=event_hook, default_on=default_on) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + input_event: Final = json.dumps( + {"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(4800)).decode()} + ) + output_event: Final = {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(4800)).decode()} + client: Final = _ga_client_ws() + client.receive_text = AsyncMock(side_effect=[input_event, ConnectionClosed(None, None)]) + client.send_text = AsyncMock() + client.close = AsyncMock() + backend: Final = MagicMock() + backend.send = AsyncMock() + backend.recv = AsyncMock(side_effect=[json.dumps(output_event), ConnectionClosed(None, None)]) + streaming: Final = RealTimeStreaming( + websocket=client, + backend_ws=backend, + logging_obj=_mock_logging(), + translation_session=True, + request_data={"litellm_metadata": {"guardrails": ["speech-policy"] if requested else []}}, + ) + + await streaming.bidirectional_forward() + + messages: Final = tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) + if default_on or requested: + assert messages[0]["error"]["type"] == "guardrail_error" + client.close.assert_awaited_once_with( + code=1008, reason="Translation sessions cannot enforce configured realtime guardrails" + ) + backend.send.assert_not_awaited() + else: + assert output_event in messages + backend.send.assert_awaited_once_with(input_event) + + +@pytest.mark.asyncio +async def test_translation_for_a_key_opted_out_of_a_legacy_callback_guardrail_gets_no_realtime_session_gate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", [CustomGuardrail(guardrail_name="legacy-callback")]) + session_update: Final = json.dumps({"type": "session.update", "session": {"audio": {"output": {"language": "es"}}}}) + session_created: Final = {"type": "session.created", "session": {"type": "translation"}} + client: Final = _ga_client_ws() + client.receive_text = AsyncMock(side_effect=[session_update, ConnectionClosed(None, None)]) + client.close = AsyncMock() + backend: Final = MagicMock() + backend.send = AsyncMock() + backend.recv = AsyncMock(side_effect=[json.dumps(session_created), ConnectionClosed(None, None)]) + streaming: Final = RealTimeStreaming( + websocket=client, + backend_ws=backend, + logging_obj=_mock_logging(), + translation_session=True, + request_data={"litellm_metadata": {"user_api_key_metadata": {"disable_global_guardrails": True}}}, + ) + + await streaming.bidirectional_forward() + + assert tuple(call.args[0] for call in backend.send.await_args_list) == (session_update,) + assert session_created in tuple(json.loads(call.args[0]) for call in client.send_text.await_args_list) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retain_close", (False, True)) +@pytest.mark.parametrize("reported_input,expected_input", [(None, 2.0), (0.0, 0.0), (0.25, 0.25)]) +async def test_translation_terminal_usage_fills_only_missing_input_duration( + monkeypatch: pytest.MonkeyPatch, retain_close: bool, reported_input: float | None, expected_input: float +) -> None: + import base64 + + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*" if retain_close else None) + backend: Final = MagicMock() + backend.send = AsyncMock() + streaming: Final = RealTimeStreaming( + websocket=_ga_client_ws(), backend_ws=backend, logging_obj=_mock_logging(), translation_session=True + ) + await streaming._send_to_backend( + json.dumps({"type": "session.input_audio_buffer.append", "audio": base64.b64encode(bytes(96000)).decode()}) + ) + close_event: Final = { + "type": "session.closed", + "usage": { + "type": "duration", + "output_seconds": 0.5, + **({"input_seconds": reported_input} if reported_input is not None else {}), + }, + } + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert sum(item.get("input_seconds") or 0.0 for item in usage) == expected_input + assert sum(item.get("output_seconds") or 0.0 for item in usage) == 0.5 + + +def test_translation_audio_duration_uses_session_output_format(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + { + "type": "session.created", + "session": {"audio": {"output": {"format": {"type": "audio/pcmu", "rate": 8000}}}}, + } + ) + streaming._capture_translation_output_audio( + { + "type": "session.output_audio.delta", + "delta": base64.b64encode(bytes(8000)).decode(), + } + ) + streaming._finalize_translation_usage() + + assert streaming.messages[-1]["usage"] == {"type": "duration", "output_seconds": 1.0} + + +def test_translation_does_not_duplicate_provider_duration_usage(): + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming.messages.append({"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + + +def test_translation_prefers_provider_duration_over_audio_byte_estimate(): + import base64 + + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5} + + +@pytest.mark.parametrize( + ("output_audio_bytes", "expected_usage"), + [ + (0, {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.0}), + (48000, {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0}), + ], +) +def test_translation_preserves_input_only_provider_usage( + output_audio_bytes: int, expected_usage: Mapping[str, str | float] +) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(output_audio_bytes)).decode()} + ) + streaming._capture_translation_output_audio( + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + ) + streaming._finalize_translation_usage() + + closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"] + assert len(closed_events) == 1 + assert closed_events[0]["usage"] == expected_usage + + +def test_translation_retained_input_only_close_event_bills_captured_output(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + close_event: Final = {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25}} + + streaming._capture_translation_output_audio(close_event) + streaming.store_message(close_event) + streaming._finalize_translation_usage() + + usage_events: Final = tuple(event["usage"] for event in streaming.messages if event.get("type") == "session.closed") + assert len(usage_events) == 2 + assert sum(usage.get("input_seconds", 0.0) for usage in usage_events) == 0.25 + assert sum(usage.get("output_seconds", 0.0) for usage in usage_events) == 1.0 + + +@pytest.mark.parametrize("malformed_output", [-1.0, float("nan"), float("inf"), 10**1000]) +def test_translation_malformed_provider_duration_uses_captured_audio(malformed_output: float | int) -> None: + streaming = RealTimeStreaming( + websocket=MagicMock(), + backend_ws=MagicMock(), + logging_obj=_mock_logging(), + model="gpt-realtime-translate", + translation_session=True, + ) + streaming._capture_translation_output_audio( + {"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()} + ) + + streaming._capture_translation_output_audio( + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": malformed_output}, + } + ) + streaming._finalize_translation_usage() + + close_events: Final = tuple(event for event in streaming.messages if event.get("type") == "session.closed") + assert len(close_events) == 1 + assert close_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 1.0} + + @pytest.mark.asyncio async def test_audio_delta_frame_parsed_at_most_once(): client_ws = _beta_client_ws() @@ -4200,6 +4895,48 @@ async def test_upstream_close_after_relayed_events_still_logs_the_session_as_suc client_ws.close.assert_awaited_once_with(code=1008, reason=_UPSTREAM_REFUSAL) +def _model_not_found_error_event() -> bytes: + return json.dumps( + { + "type": "error", + "error": { + "type": "invalid_request_error", + "code": "model_not_found", + "message": "The model `gpt-realtime-translate-typo` does not exist", + }, + } + ).encode() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("close_code", (1000, 1008)) +async def test_upstream_error_before_any_session_logs_a_failure_not_a_success(close_code: int): + upstream_close: Final = ConnectionClosed(Close(close_code, ""), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), _backend_ws_closing_with(_model_not_found_error_event(), upstream_close) + ) + + await session.run() + + assert session.logging.logged_failures == (upstream_close,) + assert session.logging.logged_sessions == () + + +@pytest.mark.asyncio +async def test_upstream_error_after_session_created_still_logs_the_session_as_success(): + session_created: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + upstream_close: Final = ConnectionClosed(Close(1000, ""), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), + _backend_ws_closing_with(session_created, _model_not_found_error_event(), upstream_close), + ) + + await session.run() + + assert len(session.logging.logged_sessions) == 1 + assert session.logging.logged_failures == () + + @pytest.mark.asyncio async def test_upstream_closing_while_a_client_message_is_forwarded_still_reaches_the_client(): upstream_close: Final = ConnectionClosed(Close(1008, _UPSTREAM_REFUSAL), None) @@ -5029,3 +5766,264 @@ async def test_blocked_transcript_is_removed_from_the_conversation_before_the_re sent: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] assert [event["type"] for event in sent] == expected_frames assert all(event["item_id"] == "item_blocked" for event in sent if event["type"] == "conversation.item.delete") + + +@pytest.mark.asyncio +async def test_beta_client_bridged_to_a_ga_backend_sends_assistant_text_items_as_ga_output_text() -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "The forecast is sunny"}], + }, + } + ), + json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Thanks"}]}, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj: Final = _mock_logging() + logging_obj.pre_call = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) + + await streaming.client_ack_messages() + + sent_items: Final = [json.loads(call.args[0])["item"] for call in backend_ws.send.call_args_list] + assert [(item["role"], item["content"]) for item in sent_items] == [ + ("assistant", [{"type": "output_text", "text": "The forecast is sunny"}]), + ("user", [{"type": "input_text", "text": "Thanks"}]), + ], ( + "Beta and GA item content types, checked 2026-10-28: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/conversation_item_content.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_conversation_item_assistant_message.py" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_type", ("response.created", "response.done")) +async def test_beta_client_bridged_to_a_ga_backend_receives_the_beta_response_shape(event_type: str) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.send_text = AsyncMock() + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + json.dumps( + { + "type": event_type, + "event_id": "e1", + "response": { + "object": "realtime.response", + "id": "resp_1", + "status": "completed", + "conversation_id": "conv_1", + "output_modalities": ["audio"], + "max_output_tokens": "inf", + "audio": {"output": {"format": {"type": "audio/pcmu"}, "voice": "marin"}}, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_audio", "transcript": "Sunny"}], + } + ], + }, + } + ).encode(), + ConnectionClosed(None, None), + ] + ) + logging_obj: Final = _mock_logging() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging_obj, backend_uses_beta_protocol=False, bridges_beta_client_to_ga=True + ) + + await streaming.backend_to_client_send_messages() + + sent: Final = json.loads(client_ws.send_text.call_args_list[0].args[0]) + assert sent["type"] == event_type + assert sent["response"] == { + "object": "realtime.response", + "id": "resp_1", + "status": "completed", + "conversation_id": "conv_1", + "modalities": ["audio", "text"], + "max_output_tokens": "inf", + "output_audio_format": "g711_ulaw", + "voice": "marin", + "output": [{"type": "message", "role": "assistant", "content": [{"type": "audio", "transcript": "Sunny"}]}], + }, ( + "Beta and GA response fields, checked 2026-10-28: " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/beta/realtime/realtime_response.py and " + "https://github.com/openai/openai-python/blob/v2.33.0/src/openai/types/realtime/realtime_response.py" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("drop_params", (False, True)) +async def test_beta_client_bridged_to_a_ga_backend_sends_ga_session_and_response_fields(drop_params: bool) -> None: + client_ws = MagicMock() + client_ws.scope = {"headers": [(b"openai-beta", b"realtime=v1")]} + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": { + "instructions": "Answer briefly", + "temperature": 0.7, + "max_response_output_tokens": 300, + "speed": 1.1, + "input_audio_noise_reduction": {"type": "near_field"}, + }, + } + ), + json.dumps( + { + "type": "response.create", + "response": { + "modalities": ["text", "audio"], + "voice": "marin", + "output_audio_format": "g711_ulaw", + "temperature": 0.7, + "max_response_output_tokens": 200, + "instructions": "Say hi", + }, + } + ), + Exception("connection closed"), + ] + ) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + logging_obj: Final = _mock_logging() + logging_obj.pre_call = MagicMock() + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + logging_obj, + backend_uses_beta_protocol=False, + bridges_beta_client_to_ga=True, + drop_params=drop_params, + ) + + await streaming.client_ack_messages() + + sent: Final = { + event["type"]: event for event in (json.loads(call.args[0]) for call in backend_ws.send.call_args_list) + } + temperature: Final = {} if drop_params else {"temperature": 0.7} + assert sent["session.update"]["session"] == { + "type": "realtime", + "instructions": "Answer briefly", + "max_output_tokens": 300, + "audio": {"input": {"noise_reduction": {"type": "near_field"}}, "output": {"speed": 1.1}}, + **temperature, + } + assert sent["response.create"]["response"] == { + "output_modalities": ["audio"], + "audio": {"output": {"voice": "marin", "format": {"type": "audio/pcmu"}}}, + "max_output_tokens": 200, + "instructions": "Say hi", + **temperature, + } + + +@pytest.mark.asyncio +async def test_gemini_live_session_update_keeps_the_client_output_token_cap() -> None: + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = _ga_client_ws() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps( + { + "type": "session.update", + "session": {"temperature": 0.4, "max_response_output_tokens": 77, "modalities": ["text"]}, + } + ), + _TEXT_ITEM_CREATE, + _RESPONSE_CREATE, + Exception("connection closed"), + ] + ) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + _mock_logging(), + provider_config=GeminiRealtimeConfig(), + model="gemini-2.0-flash-live-001", + ) + + await streaming.client_ack_messages() + + setup_frame, content_frame = (json.loads(call.args[0]) for call in backend_ws.send.call_args_list) + assert "clientContent" in content_frame + assert setup_frame["setup"]["generationConfig"] == { + "temperature": 0.4, + "maxOutputTokens": 77, + "responseModalities": ["TEXT"], + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "event_normalizer", + (pytest.param(XAIRealtimeNormalizer(), id="xai"), pytest.param(None, id="openai-ga-client")), +) +async def test_unbridged_backend_receives_session_update_and_response_create_as_the_client_sent_them( + event_normalizer: XAIRealtimeNormalizer | None, +) -> None: + client_events: Final = ( + { + "type": "session.update", + "session": { + "type": "realtime", + "instructions": "Answer briefly", + "temperature": 0.7, + "max_response_output_tokens": 300, + "speed": 1.1, + "input_audio_noise_reduction": {"type": "near_field"}, + }, + }, + { + "type": "response.create", + "response": { + "modalities": ["text", "audio"], + "voice": "marin", + "output_audio_format": "g711_ulaw", + "temperature": 0.7, + "max_response_output_tokens": 200, + }, + }, + ) + client_ws: Final = _ga_client_ws() + client_ws.receive_text = AsyncMock( + side_effect=[*(json.dumps(event) for event in client_events), Exception("connection closed")] + ) + backend_ws: Final = MagicMock(send=AsyncMock()) + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, _mock_logging(), event_normalizer=event_normalizer, drop_params=True + ) + + await streaming.client_ack_messages() + + assert tuple(json.loads(call.args[0]) for call in backend_ws.send.call_args_list) == client_events diff --git a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py index f1db76c1e21..6745421b8f8 100644 --- a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py @@ -1,4 +1,7 @@ import os +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -259,6 +262,21 @@ async def test_construct_url_v1_protocol(): assert url.count("/realtime") == 1 +def test_construct_url_translation_protocol(): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + + url = AzureOpenAIRealtime()._construct_url( + api_base="https://my-endpoint.openai.azure.com", + model="translate-deployment", + api_version=None, + realtime_protocol="GA", + query_params={"model": "translate-deployment"}, + realtime_mode="translation", + ) + + assert url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime/translations?model=translate-deployment" + + @pytest.mark.asyncio @pytest.mark.parametrize("protocol", ["ga", "Ga", "gA", "V1", "v1", "GA"]) async def test_construct_url_case_insensitive_protocol(protocol): @@ -752,3 +770,32 @@ def test_construct_url_with_a_trailing_slash_api_base_matches_the_url_without_it ) assert url_for("https://gw.example.com/azure/") == url_for("https://gw.example.com/azure") + + +@dataclass(frozen=True, slots=True) +class _ClientWebSocketWithHeaders: + scope: Mapping[str, object] + + +@pytest.mark.parametrize("pinned_protocol", ("GA", "ga", "v1")) +@pytest.mark.parametrize( + ("client_headers", "bridges_beta_client_to_ga"), + ( + pytest.param(((b"openai-beta", b"realtime=v1"),), True, id="beta-client"), + pytest.param((), False, id="ga-client"), + ), +) +def test_a_deployment_pinned_to_ga_bridges_only_a_beta_client( + pinned_protocol: str, client_headers: tuple[tuple[bytes, bytes], ...], bridges_beta_client_to_ga: bool +) -> None: + from litellm.llms.azure.realtime.handler import AzureRealtimeProtocol, azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + pinned_protocol, + model="my-realtime-deployment", + realtime_mode="realtime", + query_params=None, + websocket=_ClientWebSocketWithHeaders(scope={"headers": client_headers}), + ) + + assert protocol == AzureRealtimeProtocol(name=pinned_protocol, bridges_beta_client_to_ga=bridges_beta_client_to_ga) diff --git a/tests/unit/llms/azure/realtime/test_http_transformation.py b/tests/unit/llms/azure/realtime/test_http_transformation.py index 714ce9558cf..dae8ed10d1d 100644 --- a/tests/unit/llms/azure/realtime/test_http_transformation.py +++ b/tests/unit/llms/azure/realtime/test_http_transformation.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final import httpx @@ -7,7 +8,11 @@ import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler _DATED_PREVIEW_API_VERSION: Final = "2025-04-01-preview" -_API_BASES: Final = ("https://my.openai.azure.com", "https://my.openai.azure.com/", "https://my.openai.azure.com/openai") +_API_BASES: Final = ( + "https://my.openai.azure.com", + "https://my.openai.azure.com/", + "https://my.openai.azure.com/openai", +) class _AzureThatRecordsTheRequest: @@ -55,3 +60,62 @@ async def test_azure_sdp_exchange_targets_the_ga_v1_surface_with_the_ephemeral_k assert str(request.url) == "https://my.openai.azure.com/openai/v1/realtime/calls" assert request.headers.get("authorization") == "Bearer ek_upstream" assert "api-key" not in request.headers + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "url"), + ( + ("https://gw.example.com/azure/v1", "https://gw.example.com/azure/v1/openai/realtime/transcription_sessions"), + ("https://my.openai.azure.com/openai", "https://my.openai.azure.com/openai/realtime/transcription_sessions"), + ("https://my.openai.azure.com/openai/v1", "https://my.openai.azure.com/openai/realtime/transcription_sessions"), + ), +) +async def test_azure_transcription_session_strips_only_a_trailing_openai_path(api_base: str, url: str) -> None: + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_transcription_session( + model="azure/gpt-realtime-whisper-deployment", + api_base=api_base, + api_key="azure-key", + api_version=_DATED_PREVIEW_API_VERSION, + client=azure.client(), + ) + (request,) = azure.requests + assert str(request.url) == f"{url}?api-version={_DATED_PREVIEW_API_VERSION}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("caller_headers", "sent_headers"), + ( + pytest.param( + {"Authorization": "Bearer caller-token"}, + {"authorization": "Bearer caller-token", "api-key": None}, + id="caller-authorization", + ), + pytest.param( + {"authorization": "Bearer caller-token"}, + {"authorization": "Bearer caller-token", "api-key": None}, + id="caller-lowercase-authorization", + ), + pytest.param({"api-key": "caller-key"}, {"authorization": None, "api-key": "caller-key"}, id="caller-api-key"), + pytest.param({}, {"authorization": "Bearer entra-token", "api-key": None}, id="no-caller-auth"), + ), +) +async def test_azure_entra_fallback_never_overrides_the_callers_own_auth_header( + caller_headers: Mapping[str, str], + sent_headers: Mapping[str, str | None], + monkeypatch: pytest.MonkeyPatch, + no_ambient_azure_credentials: None, +) -> None: + monkeypatch.delenv("AZURE_API_KEY", raising=False) + monkeypatch.setenv("AZURE_AD_TOKEN", "entra-token") + azure: Final = _AzureThatRecordsTheRequest() + await litellm.acreate_realtime_client_secret( + model="azure/gpt-realtime-deployment", + api_base="https://my.openai.azure.com", + extra_headers=caller_headers, + client=azure.client(), + ) + (request,) = azure.requests + assert {name: request.headers.get(name) for name in sent_headers} == sent_headers diff --git a/tests/unit/llms/azure/test_audio_transcriptions.py b/tests/unit/llms/azure/test_audio_transcriptions.py index c5d403b3c38..f86fae494ef 100644 --- a/tests/unit/llms/azure/test_audio_transcriptions.py +++ b/tests/unit/llms/azure/test_audio_transcriptions.py @@ -4,6 +4,7 @@ from typing import Final import httpx import pytest +import respx from openai import AsyncAzureOpenAI, AzureOpenAI import litellm @@ -160,3 +161,70 @@ async def test_azure_transcription_parses_response_formats( ) assert response.text == expected_text + + +@pytest.mark.parametrize("api_version", ["2024-06-01", "2025-03-01-preview", "v1", "latest", "preview"]) +async def test_azure_router_transcription_sends_the_default_api_version( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_version: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.delenv("AZURE_API_VERSION", raising=False) + route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": { + "model": "azure/gpt-transcribe", + "api_key": "test-key", + "api_base": "https://transcribe-route.openai.azure.com", + "api_version": api_version, + }, + } + ] + ) + + response: Final = await router.atranscription(model="transcriber", file=_audio_file()) + + sent_url: Final = route.calls.last.request.url + assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions" + assert sent_url.params.get("api-version") == litellm.AZURE_DEFAULT_API_VERSION + assert response.text == "hello" + + +@pytest.mark.parametrize( + ("api_version", "expected_api_version"), + [ + ("v1", litellm.AZURE_DEFAULT_API_VERSION), + ("latest", litellm.AZURE_DEFAULT_API_VERSION), + ("preview", litellm.AZURE_DEFAULT_API_VERSION), + ("2025-03-01-preview", "2025-03-01-preview"), + ], +) +def test_azure_sync_transcription_uses_the_deployments_route( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + api_version: str, + expected_api_version: str, +) -> None: + monkeypatch.delenv("AZURE_API_VERSION", raising=False) + route: Final = respx_mock.route(host="transcribe-route.openai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + + response: Final = litellm.transcription( + model="azure/gpt-transcribe", + file=_audio_file(), + api_key="test-key", + api_base="https://transcribe-route.openai.azure.com", + api_version=api_version, + ) + + sent_url: Final = route.calls.last.request.url + assert sent_url.path == "/openai/deployments/gpt-transcribe/audio/transcriptions" + assert sent_url.params.get("api-version") == expected_api_version + assert response.text == "hello" diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 3f9da8be2d4..0a41861e088 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -491,6 +491,11 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "avector_store_search", "acreate_skill", "acreate_interaction", + "acreate_realtime_client_secret", + "acreate_realtime_transcription_session", + "acreate_realtime_translation_client_secret", + "arealtime_calls", + "arealtime_translation_calls", ] ], ) diff --git a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py index b31db62589d..8ed65ef925c 100644 --- a/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py +++ b/tests/unit/llms/openai/realtime/test_openai_realtime_handler.py @@ -1,7 +1,11 @@ +import asyncio import json +from collections.abc import Mapping +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from pydantic import TypeAdapter from litellm.llms.custom_httpx.http_handler import get_shared_realtime_ssl_context from litellm.types.realtime import RealtimeQueryParams @@ -526,3 +530,83 @@ async def test_realtime_query_params_preserve_missing_model(monkeypatch): called_kwargs = mock_async_realtime.call_args.kwargs assert called_kwargs["query_params"] == {"intent": "transcription"} + + +class _BackendConnection: + def __init__(self, connection): + self.connection = connection + + async def __aenter__(self): + return self.connection + + async def __aexit__(self, exc_type, exc, tb): + return None + + +def test_translation_url_uses_dedicated_path(): + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + handler = OpenAIRealtime() + url = handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + assert url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("beta_client", (False, True)) +async def test_translation_websocket_uses_direct_transport(beta_client: bool) -> None: + from litellm.llms.openai.realtime.handler import OpenAIRealtime + + backend: Final = AsyncMock() + pending: Final = asyncio.Event() + + async def receive_backend() -> str: + await pending.wait() + return "" + + backend.recv.side_effect = receive_backend + websocket: Final = MagicMock() + websocket.scope = {"headers": [(b"openai-beta", b"realtime=v1")] if beta_client else []} + websocket.receive_text = AsyncMock( + side_effect=( + json.dumps( + { + "type": "session.update", + "session": {"type": "translation", "audio": {"output": {"language": "fr"}}}, + } + ), + RuntimeError("connection closed"), + ) + ) + websocket.close = AsyncMock() + logging_obj: Final = MagicMock(dispatch_success_handlers=AsyncMock(), dispatch_failure_handlers=AsyncMock()) + handler: Final = OpenAIRealtime() + expected_url: Final = "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert ( + handler._construct_url( + api_base="https://api.openai.com/v1", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + == expected_url + ) + + with patch("websockets.connect", return_value=_BackendConnection(backend)) as connect: + await handler.async_realtime( + model="gpt-realtime-translate", + websocket=websocket, + logging_obj=logging_obj, + api_base="https://api.openai.com/v1", + api_key="sk-test", + query_params={"model": "gpt-realtime-translate"}, + realtime_mode="translation", + ) + + connect.assert_called_once() + assert connect.call_args.args[0] == expected_url + assert "OpenAI-Beta" not in connect.call_args.kwargs["additional_headers"] + sent: Final = TypeAdapter(Mapping[str, object]).validate_json(backend.send.call_args_list[0].args[0]) + assert sent["session"] == {"audio": {"output": {"language": "fr"}}} diff --git a/tests/unit/llms/openai/realtime/test_transcription_sessions.py b/tests/unit/llms/openai/realtime/test_transcription_sessions.py index 3bb9a4ab1ab..f637271d667 100644 --- a/tests/unit/llms/openai/realtime/test_transcription_sessions.py +++ b/tests/unit/llms/openai/realtime/test_transcription_sessions.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest - +import litellm from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -21,9 +21,7 @@ from litellm.types.realtime import RealtimeTranscriptionSessionRequest def test_openai_transcription_session_url(): cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) @@ -32,24 +30,36 @@ def test_openai_transcription_session_url_strips_trailing_v1(): """A /v1 suffix must not be duplicated in the path.""" cfg = OpenAIRealtimeHTTPConfig() assert ( - cfg.get_transcription_session_url( - api_base="https://api.openai.com/v1", model="gpt-realtime-whisper" - ) + cfg.get_transcription_session_url(api_base="https://api.openai.com/v1", model="gpt-realtime-whisper") == "https://api.openai.com/v1/realtime/transcription_sessions" ) -def test_azure_transcription_session_url_uses_deployment_and_api_version(): +@pytest.mark.parametrize( + "api_base", + ["https://my.openai.azure.com", "https://my.openai.azure.com/openai", "https://my.openai.azure.com/openai/v1/"], +) +def test_azure_transcription_session_url_uses_deployment_and_api_version(api_base: str): cfg = AzureRealtimeHTTPConfig() url = cfg.get_transcription_session_url( - api_base="https://my.openai.azure.com", + api_base=api_base, model="whisper-deploy", api_version="2025-04-01-preview", ) - assert ( - url - == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" - ) + assert url == "https://my.openai.azure.com/openai/realtime/transcription_sessions?api-version=2025-04-01-preview" + + +@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"]) +@pytest.mark.parametrize("model", ["gpt-realtime-2.1", "gpt-realtime-2", "gpt-realtime-2-2026-05-06"]) +def test_azure_ga_realtime_http_urls(api_version, model: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + cfg = AzureRealtimeHTTPConfig() + base = "https://my.openai.azure.com" + + assert cfg.get_complete_url(base, model, api_version) == (f"{base}/openai/v1/realtime/client_secrets") + assert cfg.get_realtime_calls_url(base, model, api_version) == (f"{base}/openai/v1/realtime/calls") + litellm.get_model_info.cache_clear() def test_request_resolves_model_returns_none_when_both_absent(): @@ -141,6 +151,27 @@ async def test_client_secret_handler_still_targets_client_secrets_url(): assert kwargs["url"] == "https://api.openai.com/v1/realtime/client_secrets" +@pytest.mark.asyncio +async def test_azure_client_secret_routes_on_a_provider_qualified_session_model(): + mock_response = MagicMock(spec=httpx.Response) + mock_client = MagicMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=mock_response) + + result = await litellm.acreate_realtime_client_secret( + session={"type": "realtime", "model": "azure/gpt-realtime-2.1"}, + api_base="https://my.openai.azure.com", + api_key="azure-test-key", + api_version="v1", + client=mock_client, + ) + + assert result is mock_response + request = mock_client.post.call_args.kwargs + assert request["url"] == "https://my.openai.azure.com/openai/v1/realtime/client_secrets" + assert request["headers"]["api-key"] == "azure-test-key" + assert request["json"]["session"]["model"] == "gpt-realtime-2.1" + + @pytest.mark.asyncio async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): """ @@ -169,18 +200,14 @@ async def test_sdk_fn_routes_openai_transcription_session(monkeypatch): assert kwargs["url"].endswith("/v1/realtime/transcription_sessions") # The litellm-only routing hint must not be forwarded upstream. assert "model" not in kwargs["json"] - assert kwargs["json"]["input_audio_transcription"] == { - "model": "gpt-realtime-whisper" - } + assert kwargs["json"]["input_audio_transcription"] == {"model": "gpt-realtime-whisper"} def test_append_query_params_skips_existing_keys(): from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler url = "wss://example.com/v1/realtime?model=gpt-4o" - result = BaseLLMHTTPHandler._append_query_params( - url, {"model": "ignored", "intent": "transcription"} - ) + result = BaseLLMHTTPHandler._append_query_params(url, {"model": "ignored", "intent": "transcription"}) assert "model=ignored" not in result assert "intent=transcription" in result diff --git a/tests/unit/llms/openai/realtime/test_translation.py b/tests/unit/llms/openai/realtime/test_translation.py new file mode 100644 index 00000000000..525782704c1 --- /dev/null +++ b/tests/unit/llms/openai/realtime/test_translation.py @@ -0,0 +1,185 @@ +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest + +from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig +from litellm.types.realtime import RealtimeSessionConfig +from litellm.types.router import GenericLiteLLMParams + + +def test_azure_realtime_http_headers_use_ad_token_without_api_key(): + config = AzureRealtimeHTTPConfig() + params = GenericLiteLLMParams(azure_ad_token="test-ad-token") + + assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == { + "X-Trace": "trace", + "Authorization": "Bearer test-ad-token", + } + assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"} + + +def test_realtime_session_config_supports_translation_and_live_transcription_fields(): + session = RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio={ + "input": { + "transcription": { + "model": "gpt-live-transcribe", + "delay": "minimal", + "languages": ["en", "fr"], + "keywords": ["LiteLLM"], + } + }, + "output": {"language": "es"}, + }, + ) + + assert session.audio is not None + assert session.audio.input is not None + assert session.audio.input.transcription is not None + assert session.audio.input.transcription.delay == "minimal" + assert session.audio.input.transcription.languages == ["en", "fr"] + assert session.audio.input.transcription.keywords == ["LiteLLM"] + assert session.audio.output is not None + assert session.audio.output.language == "es" + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ( + "https://api.openai.com", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ( + "https://api.openai.com/v1", + "https://api.openai.com/v1/realtime/translations/client_secrets", + ), + ], +) +def test_openai_translation_client_secret_url(api_base: str, expected: str): + config = OpenAIRealtimeHTTPConfig() + assert config.get_translation_client_secret_url(api_base, "gpt-realtime-translate") == expected + + +def test_openai_translation_calls_url(): + config = OpenAIRealtimeHTTPConfig() + assert ( + config.get_translation_calls_url("https://api.openai.com/v1", "gpt-realtime-translate") + == "https://api.openai.com/v1/realtime/translations/calls" + ) + + +@pytest.mark.parametrize("suffix", ("", "/", "/openai", "/openai/v1/")) +def test_azure_translation_urls_use_ga_paths(suffix: str): + config: Final = AzureRealtimeHTTPConfig() + api_base: Final = f"https://example.openai.azure.com{suffix}" + assert ( + config.get_translation_client_secret_url(api_base, "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets" + ) + assert ( + config.get_translation_calls_url(api_base, "translate-deployment") + == "https://example.openai.azure.com/openai/v1/realtime/translations/calls" + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_uses_custom_translation_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/client_secrets"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + request_data = {"session": {"type": "translation", "model": "gpt-realtime-translate"}} + + response = await handler.async_realtime_translation_client_secret_handler( + api_base="https://api.openai.com", + api_key="sk-test", + request_data=request_data, + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + ) + + assert response.status_code == 200 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/client_secrets" + assert call["json"] == request_data + + +@pytest.mark.asyncio +async def test_azure_translation_client_secret_supports_entra_bearer_auth(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 200, + json={"value": "ek_test"}, + request=httpx.Request( + "POST", + "https://example.openai.azure.com/openai/v1/realtime/translations/client_secrets", + ), + ) + ) + handler = BaseLLMHTTPHandler() + + await handler.async_realtime_translation_client_secret_handler( + api_base="https://example.openai.azure.com", + api_key="", + request_data={"session": {"type": "translation", "model": "translate-deployment"}}, + logging_obj=MagicMock(), + timeout=10, + provider_config=AzureRealtimeHTTPConfig(), + model="translate-deployment", + extra_headers={"Authorization": "Bearer entra-token"}, + client=client, + ) + + headers = client.post.call_args.kwargs["headers"] + assert headers["Authorization"] == "Bearer entra-token" + assert "api-key" not in headers + + +@pytest.mark.asyncio +async def test_translation_calls_use_translation_session_and_path(): + client = MagicMock(spec=AsyncHTTPHandler) + client.post = AsyncMock( + return_value=httpx.Response( + 201, + content=b"v=0\r\n", + request=httpx.Request("POST", "https://api.openai.com/v1/realtime/translations/calls"), + ) + ) + logging_obj = MagicMock() + handler = BaseLLMHTTPHandler() + + response = await handler.async_realtime_calls_handler( + api_base="https://api.openai.com", + openai_ephemeral_key="ek_test", + sdp_body=b"v=0\r\n", + logging_obj=logging_obj, + timeout=10, + provider_config=OpenAIRealtimeHTTPConfig(), + model="gpt-realtime-translate", + client=client, + translation=True, + ) + + assert response.status_code == 201 + call = client.post.call_args.kwargs + assert call["url"] == "https://api.openai.com/v1/realtime/translations/calls" + assert call["headers"]["Content-Type"] == "application/sdp" + assert call["content"] == "v=0\r\n" diff --git a/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py new file mode 100644 index 00000000000..9bb5fb6d300 --- /dev/null +++ b/tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py @@ -0,0 +1,951 @@ +import asyncio +import io +import json +import queue +import wave +from collections.abc import Callable, Iterator +from datetime import datetime +from typing import Final, NamedTuple +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI, OpenAI + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.audio_utils.transcription_streaming import ( + litellm_exception_mapper, + wrap_transcription_stream, +) +from litellm.litellm_core_utils.redact_messages import REDACTED_BY_LITELLM +from litellm.llms.azure.audio_transcriptions import AzureAudioTranscription +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTTranscribeAudioTranscriptionConfig, +) +from litellm.llms.openai.transcriptions.handler import OpenAIAudioTranscription +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import TranscriptionResponse +from litellm.utils import get_optional_params_transcription + + +@pytest.fixture +def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def test_gpt_transcribe_config_uses_native_parameters_and_json(): + config = OpenAIGPTTranscribeAudioTranscriptionConfig() + supported = config.get_supported_openai_params("gpt-transcribe") + assert supported == ["prompt", "response_format", "keywords", "languages", "stream"] + + audio_file = io.BytesIO(b"audio") + request = config.transform_audio_transcription_request( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en", "fr"], "stream": True, "response_format": "json"}, + litellm_params={}, + ) + assert request.data["response_format"] == "json" + assert request.data["keywords"] == ["LiteLLM"] + assert request.data["languages"] == ["en", "fr"] + assert request.data["stream"] is True + + +def test_gpt_transcribe_optional_params_are_preserved(): + params = get_optional_params_transcription( + model="gpt-transcribe", + custom_llm_provider="openai", + keywords=["LiteLLM", "Realtime API"], + languages=["en", "fr"], + stream=True, + ) + assert params == { + "keywords": ["LiteLLM", "Realtime API"], + "languages": ["en", "fr"], + "stream": True, + } + + +def test_transcription_response_preserves_empty_languages(): + response = TranscriptionResponse(text="hello", languages=[]) + assert json.loads(response.model_dump_json())["languages"] == [] + + +def test_sync_transcription_stream_logs_final_text_and_usage_once() -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=( + 'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + 'data: {"type":"transcript.text.done","text":"hello world",' + '"usage":{"type":"duration","seconds":2.5},"languages":[{"code":"en"}]}\n\n' + ), + ) + + logging_obj: Final = MagicMock() + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + stream: Final = client.audio.transcriptions.create( + model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True + ) + wrapped: Final = wrap_transcription_stream(stream, logging_obj, datetime(2026, 1, 1), litellm_exception_mapper({})) + received: Final = tuple(wrapped) + wrapped.close() + + assert tuple(event.type for event in received) == ("transcript.text.delta", "transcript.text.done") + logging_obj.success_handler.assert_called_once() + logged_response: Final = logging_obj.success_handler.call_args.args[0] + assert logged_response.text == "hello world" + assert logged_response.usage.model_dump(exclude_none=True) == {"type": "duration", "seconds": 2.5} + assert json.loads(logged_response.model_dump_json())["languages"] == [{"code": "en"}] + logging_obj.failure_handler.assert_not_called() + + +@pytest.mark.asyncio +async def test_openai_handler_returns_native_typed_stream(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + assert b'name="stream"' in body + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "languages": [], + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + handler = OpenAIAudioTranscription() + result = handler.audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"], "stream": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + logging_obj=logging_obj, + api_key="sk-test", + api_base="https://example.com/v1", + client=openai_client, + atranscription=True, + provider_config=OpenAIGPTTranscribeAudioTranscriptionConfig(), + ) + stream = await result + assert isinstance(stream, AsyncStream) + logging_obj.async_success_handler = AsyncMock() + logging_obj.async_failure_handler = AsyncMock() + wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now(), litellm_exception_mapper({})) + received = [event async for event in wrapped_stream] + await wrapped_stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + assert received[-1].languages == [] + logging_obj.async_success_handler.assert_awaited_once() + logged_response = logging_obj.async_success_handler.await_args.kwargs["result"] + assert logged_response.text == "hello world" + assert logged_response.languages == () + + +@pytest.mark.asyncio +async def test_atranscription_stream_preserves_duration_for_callback_cost(): + async def send_response(request: httpx.Request) -> httpx.Response: + events = ( + {"type": "transcript.text.delta", "delta": "hello "}, + { + "type": "transcript.text.done", + "text": "hello world", + "usage": { + "type": "tokens", + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + }, + }, + ) + content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode() + return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + openai_client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client) + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-transcribe", messages=[], stream=True, call_type="atranscription", + start_time=datetime.now(), litellm_call_id="transcription-cost-test", function_id="transcription-cost-test", + ) + audio_file = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + + stream = await litellm.atranscription( + model="openai/gpt-transcribe", + file=audio_file, + stream=True, + client=openai_client, + litellm_logging_obj=logging_obj, + ) + received = [event async for event in stream] + await stream.close() + await openai_client.close() + + assert [event.type for event in received] == ["transcript.text.delta", "transcript.text.done"] + logged_response = logging_obj.model_call_details["async_complete_streaming_response"] + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.000075) + assert logging_obj.model_call_details["standard_logging_object"]["response_cost"] == pytest.approx(0.000075) + assert logged_response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0) + + +@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"]) +def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None: + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"): + litellm.transcription( + model=model, + file=audio_file, + response_format="verbose_json", + api_key="sk-test", + ) + + +@pytest.mark.parametrize("drop_globally", (True, False)) +def test_gpt_transcribe_drops_whisper_response_formats_under_drop_params( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, drop_globally: bool +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}}) + + monkeypatch.setattr(litellm, "drop_params", drop_globally) + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) as client: + response: Final = litellm.transcription( + model="gpt-transcribe", + file=audio_file, + response_format="verbose_json", + client=client, + **({} if drop_globally else {"drop_params": True}), + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b"verbose_json" not in sent_bodies[0] + + +def _openai_transcription_client(transport: httpx.MockTransport) -> OpenAI: + return OpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=httpx.Client(transport=transport)) + + +def _azure_transcription_client(transport: httpx.MockTransport) -> AzureOpenAI: + return AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=httpx.Client(transport=transport), + ) + + +@pytest.mark.parametrize( + ("model", "make_client"), + (("gpt-transcribe", _openai_transcription_client), ("azure/gpt-transcribe", _azure_transcription_client)), +) +def test_gpt_transcribe_sends_text_response_format_to_the_provider( + local_model_cost_map: None, model: str, make_client: Callable[[httpx.MockTransport], OpenAI | AzureOpenAI] +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, text="hello", headers={"content-type": "text/plain"}) + + audio_file: Final = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + with make_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model=model, + file=audio_file, + response_format="text", + api_key="sk-test", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="response_format"\r\n\r\ntext\r\n' in sent_bodies[0] + + +def test_file_transcription_reaches_models_whose_cost_map_lists_other_endpoints( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + sent_requests: Final[list[httpx.Request]] = [] # mutable-ok: collects what the mock upstreams received + + def send_gemini_response(request: httpx.Request) -> httpx.Response: + sent_requests.append(request) + return httpx.Response( + 200, + json={ + "id": "interaction-1", + "status": "completed", + "steps": [ + {"type": "model_generation", "content": [{"type": "text", "text": "hello", "annotations": []}]} + ], + }, + ) + + def send_whisper_response(request: httpx.Request) -> httpx.Response: + sent_requests.append(request) + return httpx.Response(200, json={"text": "hello"}) + + gemini_response: Final = litellm.transcription( + model="gemini/gemini-2.5-flash", + file=_one_second_wav(), + api_key="gemini-test-key", + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(send_gemini_response))), + ) + monkeypatch.setattr(litellm, "client_session", httpx.Client(transport=httpx.MockTransport(send_whisper_response))) + whisper_response: Final = litellm.transcription( + model="openai/gpt-realtime-whisper", + file=_one_second_wav(), + api_key="self-hosted-whisper-key", + api_base="https://whisper.self-hosted.example/v1", + ) + + assert gemini_response.text == "hello" + assert whisper_response.text == "hello" + assert tuple(str(request.url.host) for request in sent_requests) == ( + "generativelanguage.googleapis.com", + "whisper.self-hosted.example", + ) + + +def test_azure_gpt_transcribe_uses_deployment_scoped_route(): + def send_response(request: httpx.Request) -> httpx.Response: + assert str(request.url) == ( + "https://example.openai.azure.com/openai/deployments/gpt-transcribe/audio/transcriptions" + f"?api-version={litellm.AZURE_DEFAULT_API_VERSION}" + ) + return httpx.Response( + 200, + json={"text": "hello", "languages": [{"code": "en"}], "usage": {"type": "duration", "seconds": 1}}, + ) + + http_client = httpx.Client(transport=httpx.MockTransport(send_response)) + client = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"response_format": "json"}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + client=client, + ) + + assert response.text == "hello" + assert response.languages is not None + assert [language.code for language in response.languages] == ["en"] + client.close() + + +@pytest.mark.asyncio +async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body(): + async def send_response(request: httpx.Request) -> httpx.Response: + body = await request.aread() + assert b'name="keywords[]"' in body + assert b'name="languages[]"' in body + return httpx.Response(200, json={"text": "hello"}) + + http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response)) + client = AsyncAzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version="2025-04-01-preview", + http_client=http_client, + ) + audio_file = io.BytesIO(b"audio") + audio_file.name = "sample.wav" + + response = await AzureAudioTranscription().audio_transcriptions( + model="gpt-transcribe", + audio_file=audio_file, + optional_params={"keywords": ["LiteLLM"], "languages": ["en"]}, + logging_obj=MagicMock(), + model_response=TranscriptionResponse(), + timeout=10, + max_retries=0, + api_key="azure-test-key", + api_base="https://example.openai.azure.com", + api_version="2025-04-01-preview", + client=client, + atranscription=True, + ) + + assert response.text == "hello" + await client.close() + + +def _one_second_wav() -> io.BytesIO: + audio_file: Final = io.BytesIO() + with wave.open(audio_file, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(16000) + wav_file.writeframes(b"\x00\x00" * 16000) + audio_file.name = "sample.wav" + audio_file.seek(0) + return audio_file + + +class _LoggedSpend(NamedTuple): + cost: float | None + usage: dict[str, object] | None + + +def _logged_spend(kwargs: dict[str, object]) -> _LoggedSpend: + payload: Final = kwargs["standard_logging_object"] + return _LoggedSpend(payload["response_cost"], payload["metadata"]["usage_object"]) + + +class _SuccessSpendRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.async_spends: Final[asyncio.Queue[_LoggedSpend]] = asyncio.Queue() + self.sync_spends: Final[queue.Queue[_LoggedSpend]] = queue.Queue() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + await self.async_spends.put(_logged_spend(kwargs)) + + def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.sync_spends.put(_logged_spend(kwargs)) + + +@pytest.mark.asyncio +async def test_atranscription_bills_the_transcript_when_the_model_ignores_stream( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=False) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/whisper-1", file=_one_second_wav(), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + ignored_stream: Final = await litellm.atranscription( + model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client + ) + ignored_stream_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + + assert isinstance(ignored_stream, TranscriptionResponse) + assert ignored_stream.text == "hello world" + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert ignored_stream_spend == unstreamed_spend + + +def test_azure_transcription_bills_the_transcript_when_the_deployment_ignores_stream( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=False) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AzureOpenAI( + api_key="azure-test-key", + azure_endpoint="https://example.openai.azure.com", + api_version=litellm.AZURE_DEFAULT_API_VERSION, + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) + azure_params: Final = { + "client": client, + "api_key": "azure-test-key", + "api_base": "https://example.openai.azure.com", + "api_version": litellm.AZURE_DEFAULT_API_VERSION, + } + litellm.transcription(model="azure/whisper-1", file=_one_second_wav(), **azure_params) + unstreamed_spend: Final = recorder.sync_spends.get(timeout=5) + ignored_stream: Final = litellm.transcription( + model="azure/whisper-1", file=_one_second_wav(), stream=True, **azure_params + ) + ignored_stream_spend: Final = recorder.sync_spends.get(timeout=5) + client.close() + + assert isinstance(ignored_stream, TranscriptionResponse) + assert ignored_stream.text == "hello world" + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert ignored_stream_spend == unstreamed_spend + + +def _duration_priced_transcript(streamed: bool) -> httpx.Response: + usage: Final = {"type": "duration", "seconds": 2} + if not streamed: + return httpx.Response(200, json={"text": "hello world", "usage": usage}) + events: Final = ( + {"type": "transcript.text.delta", "delta": "hello world"}, + {"type": "transcript.text.done", "text": "hello world", "usage": usage}, + ) + return httpx.Response( + 200, + content="".join(f"data: {json.dumps(event)}\n\n" for event in events).encode(), + headers={"content-type": "text/event-stream"}, + ) + + +def test_whisper_stream_request_answered_with_json_keeps_the_unstreamed_response_shape( + local_model_cost_map: None, +) -> None: + verbose_transcript: Final = { + "task": "transcribe", + "language": "english", + "duration": 1.0, + "text": "hello world", + "usage": {"type": "duration", "seconds": 1}, + } + with _openai_transcription_client( + httpx.MockTransport(lambda request: httpx.Response(200, json=verbose_transcript)) + ) as client: + unstreamed: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + ignored_stream: Final = litellm.transcription( + model="openai/whisper-1", file=_one_second_wav(), stream=True, client=client + ) + + assert isinstance(ignored_stream, TranscriptionResponse) + assert json.loads(ignored_stream.model_dump_json()) == json.loads(unstreamed.model_dump_json()) + + +@pytest.mark.parametrize("requested_stream", ("true", "True", "1", "yes", "on")) +def test_transcription_stream_requested_as_a_form_string_streams_like_a_bool( + local_model_cost_map: None, requested_stream: str +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=_asks_for_stream(sent_bodies[-1])) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=requested_stream, client=client + ) + events: Final = [event.type for event in stream] + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert len(sent_bodies) == 1 + + +def test_transcription_stream_that_is_not_a_boolean_answers_400_without_calling_the_upstream( + local_model_cost_map: None, +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=False) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + with pytest.raises(litellm.BadRequestError) as raised: + litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), stream="maybe", client=client) + + assert raised.value.status_code == 400 + assert sent_bodies == [] + + +def test_transcription_with_an_empty_stream_form_value_runs_unstreamed_without_a_stream_field( + local_model_cost_map: None, +) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=False) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream="", client=client + ) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "hello world" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def test_transcription_stream_that_is_not_a_boolean_is_dropped_under_drop_params(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return _duration_priced_transcript(streamed=False) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream="maybe", drop_params=True, client=client + ) + + assert isinstance(response, TranscriptionResponse) + assert response.text == "hello world" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def _asks_for_stream(request_body: bytes) -> bool: + return b'name="stream"\r\n\r\ntrue' in request_body + + +@pytest.mark.asyncio +async def test_atranscription_stream_bills_the_provider_duration_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + events: Final = [event.type async for event in stream] + streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert streamed_spend == unstreamed_spend + + +class _TranscriptRedactor(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call) + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + if isinstance(response, TranscriptionResponse): + response.text = "[REDACTED]" + + +@pytest.mark.asyncio +async def test_atranscription_stream_under_a_post_call_guardrail_streams_the_guarded_transcript( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", + file=_one_second_wav(), + stream=True, + client=client, + guardrails=["transcript-redactor"], + ) + events: Final = [(event.type, event.model_dump(exclude={"type"}, exclude_unset=True)) async for event in stream] + await client.close() + + assert events == [ + ("transcript.text.delta", {"delta": "[REDACTED]"}), + ("transcript.text.done", {"text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}), + ] + + +class _StreamedTranscriptRecorder(CustomLogger): + def __init__(self, message_logging: bool) -> None: + super().__init__(message_logging=message_logging) + self.logged: Final[asyncio.Queue[frozenset[object]]] = asyncio.Queue() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + logged_responses: Final = ( + response_obj, + *(kwargs.get(key) for key in ("complete_streaming_response", "async_complete_streaming_response")), + ) + await self.logged.put(frozenset(response.text for response in logged_responses if response is not None)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("opt_out", ("global", "callback")) +async def test_atranscription_stream_keeps_the_transcript_out_of_callbacks_that_turned_off_message_logging( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, opt_out: str +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(await request.aread())) + + recorder: Final = _StreamedTranscriptRecorder(message_logging=opt_out != "callback") + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "turn_off_message_logging", opt_out == "global") + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + streamed_text: Final = [event.delta async for event in stream if event.type == "transcript.text.delta"] + logged_texts: Final = await asyncio.wait_for(recorder.logged.get(), timeout=5) + await client.close() + + assert streamed_text == ["hello world"] + assert logged_texts == {REDACTED_BY_LITELLM} + + +def test_transcription_stream_bills_the_provider_duration_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + def send_response(request: httpx.Request) -> httpx.Response: + return _duration_priced_transcript(streamed=_asks_for_stream(request.read())) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = OpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.Client(transport=httpx.MockTransport(send_response)), + ) + litellm.transcription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + unstreamed_spend: Final = recorder.sync_spends.get(timeout=5) + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), stream=True, client=client + ) + events: Final = [event.type for event in stream] + streamed_spend: Final = recorder.sync_spends.get(timeout=5) + client.close() + + assert events == ["transcript.text.delta", "transcript.text.done"] + assert unstreamed_spend.cost is not None + assert unstreamed_spend.cost > 0 + assert unstreamed_spend.usage + assert streamed_spend == unstreamed_spend + + +class _LoggedResponseRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.responses: Final[queue.Queue[object]] = queue.Queue() + + def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.responses.put(kwargs["standard_logging_object"]["response"]) + + +@pytest.mark.parametrize( + "provider_body", + ({"text": "hello"}, {"text": "hello", "languages": ["en"]}, {"text": "hello", "languages": None}), +) +def test_transcription_without_parsed_languages_has_no_languages_key( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object] +) -> None: + recorder: Final = _LoggedResponseRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + with _openai_transcription_client( + httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body)) + ) as client: + response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + logged_response: Final = recorder.responses.get(timeout=5) + + assert response.text == "hello" + assert "languages" not in json.loads(response.model_dump_json()) + assert isinstance(logged_response, dict) + assert "languages" not in logged_response + + +def test_unstreamed_transcription_sends_no_stream_field(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello"}) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription(model="openai/whisper-1", file=_one_second_wav(), client=client) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="stream"' not in sent_bodies[0] + + +def test_transcription_forwards_both_language_and_languages(local_model_cost_map: None) -> None: + sent_bodies: Final[list[bytes]] = [] # mutable-ok: collects what the mock upstream received + + def send_response(request: httpx.Request) -> httpx.Response: + sent_bodies.append(request.read()) + return httpx.Response(200, json={"text": "hello", "usage": {"type": "duration", "seconds": 1}}) + + with _openai_transcription_client(httpx.MockTransport(send_response)) as client: + response: Final = litellm.transcription( + model="openai/gpt-transcribe", file=_one_second_wav(), language="en", languages=["fr"], client=client + ) + + assert response.text == "hello" + assert len(sent_bodies) == 1 + assert b'name="language"\r\n\r\nen\r\n' in sent_bodies[0] + assert b'name="languages[]"\r\n\r\nfr\r\n' in sent_bodies[0] + + +def _gpt_transcribe_cost_per_second() -> float: + rate: Final = litellm.model_cost["gpt-transcribe"]["input_cost_per_second"] + assert isinstance(rate, float) + return rate + + +async def _spend_for_one_second_wav( + monkeypatch: pytest.MonkeyPatch, provider_body: dict[str, object], router: litellm.Router | None +) -> _LoggedSpend: + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, json=provider_body)) + ), + ) + if router is None: + await litellm.atranscription(model="openai/gpt-transcribe", file=_one_second_wav(), client=client) + else: + await router.atranscription(model="my-transcribe", file=_one_second_wav(), client=client) + spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await client.close() + return spend + + +@pytest.mark.asyncio +async def test_custom_named_gpt_transcribe_deployment_bills_the_measured_upload_over_reported_seconds( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "my-transcribe", + "litellm_params": {"model": "openai/my-transcribe", "api_key": "sk-test"}, + "model_info": {"base_model": "gpt-transcribe"}, + } + ] + ) + + spend: Final = await _spend_for_one_second_wav( + monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 2}}, router + ) + + assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second()) + + +@pytest.mark.asyncio +async def test_gpt_transcribe_bills_the_file_length_when_the_provider_reports_zero_seconds( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + spend: Final = await _spend_for_one_second_wav( + monkeypatch, {"text": "hello", "usage": {"type": "duration", "seconds": 0}}, None + ) + + assert spend.cost == pytest.approx(_gpt_transcribe_cost_per_second()) + + +@pytest.mark.asyncio +async def test_m4a_transcription_stream_that_ends_early_logs_one_spend_row_like_the_unstreamed_call( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + async def send_response(request: httpx.Request) -> httpx.Response: + if not _asks_for_stream(await request.aread()): + return httpx.Response(200, json={"text": "hello"}) + return httpx.Response( + 200, + content=f"data: {json.dumps({'type': 'transcript.text.delta', 'delta': 'hello'})}\n\n".encode(), + headers={"content-type": "text/event-stream"}, + ) + + recorder: Final = _SuccessSpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + client: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://example.com/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(send_response)), + ) + await litellm.atranscription(model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), client=client) + unstreamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.m4a", b"m4a audio"), stream=True, client=client + ) + events: Final = [event.type async for event in stream] + streamed_spend: Final = await asyncio.wait_for(recorder.async_spends.get(), timeout=5) + await asyncio.sleep(0.2) + await client.close() + + assert events == ["transcript.text.delta"] + assert streamed_spend.cost == unstreamed_spend.cost + assert recorder.async_spends.empty() diff --git a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py index 703fa13cbc9..61798b49be0 100644 --- a/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py +++ b/tests/unit/llms/openai/transcriptions/test_transcription_duration_hidden.py @@ -9,6 +9,9 @@ TranscriptionVerbose/Diarized type. from unittest.mock import patch +import pytest + +import litellm from litellm.cost_calculator import completion_cost from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( convert_to_model_response_object, @@ -55,10 +58,7 @@ class TestDiarizedJsonUsageParsing: assert result.usage.seconds == 295.8 def test_usage_duration_object_accepts_float_seconds(self): - assert ( - TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds - == 295.8 - ) + assert TranscriptionUsageDurationObject(type="duration", seconds=295.8).seconds == 295.8 class TestTranscriptionDurationNotInResponseBody: @@ -126,6 +126,61 @@ class TestTranscriptionDurationNotInResponseBody: class TestCostCalculatorReadsDurationFromHiddenParams: """The cost calculator should read duration from _hidden_params via completion_cost().""" + @patch( # test-quality-ok: isolates duration selection from the model pricing table + "litellm.cost_calculator.openai_cost_per_second" + ) + def test_completion_cost_prefers_measured_duration_over_provider_usage_duration(self, mock_cost_fn): + mock_cost_fn.return_value = (0.001, 0.0) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = { + "audio_transcription_duration": 17.5, + "custom_llm_provider": "openai", + } + + completion_cost( + completion_response=response, + model="gpt-transcribe", + call_type="atranscription", + ) + + mock_cost_fn.assert_called_once() + _, kwargs = mock_cost_fn.call_args + assert kwargs["duration"] == 17.5 + + @pytest.mark.parametrize( + ("model", "provider"), + [ + ("gpt-transcribe", "openai"), + ("azure/gpt-transcribe", "azure"), + ], + ) + def test_gpt_transcribe_provider_usage_duration_costs_without_local_decode( + self, + monkeypatch, + model: str, + provider: str, + ): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + monkeypatch.setitem(litellm.model_cost, model, {**litellm.model_cost[model], "input_cost_per_second": 0.0125}) + response = TranscriptionResponse( + text="test", + usage={"type": "duration", "seconds": 60.0}, + ) + response._hidden_params = {"custom_llm_provider": provider} + + actual = completion_cost( + completion_response=response, + model=model, + call_type="atranscription", + custom_llm_provider=provider, + ) + + assert actual == pytest.approx(60.0 * 0.0125) + @patch("litellm.cost_calculator.openai_cost_per_second") def test_completion_cost_uses_hidden_params_duration(self, mock_cost_fn): """ diff --git a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py index 2b682fda982..37dfdbf489c 100644 --- a/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py +++ b/tests/unit/proxy/agent_endpoints/auth/test_managed_authorization.py @@ -364,6 +364,10 @@ async def test_unknown_invocation_target_leaves_billing_unset(monkeypatch: pytes ("/v1beta/models/gemini-model:generateContent", "POST", True), ("/v1/realtime", "GET", True), ("/v1/realtime", "POST", False), + ("/v1/realtime/translations", "GET", True), + ("/v1/realtime/translations", "POST", False), + ("/v1/realtime/translations/client_secrets", "POST", False), + ("/v1/realtime/translations/calls", "POST", False), ("/v1/realtime/client_secrets", "POST", False), ("/mcp/tools/call", "POST", True), ("/a2a/target/message/send", "POST", True), @@ -437,14 +441,21 @@ def test_managed_inference_ignores_unsupported_query_model(): ) -@pytest.mark.parametrize("route", ["/realtime", "/v1/realtime", "/openai/v1/realtime"]) +@pytest.mark.parametrize( + "route", + ( + "/realtime", + "/v1/realtime", + "/openai/v1/realtime", + ), +) def test_managed_realtime_requires_a_model_and_ignores_completion_defaults(route: str) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request with pytest.raises(HTTPException, match="explicit or configured model"): managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli") assert ( - managed_inference_request(route, {"model": "requested"}, {"completion_model": "allowed-default"}, "cli")[ + managed_inference_request(route, {}, {"completion_model": "allowed-default"}, "cli", query_model="requested")[ "model" ] == "requested" @@ -465,6 +476,20 @@ def test_managed_live_upgrade_defers_the_model_to_the_session_start_frame(route: ) == {"model": "frame-model"} +@pytest.mark.parametrize("route", ("/realtime/translations", "/v1/realtime/translations")) +@pytest.mark.parametrize("query_model", (None, "requested")) +def test_managed_translation_authorizes_the_same_model_as_the_endpoint(route: str, query_model: str | None) -> None: + from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_inference_request + from litellm.proxy.common_utils.http_parsing_utils import resolve_realtime_route_model + + admitted: Final = managed_inference_request( + route, {}, {"completion_model": "unrelated-default"}, "cli", query_model=query_model + ) + + assert admitted["model"] == resolve_realtime_route_model(query_model, None, is_translation=True) + assert isinstance(admitted["model"], str) and admitted["model"] + + @pytest.mark.parametrize("mode,user", [("autonomous", None), ("delegated", "verified-human")]) def test_matching_identity_revision_and_execution_mode_pass_admission(mode: str, user: str | None) -> None: context: Final = ManagedAgentContext.model_validate( @@ -573,21 +598,41 @@ async def test_ordinary_agent_admission_preserves_legacy_authentication( @pytest.mark.parametrize( "route", - tuple(dict.fromkeys( - LiteLLMRoutes.openai_routes.value - + LiteLLMRoutes.anthropic_routes.value - + LiteLLMRoutes.google_routes.value - )), + tuple( + dict.fromkeys( + LiteLLMRoutes.openai_routes.value + LiteLLMRoutes.anthropic_routes.value + LiteLLMRoutes.google_routes.value + ) + ), ) def test_registered_inference_routes_have_an_explicit_managed_access_decision(route: str) -> None: from litellm.proxy.agent_endpoints.auth.managed_authorization import managed_agent_route_allowed normalized: Final = route.removeprefix("/openai").removeprefix("/v1beta").removeprefix("/v1") - unsupported: Final = normalized.startswith(( - "/videos", "/batches", "/files", "/fine_tuning", "/assistants", "/threads", "/utils/", - "/vector_stores", "/vector_store/", "/search", "/containers", "/skills", "/claude-code/", - "/interactions", "/agents", "/responses/{", "/responses/input_tokens", - "/realtime/client_secrets", "/realtime/calls", "/realtime/transcription_sessions", - )) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models") + unsupported: Final = normalized.startswith( + ( + "/videos", + "/batches", + "/files", + "/fine_tuning", + "/assistants", + "/threads", + "/utils/", + "/vector_stores", + "/vector_store/", + "/search", + "/containers", + "/skills", + "/claude-code/", + "/interactions", + "/agents", + "/responses/{", + "/responses/input_tokens", + "/realtime/client_secrets", + "/realtime/calls", + "/realtime/transcription_sessions", + "/realtime/translations/client_secrets", + "/realtime/translations/calls", + ) + ) or normalized in ("/models", "/cursor/models", "/cursor/v1/models", "/claude_code_gateway/v1/models") concrete: Final = route.split("?")[0].replace("{model}", "model").replace("{model_name:path}", "model") assert managed_agent_route_allowed(concrete, None) is not unsupported, route diff --git a/tests/unit/proxy/auth/test_route_checks.py b/tests/unit/proxy/auth/test_route_checks.py index 5df3fbab9a6..efb4451f2fc 100644 --- a/tests/unit/proxy/auth/test_route_checks.py +++ b/tests/unit/proxy/auth/test_route_checks.py @@ -818,6 +818,52 @@ def test_realtime_webrtc_http_routes_classified_as_llm_api(route): assert RouteChecks.is_management_route(route=route) is False +@pytest.mark.parametrize( + "route", + [ + "/realtime/translations", + "/v1/realtime/translations", + "/realtime/translations/client_secrets", + "/v1/realtime/translations/client_secrets", + "/realtime/translations/calls", + "/v1/realtime/translations/calls", + ], +) +def test_realtime_translation_routes_allowed_for_openai_virtual_keys(route): + valid_token = UserAPIKeyAuth( + user_id="test_user", + allowed_routes=["openai_routes"], + ) + + assert RouteChecks.is_llm_api_route(route=route) is True + assert RouteChecks.is_management_route(route=route) is False + assert ( + RouteChecks.is_virtual_key_allowed_to_call_route( + route=route, + valid_token=valid_token, + ) + is True + ) + + +@pytest.mark.parametrize( + "route", + [ + "/openai/v1/realtime/translations", + "/openai/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/calls", + ], +) +def test_openai_prefixed_realtime_translation_routes_denied_for_openai_virtual_keys(route: str) -> None: + valid_token: Final = UserAPIKeyAuth(user_id="test_user", allowed_routes=["openai_routes"]) + + with pytest.raises(HTTPException) as exc_info: + RouteChecks.is_virtual_key_allowed_to_call_route(route=route, valid_token=valid_token) + + assert exc_info.value.status_code == 403 + assert f"Tried to call route: {route}" in str(exc_info.value.detail) + + def test_virtual_key_allowed_routes_with_litellm_routes_member_name_denied(): """Test that virtual key is denied when route is not in the allowed LiteLLMRoutes group""" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index 81ea0e3f60b..43a096e0f42 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -931,6 +931,43 @@ async def test_user_api_key_auth_live_websocket_ignores_query_model(): mock_user_api_key_auth.assert_called_once() +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("auth_name", "path"), + [("user_api_key_auth_websocket", "/v1/responses"), ("user_api_key_auth_realtime_websocket", "/v1/realtime")], +) +async def test_websocket_without_a_model_is_refused_for_a_role_limited_to_other_models( + auth_name: str, path: str +) -> None: + from litellm.proxy._types import RoleBasedPermissions, UserAPIKeyAuth + from litellm.proxy.auth import user_api_key_auth as auth_module + from litellm.proxy.auth.handle_jwt import JWTAuthManager + + async def role_limited_auth(request, api_key: str) -> UserAPIKeyAuth: + JWTAuthManager.can_rbac_role_call_model( + rbac_role=LitellmUserRoles.INTERNAL_USER, + general_settings={ + "role_permissions": [RoleBasedPermissions(role=LitellmUserRoles.INTERNAL_USER, models=["chat"])] + }, + model=json.loads(await request.body())["model"], + ) + return UserAPIKeyAuth(api_key=api_key) + + websocket: Final = MagicMock(spec=WebSocket) + websocket.query_params = {} + websocket.headers = {"authorization": "Bearer some_api_key"} + websocket.scope = {"type": "websocket", "path": path, "headers": [(b"authorization", b"Bearer some_api_key")]} + websocket.url = URL(url=path) + websocket.close = AsyncMock() + + with patch("litellm.proxy.auth.user_api_key_auth.user_api_key_auth", side_effect=role_limited_auth): + with pytest.raises(HTTPException) as raised: + await getattr(auth_module, auth_name)(websocket) + + assert raised.value.status_code == 403 + websocket.close.assert_awaited_once_with(code=status.WS_1008_POLICY_VIOLATION) + + @pytest.mark.asyncio async def test_accepted_websocket_reauthorizes_the_frame_model_without_closing(): from litellm.proxy._types import ProxyException diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 10738eabddf..17e6c004566 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -12872,6 +12872,188 @@ async def test_websocket_auth_hands_the_reservation_to_the_socket_state(): assert websocket.scope["state"]["budget_reservation"] is reservation +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime/translations", "/v1/realtime/translations")) +@pytest.mark.parametrize("query", (b"", b"model=gpt-realtime-translate"), ids=("default_model", "explicit_model")) +async def test_translation_websocket_auth_enforces_the_default_model_budget( + monkeypatch: pytest.MonkeyPatch, path: str, query: bytes +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket + + model: Final = "gpt-realtime-translate" + token: Final = UserAPIKeyAuth( + token="test-key", + models=[model], + model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0}) + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=token, model=model) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": query, + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + with pytest.raises(HTTPException) as denied: + await user_api_key_auth_realtime_websocket(websocket) + + assert denied.value.status_code == 403 + assert model in str(denied.value.detail) + assert "budget" in str(denied.value.detail).lower() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime/translations/client_secrets", "/v1/realtime/translations/client_secrets")) +async def test_translation_client_secret_auth_enforces_the_session_model_budget( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + model: Final = "gpt-realtime-translate" + token: Final = UserAPIKeyAuth( + token="test-key", + models=[model], + model_max_budget={model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={model: 10.0}) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + body: Final = json.dumps({"session": {"type": "translation", "model": model}}).encode() + request: Final = Request( + scope={ + "type": "http", + "method": "POST", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test"), (b"content-type", b"application/json")], + "query_string": b"", + }, + receive=AsyncMock(return_value={"type": "http.request", "body": body, "more_body": False}), + ) + + with pytest.raises(ProxyException) as denied: + await user_api_key_auth(request=request, api_key="Bearer sk-test") + + assert denied.value.type == "budget_exceeded" + assert model in denied.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/realtime", "/v1/realtime", "/openai/v1/realtime")) +async def test_managed_agent_transcription_websocket_without_a_model_is_refused( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_realtime_websocket + from litellm.types.agents import AgentResponse + + token: Final = UserAPIKeyAuth(token="test-key") + token.managed_agent_policy = AgentResponse.model_validate( + {"agent_id": "agent", "agent_name": "Agent", "agent_card_params": {}, "execution_mode": "both"} + ) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = {**_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), "prisma_client": None} + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": b"intent=transcription", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + with pytest.raises(HTTPException) as denied: + await user_api_key_auth_realtime_websocket(websocket) + + assert denied.value.status_code == 403 + assert "Managed inference requires an explicit or configured model" in str(denied.value.detail) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ("/openai_passthrough/v1/realtime", "/v1/responses", "/vertex_ai/live")) +async def test_non_realtime_websocket_auth_checks_no_default_transcription_model( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from fastapi import WebSocket + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + + exhausted_model: Final = "gpt-realtime-whisper" + token: Final = UserAPIKeyAuth( + token="test-key", + model_max_budget={exhausted_model: {"max_budget": 5.0, "budget_duration": "1d"}}, + ) + limiter, _ = await _real_model_budget_limiter(key_spend={exhausted_model: 10.0}) + with pytest.raises(litellm.BudgetExceededError): + await limiter.is_key_within_model_budget(user_api_key_dict=token, model=exhausted_model) + + async def authenticate(request: Request, api_key: str) -> UserAPIKeyAuth: + return token + + attrs: Final = { + **_proxy_server_attrs_for_custom_auth(user_custom_auth=authenticate), + "prisma_client": None, + "model_max_budget_limiter": limiter, + } + for name, value in attrs.items(): + monkeypatch.setattr(proxy_server, name, value) + monkeypatch.setattr(litellm, "enable_post_custom_auth_checks", True, raising=False) + websocket: Final = WebSocket( + scope={ + "type": "websocket", + "path": path, + "headers": [(b"authorization", b"Bearer sk-test")], + "query_string": b"intent=transcription", + }, + receive=AsyncMock(), + send=AsyncMock(), + ) + + assert (await user_api_key_auth_websocket(websocket)).token == token.token + + @pytest.mark.asyncio async def test_admission_and_budget_reservation_read_the_key_spend_counter_with_one_redis_mget(): from fastapi import Request diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index 62189f5b0e7..84cb62f0426 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -786,6 +786,17 @@ async def test_get_form_data(): assert result["temperature"] == "0.2" assert result["include"] == ["logprobs"] assert result["timestamp_granularities"] == ["word", "segment"] + assert tuple(result) == ( + "file", + "model", + "include", + "language", + "prompt", + "response_format", + "stream", + "temperature", + "timestamp_granularities", + ) def test_get_tags_from_request_body_with_metadata_tags(): diff --git a/tests/unit/proxy/common_utils/test_realtime_cache.py b/tests/unit/proxy/common_utils/test_realtime_cache.py index 9e76caff054..893e39b309a 100644 --- a/tests/unit/proxy/common_utils/test_realtime_cache.py +++ b/tests/unit/proxy/common_utils/test_realtime_cache.py @@ -1,7 +1,9 @@ import json +from collections.abc import Mapping from typing import Any, Final, cast import pytest +from pydantic import TypeAdapter from litellm.proxy.common_utils.realtime_utils import realtime_request_body from litellm.proxy.proxy_server import _realtime_query_params_template @@ -41,6 +43,13 @@ def test_realtime_request_body_caches_each_model_separately(): assert gpt4o_body_first is not gpt4o_mini_body +@pytest.mark.parametrize("model", (None, "requested", 'model"with\\quotes', "model\nwith-newline")) +def test_realtime_request_body_preserves_model_names(model: str | None) -> None: + body: Final = TypeAdapter(Mapping[str, object]).validate_json(realtime_request_body(model)) + + assert body["model"] == model + + def test_realtime_request_body_keeps_a_model_name_with_quotes_inside_its_own_field(): body: Final = realtime_request_body('gpt-4o", "user": "someone-else') diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index 9b39678f203..9fa617c0dc3 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -11,21 +11,35 @@ from __future__ import annotations import contextlib import io -from typing import TYPE_CHECKING, Callable, Final +import json +import math +import wave +from collections.abc import AsyncIterator, Callable, Mapping +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import AsyncMock, MagicMock import httpx import pytest +import respx from fastapi import HTTPException +from fastapi.testclient import TestClient +from openai import AsyncOpenAI +from starlette.datastructures import FormData, UploadFile +import litellm from litellm import Router +from litellm.caching.caching import Cache +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy import proxy_server from litellm.proxy._types import ProxyErrorTypes, ProxyException +from litellm.proxy.route_llm_request import route_request +from litellm.proxy.utils import _failure_fields_to_lift +from litellm.types.guardrails import GuardrailEventHooks, GuardrailStreamScope from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep from litellm.types.router import RouterErrors - -if TYPE_CHECKING: - from fastapi.testclient import TestClient +from litellm.types.utils import TranscriptionResponse _TAG_ROUTING_DENIAL = f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model=tts-1 and tags=['x']" @@ -130,9 +144,7 @@ def patched_transcription(monkeypatch): return data monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - monkeypatch.setattr( - proxy_server, "check_file_size_under_limit", lambda **kwargs: True - ) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True) async def _form_data(request): from starlette.datastructures import FormData, UploadFile @@ -164,6 +176,47 @@ def patched_transcription_error(monkeypatch, patched_transcription): yield +@pytest.fixture +def patched_transcription_stream(monkeypatch, patched_transcription): + class _FakeEvent: + def model_dump_json(self, exclude_unset: bool = False) -> str: + return '{"type":"transcript.text.done","text":"hello world"}' + + class _FakeAsyncStream: + def __init__(self): + self.closed = False + + def __aiter__(self): + async def _events(): + yield _FakeEvent() + + return _events() + + async def aclose(self): + self.closed = True + + async def _form_data(request): + from starlette.datastructures import FormData, UploadFile + + upload = UploadFile( + filename="audio.mp3", + file=io.BytesIO(b"\x00\x01\x02"), + ) + return FormData([("file", upload), ("model", "gpt-transcribe"), ("stream", "true")]) + + stream = _FakeAsyncStream() + + async def _llm_call(): + return stream + + async def _fake_route_request(*args, **kwargs): + return _llm_call() + + monkeypatch.setattr(proxy_server, "get_form_data", _form_data) + monkeypatch.setattr(proxy_server, "route_request", _fake_route_request) + yield stream + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_happy_path(client, auth_as, patched_speech, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (happy).""" @@ -366,6 +419,637 @@ def test_audio_transcription_error(client, auth_as, patched_transcription_error, assert len(response.content) > 0 +@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) +def test_audio_transcription_stream_returns_sse(client, auth_as, patched_transcription_stream, path): + files = {"file": ("audio.mp3", b"\x00\x01\x02", "audio/mpeg")} + data = {"model": "gpt-transcribe", "stream": "true"} + with auth_as(): + response = client.post(path, files=files, data=data) + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert response.text == 'data: {"type":"transcript.text.done","text":"hello world"}\n\ndata: [DONE]\n\n' + assert patched_transcription_stream.closed is True + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize("configuration", ["default", "policy"]) +def test_streaming_transcription_keeps_streaming_with_output_guardrails( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + configuration: str, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="transcription-output", + event_hook=GuardrailEventHooks.post_call, + default_on=configuration == "default", + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + if configuration == "policy": + pipeline: Final = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="transcription-output")]) + proxy_server.proxy_logging_obj.pre_call_hook.side_effect = lambda **kwargs: { + **kwargs["data"], + "metadata": {"_guardrail_pipelines": [("transcription-policy", pipeline)]}, + } + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + assert '"text":"hello world"' in response.text + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_returns_the_litellm_call_and_model_headers( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + content=b'data: {"type":"transcript.text.done","text":"hello world"}\n\n', + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription( + model="openai/gpt-transcribe", + file=data["file"], + stream=True, + client=upstream, + litellm_call_id=data["litellm_call_id"], + model_info={"id": "transcription-deployment"}, + ) + + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + headers={"x-litellm-call-id": "streamed-transcription-call"}, + ) + + assert response.status_code == 200 + assert '"text":"hello world"' in response.text + assert response.headers["x-litellm-call-id"] == "streamed-transcription-call" + assert response.headers["x-litellm-model-id"] == "transcription-deployment" + assert [name for name in response.headers if name.startswith("x-litellm-response-cost")] == [] + proxy_server.proxy_logging_obj.update_request_status.assert_called_once_with( + litellm_call_id="streamed-transcription-call", status="success" + ) + + +class _TranscriptRedactor(CustomGuardrail): + def __init__(self, stream_scope: GuardrailStreamScope = "both") -> None: + super().__init__( + guardrail_name="transcript-redactor", event_hook=GuardrailEventHooks.post_call, stream_scope=stream_scope + ) + + async def async_post_call_success_hook( + self, data: dict[str, object], user_api_key_dict: object, response: object + ) -> None: + if isinstance(response, TranscriptionResponse): + response.text = "[REDACTED]" + + +def _streams_when_asked(request: httpx.Request) -> httpx.Response: + if b'name="stream"' in request.content: + return httpx.Response( + 200, + content=b'data: {"type":"transcript.text.done","text":"my pin is 1234"}\n\n', + headers={"content-type": "text/event-stream"}, + ) + return httpx.Response(200, json={"text": "my pin is 1234", "usage": {"type": "duration", "seconds": 2}}) + + +def _sse_events(body: str) -> tuple[object, ...]: + assert body.endswith("data: [DONE]\n\n") + return tuple(json.loads(frame.removeprefix("data: ")) for frame in body.removesuffix("data: [DONE]\n\n").split("\n\n") if frame) + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize("deployment_guardrails", (["transcript-redactor"], None)) +def test_streamed_transcription_honors_a_deployment_post_call_guardrail( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_guardrails: list[str] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(side_effect=_streams_when_asked) + deployment_params: Final = { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + } + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": ( + {**deployment_params, "guardrails": deployment_guardrails} + if deployment_guardrails + else deployment_params + ), + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + if deployment_guardrails: + assert response.headers["content-type"].startswith("text/event-stream") + assert _sse_events(response.text) == ( + {"type": "transcript.text.delta", "delta": "[REDACTED]"}, + {"type": "transcript.text.done", "text": "[REDACTED]", "usage": {"type": "duration", "seconds": 2}}, + ) + else: + assert response.headers["content-type"].startswith("text/event-stream") + assert '"text":"my pin is 1234"' in response.text + + +@pytest.mark.usefixtures("patched_transcription_stream") +@pytest.mark.parametrize( + "stream_scope,upstream_streamed,expected_text", + [("streaming", False, "[REDACTED]"), ("non_streaming", True, "my pin is 1234")], +) +def test_streamed_transcription_applies_a_deployment_guardrail_only_in_its_stream_scope( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + stream_scope: GuardrailStreamScope, + upstream_streamed: bool, + expected_text: str, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor(stream_scope=stream_scope)]) + upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock( + side_effect=_streams_when_asked + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "guardrails": ["transcript-redactor"], + }, + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream") + assert (b'name="stream"' in upstream_route.calls.last.request.content) is upstream_streamed + assert _sse_events(response.text)[-1]["text"] == expected_text + + +async def _form_without_stream_field(request: object) -> FormData: + return FormData( + [("file", UploadFile(filename="sample.wav", file=io.BytesIO(b"audio"))), ("model", "gpt-transcribe")] + ) + + +@pytest.mark.usefixtures("patched_transcription") +@pytest.mark.parametrize("deployment_guardrails", (["transcript-redactor"], None)) +def test_a_deployment_that_streams_transcription_answers_sse_to_a_client_that_sent_no_stream_field( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + deployment_guardrails: list[str] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + monkeypatch.setattr(proxy_server, "get_form_data", _form_without_stream_field) + respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock(side_effect=_streams_when_asked) + deployment_params: Final = { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "stream": True, + } + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": ( + {**deployment_params, "guardrails": deployment_guardrails} + if deployment_guardrails + else deployment_params + ), + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe"}, + ) + + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("text/event-stream") + assert _sse_events(response.text)[-1]["text"] == ("[REDACTED]" if deployment_guardrails else "my pin is 1234") + header_hook_response: Final = proxy_server.proxy_logging_obj.post_call_response_headers_hook.await_args.kwargs[ + "response" + ] + if deployment_guardrails: + assert isinstance(header_hook_response, TranscriptionResponse) + assert header_hook_response.text == "[REDACTED]" + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_cached_guarded_transcription_still_streams_to_a_stream_client( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "callbacks", [_TranscriptRedactor()]) + monkeypatch.setattr(litellm, "cache", Cache(type="local")) + upstream_route: Final = respx_mock.post("https://transcribe-upstream.example/v1/audio/transcriptions").mock( + side_effect=_streams_when_asked + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-transcribe", + "litellm_params": { + "model": "openai/gpt-transcribe", + "api_key": "sk-test", + "api_base": "https://transcribe-upstream.example/v1", + "guardrails": ["transcript-redactor"], + }, + } + ], + cache_responses=True, + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + + with auth_as(): + responses: Final = [ + client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + for _ in range(2) + ] + + assert upstream_route.call_count == 1 + assert [response.headers["content-type"].split(";")[0] for response in responses] == ["text/event-stream"] * 2 + assert _sse_events(responses[1].text)[-1] == { + "type": "transcript.text.done", + "text": "[REDACTED]", + "usage": {"type": "duration", "seconds": 2}, + } + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_frames_keep_the_provider_event_shape( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider_events: Final = ( + {"type": "transcript.text.delta", "delta": "hello"}, + {"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 2}}, + ) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, + content="".join(f"data: {json.dumps(event)}\n\n" for event in provider_events).encode(), + headers={"content-type": "text/event-stream"}, + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription( + model="openai/gpt-transcribe", file=data["file"], stream=data["stream"], client=upstream + ) + + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert _sse_events(response.text) == provider_events + + +class _FirstDeltaThenReset(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise httpx.ReadError("connection reset") + + +def _silent_wav(seconds: int) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16_000) + wav.writeframes(b"\x00\x00" * 16_000 * seconds) + return buffer.getvalue() + + +@pytest.mark.usefixtures("patched_transcription") +def test_a_streamed_transcription_that_fails_mid_transcript_bills_the_uploaded_seconds( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenReset() + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream) + + four_second_upload: Final = _silent_wav(4) + + async def stream_form(request: object) -> FormData: + return FormData( + [ + ("file", UploadFile(filename="sample.wav", file=io.BytesIO(four_second_upload))), + ("model", "gpt-transcribe"), + ("stream", "true"), + ] + ) + + monkeypatch.setattr(proxy_server, "get_form_data", stream_form) + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", four_second_upload, "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert '"error"' in response.text + failure_request: Final = proxy_server.proxy_logging_obj.post_call_failure_hook.await_args.kwargs["request_data"] + billed_cost: Final = _failure_fields_to_lift(failure_request)["response_cost"] + assert isinstance(billed_cost, float) + assert math.isclose(billed_cost, 4 * litellm.model_cost["gpt-transcribe"]["input_cost_per_second"]) + + +async def _scribe_stream_form(request: object) -> FormData: + return FormData( + [ + ("file", UploadFile(filename="sample.wav", file=io.BytesIO(b"audio"))), + ("model", "scribe"), + ("stream", "true"), + ] + ) + + +@pytest.mark.usefixtures("patched_transcription") +def test_transcription_stream_form_field_reaches_other_providers_as_sent( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + upstream: Final = respx_mock.post("https://elevenlabs.example/v1/speech-to-text").mock( + return_value=httpx.Response(200, json={"text": "hello world", "language_code": "en"}) + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "scribe", + "litellm_params": { + "model": "elevenlabs/scribe_v1", + "api_key": "sk-test", + "api_base": "https://elevenlabs.example", + }, + } + ] + ), + ) + monkeypatch.setattr(proxy_server, "route_request", route_request) + monkeypatch.setattr(proxy_server, "get_form_data", _scribe_stream_form) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "scribe", "stream": "true"}, + ) + + assert response.status_code == 200 + assert response.json()["text"] == "hello world" + assert b'name="stream"\r\n\r\ntrue\r\n' in upstream.calls.last.request.content + + +def _streams_chat_and_transcription(request: httpx.Request) -> httpx.Response: + sse_body: Final = ( + b'data: {"id":"chat-1","object":"chat.completion.chunk","created":0,"model":"gpt-4o",' + b'"choices":[{"index":0,"delta":{"content":"hi"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' + if request.url.path.endswith("/chat/completions") + else b'data: {"type":"transcript.text.done","text":"hello world"}\n\n' + ) + return httpx.Response( + 200, + content=sse_body, + headers={"content-type": "text/event-stream", "x-ratelimit-remaining-requests": "99"}, + ) + + +def _litellm_header_names(response: httpx.Response) -> frozenset[str]: + return frozenset( + name + for name in response.headers + if name.startswith("x-litellm-") and not name.startswith("x-litellm-response-cost") + ) + + +def test_streamed_transcription_carries_the_headers_a_streamed_chat_completion_carries( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.route(host="stream-upstream.example").mock(side_effect=_streams_chat_and_transcription) + upstream: Final = {"api_key": "sk-test", "api_base": "https://stream-upstream.example/v1"} + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", **upstream}}, + {"model_name": "gpt-transcribe", "litellm_params": {"model": "openai/gpt-transcribe", **upstream}}, + ] + ), + ) + + with auth_as(): + chat: Final = client.post( + "/v1/chat/completions", + json={"model": "gpt-4o", "stream": True, "messages": [{"role": "user", "content": "hi"}]}, + ) + transcription: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + assert chat.status_code == 200 + assert transcription.status_code == 200 + assert transcription.headers["content-type"].startswith("text/event-stream") + assert _litellm_header_names(chat) - _litellm_header_names(transcription) <= {"x-litellm-overhead-duration-ms"} + assert transcription.headers["x-litellm-model-name"] == "openai/gpt-transcribe" + assert transcription.headers["llm_provider-x-ratelimit-remaining-requests"] == "99" + assert [name for name in transcription.headers if name.startswith("x-litellm-response-cost")] == [] + + +class _FailsAfterTwoDeltas(httpx.AsyncByteStream): + async def __aiter__(self) -> AsyncIterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"hello "}\n\n' + yield b'data: {"type":"transcript.text.delta","delta":"world"}\n\n' + raise httpx.ReadError("upstream connection dropped at /srv/app/stt_stream.py via stt.internal.corp") + + +@pytest.mark.usefixtures("patched_transcription_stream") +def test_streamed_transcription_error_mid_stream_ends_with_an_error_event_and_runs_the_failure_hook( + client: TestClient, + auth_as: Callable[[], AbstractContextManager[None]], + monkeypatch: pytest.MonkeyPatch, +) -> None: + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient( + transport=httpx.MockTransport( + lambda request: httpx.Response( + 200, stream=_FailsAfterTwoDeltas(), headers={"content-type": "text/event-stream"} + ) + ) + ), + ) + + async def route_to_the_sdk( + data: Mapping[str, object], route_type: str, llm_router: object, user_model: object + ) -> object: + return litellm.atranscription(model="openai/gpt-transcribe", file=data["file"], stream=True, client=upstream) + + failure_hook: Final = AsyncMock(return_value=None) + monkeypatch.setattr(proxy_server.proxy_logging_obj, "post_call_failure_hook", failure_hook) + monkeypatch.setattr(proxy_server, "route_request", route_to_the_sdk) + + with auth_as(): + response: Final = client.post( + "/v1/audio/transcriptions", + files={"file": ("sample.wav", b"audio", "audio/wav")}, + data={"model": "gpt-transcribe", "stream": "true"}, + ) + + data_lines: Final = tuple( + line.removeprefix("data: ") for line in response.text.splitlines() if line.startswith("data: ") + ) + frames: Final = tuple(json.loads(line) for line in data_lines[:-1]) + assert response.status_code == 200 + assert tuple(frame.get("delta") for frame in frames[:2]) == ("hello ", "world") + assert len(frames) == 3 + assert data_lines[-1] == "[DONE]" + assert "upstream connection dropped" in frames[2]["error"]["message"] + assert "/srv/app/stt_stream.py" not in frames[2]["error"]["message"] + assert "stt.internal.corp" not in frames[2]["error"]["message"] + assert set(frames[2]["error"]) == {"message", "type", "param", "code"} + assert (frames[2]["error"]["type"], frames[2]["error"]["param"], frames[2]["error"]["code"]) == ( + "internal_server_error", + None, + "500", + ) + failure_hook.assert_awaited_once() + assert isinstance(failure_hook.await_args.kwargs["original_exception"], litellm.APIConnectionError) + + def _post_speech(client: TestClient) -> httpx.Response: return client.post("/v1/audio/speech", json={"model": "tts-1", "input": "Hi", "voice": "alloy"}) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d399adaea7d..29413d2b189 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -147,29 +147,20 @@ def test_get_litellm_model_cost_map_catalog_only_excludes_runtime_registered_ent ) -def test_get_litellm_model_cost_map_returns_cost_map(): - app = FastAPI() +def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None: + model_cost: Final = { + "audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"}, + "token-model": {"input_cost_per_token": 0.01, "mode": "chat"}, + } + monkeypatch.setattr(litellm, "model_cost", model_cost) + app: Final = FastAPI() app.include_router(router) - client = TestClient(app) + client: Final = TestClient(app) - response = client.get("/public/litellm_model_cost_map") + response: Final = client.get("/public/litellm_model_cost_map") assert response.status_code == 200 - payload = response.json() - assert isinstance(payload, dict) - assert len(payload) > 0, "Expected model cost map to contain at least one model" - - # Verify the structure contains expected keys for at least one model - # Check for a common model like gpt-4 or gpt-3.5-turbo - model_keys = list(payload.keys()) - assert len(model_keys) > 0 - - # Verify at least one model has expected cost fields - sample_model = model_keys[0] - sample_model_data = payload[sample_model] - assert isinstance(sample_model_data, dict) - # Check for common cost fields that should be present - assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data + assert response.json() == model_cost def test_public_ai_hub_info_is_public_by_default(monkeypatch): diff --git a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py index f5c97142dde..ab8dd8567b6 100644 --- a/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py +++ b/tests/unit/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -4,18 +4,23 @@ Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: - POST /v1/realtime/calls """ +import functools import json import time -from collections.abc import Awaitable -from typing import Protocol +from collections.abc import Awaitable, Mapping +from typing import Final, Protocol from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx from fastapi import FastAPI, HTTPException from fastapi.testclient import TestClient +from starlette.routing import Match - +import litellm +from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE +from litellm.proxy._lazy_features import LAZY_FEATURES, attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.encrypt_decrypt_utils import ( @@ -23,8 +28,18 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( encrypt_value_helper, ) from litellm.proxy.realtime_endpoints.endpoints import ( + _ALLOWED_SESSION_TYPES, + _coerce_realtime_session_type, _decode_realtime_token_payload, _encode_realtime_token_payload, + _prepare_client_secret_session, +) +from litellm.types.realtime import ( + RealtimeAudioInputConfig, + RealtimeAudioTranscriptionConfig, + RealtimeClientSecretRequest, + RealtimeSessionAudioConfig, + RealtimeSessionConfig, ) # --- Unit tests: token encode/decode helpers --- @@ -126,9 +141,7 @@ def mock_route_request_client_secrets(): future_expires_at = int(time.time()) + 3600 mock_resp = MagicMock(spec=httpx.Response) mock_resp.status_code = 200 - mock_resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() mock_resp.headers = {} mock_resp.json.return_value = { @@ -215,9 +228,7 @@ async def test_client_secrets_success_with_mock( mock_pre_call_hook, ): """POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream.""" - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -275,13 +286,7 @@ async def test_client_secrets_transcription_rejects_disallowed_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -312,12 +317,8 @@ async def test_client_secrets_transcription_routes_on_nested_model( async def _inner(): resp = MagicMock(spec=httpx.Response) resp.status_code = 200 - resp.text = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ) - resp.content = ( - f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' - ).encode() + resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + resp.content = (f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}').encode() resp.headers = {} resp.json.return_value = { "value": "upstream_ephemeral_key", @@ -351,13 +352,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( "session": { "type": "transcription", "model": "gpt-4o-realtime-preview", - "audio": { - "input": { - "transcription": { - "model": "gpt-realtime-whisper" - } - } - }, + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, }, }, ) @@ -367,10 +362,7 @@ async def test_client_secrets_transcription_routes_on_nested_model( session = captured["data"]["session"] assert session["type"] == "transcription" assert "model" not in session - assert ( - session["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert session["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" encrypted_value = response.json()["value"] decoded = _decode_realtime_token_payload( decrypt_value_helper( @@ -530,10 +522,7 @@ async def test_realtime_calls_replays_transcription_session_type( ) assert captured["session"]["type"] == "transcription" - assert ( - captured["session"]["audio"]["input"]["transcription"]["model"] - == "gpt-realtime-whisper" - ) + assert captured["session"]["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" # --- transcription_sessions endpoint --- @@ -587,8 +576,12 @@ def test_transcription_sessions_requires_auth(proxy_app): @pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, + {"audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, +]) async def test_transcription_sessions_rejects_disallowed_resolved_model( - proxy_app, + proxy_app, body, ): proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( user_id="test-user", @@ -605,9 +598,7 @@ async def test_transcription_sessions_rejects_disallowed_resolved_model( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json=body, ) assert response.status_code == 403 @@ -651,9 +642,7 @@ async def test_transcription_sessions_rejects_disallowed_team_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -696,9 +685,7 @@ async def test_transcription_sessions_rejects_disallowed_project_model_scope( response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -751,9 +738,7 @@ async def test_transcription_sessions_rejects_disallowed_team_member_model_scope response = client.post( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, - json={ - "input_audio_transcription": {"model": "gpt-realtime-whisper"} - }, + json={"input_audio_transcription": {"model": "gpt-realtime-whisper"}}, ) assert response.status_code == 403 @@ -947,13 +932,12 @@ async def test_transcription_sessions_encrypts_client_secret( POST /v1/realtime/transcription_sessions returns 200 and the ephemeral key under client_secret.value must be encrypted (never the raw upstream key). """ - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") captured_route_type = {} async def _capturing_route(*args, **kwargs): captured_route_type["route_type"] = kwargs.get("route_type") + captured_route_type["session"] = kwargs["data"]["transcription_session"] return await mock_route_request_transcription_sessions(*args, **kwargs) try: @@ -976,6 +960,8 @@ async def test_transcription_sessions_encrypts_client_secret( "/v1/realtime/transcription_sessions", headers={"Authorization": "Bearer sk-test-master-key"}, json={ + "model": "gpt-realtime-whisper", + "audio": {"input": {"transcription": {"model": "gpt-live-transcribe", "language": "en"}}}, "input_audio_format": "pcm16", "input_audio_transcription": {"model": "gpt-realtime-whisper"}, }, @@ -993,16 +979,61 @@ async def test_transcription_sessions_encrypts_client_secret( assert decrypted is not None assert "upstream_ephemeral_key" in decrypted # Routed through the dedicated transcription_sessions route type. - assert ( - captured_route_type["route_type"] - == "acreate_realtime_transcription_session" - ) + assert captured_route_type["route_type"] == "acreate_realtime_transcription_session" + assert captured_route_type["session"]["input_audio_transcription"]["model"] == "gpt-realtime-whisper" + assert captured_route_type["session"]["audio"]["input"]["transcription"] == { + "model": "gpt-realtime-whisper", "language": "en", + } + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("body", [ + {"model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"input_audio_transcription": {"model": "gpt-4o-realtime-preview"}, "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}}, + {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "gpt-realtime-whisper"}}, +]) +async def test_transcription_sessions_route_the_allowed_model_over_a_nested_model_the_key_cannot_call( + proxy_app, + mock_route_request_transcription_sessions, + mock_add_litellm_data, + mock_pre_call_hook, + body, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", models=["gpt-4o-realtime-preview"] + ) + route: Final = AsyncMock(side_effect=mock_route_request_transcription_sessions) + + try: + client = TestClient(proxy_app) + with ( + patch("litellm.proxy.proxy_server.route_request", route), + patch("litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data), + patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_logging, + ): + mock_logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/transcription_sessions", + headers={"Authorization": "Bearer sk-test-master-key"}, + json=body, + ) + + assert response.status_code == 200, response.text + route.assert_awaited_once() + routed_session: Final = route.await_args.kwargs["data"]["transcription_session"] + assert { + routed_session.get("input_audio_transcription", {}).get("model"), + routed_session.get("audio", {}).get("input", {}).get("transcription", {}).get("model"), + } - {None} == {"gpt-4o-realtime-preview"} finally: proxy_app.dependency_overrides.pop(user_api_key_auth, None) def test_session_type_coerced_for_unknown_value(): - """An unrecognized session_type in the token falls back to 'realtime'.""" payload = _encode_realtime_token_payload( ephemeral_key="epk", model_id="gpt-4o", @@ -1011,12 +1042,327 @@ def test_session_type_coerced_for_unknown_value(): expires_at=None, session_type="INJECTED_TYPE", ) - # Force-deserialize and check the coercion that happens in proxy_realtime_calls. decoded = json.loads(payload) - session_type = decoded.get("session_type") or "realtime" - if session_type not in ("realtime", "transcription"): - session_type = "realtime" - assert session_type == "realtime" + assert decoded["session_type"] == "INJECTED_TYPE" + assert _coerce_realtime_session_type("INJECTED_TYPE") == "realtime" + assert _coerce_realtime_session_type(None) == "realtime" + for allowed_session_type in _ALLOWED_SESSION_TYPES: + assert _coerce_realtime_session_type(allowed_session_type) == allowed_session_type + + +@pytest.mark.asyncio +async def test_translation_client_secret_rejects_disallowed_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-live-transcribe"), + ) + ), + ), + ) + + with pytest.raises(Exception, match=r"gpt-live-transcribe.*not available for this API key"): + await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + +@pytest.mark.asyncio +async def test_translation_client_secret_binds_authorized_nested_transcription_model() -> None: + req = RealtimeClientSecretRequest( + model="gpt-realtime-translate", + session=RealtimeSessionConfig( + type="translation", + model="gpt-realtime-translate", + audio=RealtimeSessionAudioConfig( + input=RealtimeAudioInputConfig( + transcription=RealtimeAudioTranscriptionConfig(model="gpt-realtime-whisper"), + ) + ), + ), + ) + + model, session_data, session_type = await _prepare_client_secret_session( + req=req, + user_api_key_dict=UserAPIKeyAuth(models=["gpt-realtime-translate", "gpt-realtime-whisper"]), + llm_model_list=None, + llm_router=None, + forced_session_type="translation", + ) + + assert model == "gpt-realtime-translate" + assert session_type == "translation" + assert session_data is not None + assert session_data["model"] == "gpt-realtime-translate" + assert session_data["audio"]["input"]["transcription"]["model"] == "gpt-realtime-whisper" + + +@pytest.mark.parametrize( + "path", + ( + "/openai/v1/realtime/client_secrets", + "/openai/v1/realtime/calls", + "/openai/v1/realtime/translations/client_secrets", + "/openai/v1/realtime/translations/calls", + ), +) +def test_openai_prefixed_realtime_http_paths_stay_with_the_openai_pass_through( + monkeypatch: pytest.MonkeyPatch, path: str +) -> None: + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import openai_proxy_route + + monkeypatch.setenv("LITELLM_DISABLE_LAZY_ROUTES", "true") + features = tuple(feature for feature in LAZY_FEATURES if feature.name in ("llm_passthrough", "realtime")) + app = FastAPI() + reserve_lazy_slot(app, "llm_passthrough", features=features) + attach_lazy_features(app, features) + scope = {"type": "http", "method": "POST", "path": path, "root_path": "", "headers": [], "query_string": b""} + + with TestClient(app): + resolved = next(route for route in app.router.routes if route.matches(scope)[0] is Match.FULL) + + assert getattr(resolved, "endpoint", None) is openai_proxy_route + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/client_secrets", + "/realtime/translations/client_secrets", + ], +) +def test_translation_client_secret_aliases_bind_token_family( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_client_secrets(*args, **kwargs) + + try: + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + + assert response.status_code == 200 + assert captured["route_type"] == "acreate_realtime_translation_client_secret" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + decrypted = decrypt_value_helper( + response.json()["value"], + key="client_secret.value", + exception_type="debug", + ) + decoded = _decode_realtime_token_payload(decrypted or "") + assert decoded is not None + assert decoded["session_type"] == "translation" + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + +@pytest.mark.parametrize( + "path", + [ + "/v1/realtime/translations/calls", + "/realtime/translations/calls", + ], +) +def test_translation_calls_aliases_route_translation_session( + proxy_app, + mock_route_request_realtime_calls, + mock_add_litellm_data, + mock_pre_call_hook, + path, +): + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id="gpt-realtime-translate", + user_id="test-user", + team_id=None, + expires_at=int(time.time()) + 3600, + session_type="translation", + ) + ) + captured = {} + + async def capture_route(*args, **kwargs): + captured["route_type"] = kwargs["route_type"] + captured["data"] = kwargs["data"] + return await mock_route_request_realtime_calls(*args, **kwargs) + + client = TestClient(proxy_app) + with ( + patch( # test-quality-ok: endpoint test captures the proxy routing boundary + "litellm.proxy.proxy_server.route_request", side_effect=capture_route + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response = client.post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + + assert response.status_code == 201 + assert captured["route_type"] == "arealtime_translation_calls" + assert captured["data"]["session"] == { + "type": "translation", + "model": "gpt-realtime-translate", + } + + +@pytest.mark.parametrize( + ("disable_global_guardrails", "expected_status", "expected_body"), + [(True, 201, "v=0"), (False, 400, TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE)], +) +def test_translation_calls_honor_the_minting_keys_guardrail_opt_out( + proxy_app: FastAPI, + mock_route_request_client_secrets: object, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + monkeypatch: pytest.MonkeyPatch, + disable_global_guardrails: bool, + expected_status: int, + expected_body: str, +) -> None: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy import proxy_server + from litellm.types.guardrails import GuardrailEventHooks + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "gpt-realtime-translate", + "litellm_params": {"model": "openai/gpt-realtime-translate", "api_key": "sk-upstream"}, + } + ] + ), + ) + monkeypatch.setattr( + litellm, + "callbacks", + [CustomGuardrail(guardrail_name="global-pre-call", event_hook=GuardrailEventHooks.pre_call, default_on=True)], + ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_id="test-user", + models=["gpt-realtime-translate"], + metadata={"disable_global_guardrails": disable_global_guardrails}, + ) + client: Final = TestClient(proxy_app, raise_server_exceptions=False) + try: + with ( + patch( # test-quality-ok: endpoint test stands in for the upstream client_secrets call + "litellm.proxy.proxy_server.route_request", side_effect=mock_route_request_client_secrets + ), + patch( # test-quality-ok: endpoint test isolates request metadata enrichment while minting + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger while minting + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + minted: Final = client.post( + "/v1/realtime/translations/client_secrets", + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-realtime-translate"}, + ) + finally: + proxy_app.dependency_overrides.pop(user_api_key_auth, None) + + with respx.mock(assert_all_called=False) as upstream: + upstream.post("https://api.openai.com/v1/realtime/translations/calls").mock( + return_value=httpx.Response(201, content=b"v=0\r\n", headers={"content-type": "application/sdp"}) + ) + answered: Final = client.post( + "/v1/realtime/translations/calls", + headers={"Authorization": f"Bearer {minted.json()['value']}"}, + content=b"v=0\r\n", + ) + + assert minted.status_code == 200 + assert answered.status_code == expected_status + assert expected_body in answered.text + + +@pytest.mark.parametrize( + "session_type,path", + [ + ("realtime", "/v1/realtime/translations/calls"), + ("translation", "/v1/realtime/calls"), + ], +) +def test_realtime_calls_reject_cross_family_token(proxy_app, session_type, path): + model = "gpt-realtime-translate" if session_type == "translation" else "gpt-realtime-2" + token = encrypt_value_helper( + _encode_realtime_token_payload( + ephemeral_key="ek_test", + model_id=model, + user_id=None, + team_id=None, + expires_at=int(time.time()) + 3600, + session_type=session_type, + ) + ) + response = TestClient(proxy_app).post( + path, + headers={"Authorization": f"Bearer {token}"}, + content=b"v=0\r\n", + ) + assert response.status_code == 401 + assert response.json()["error"] == "Token is not valid for this Realtime endpoint" @pytest.mark.asyncio @@ -1142,9 +1488,7 @@ async def test_transcription_sessions_returns_upstream_error_verbatim( return _inner() - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user", team_id="test-team" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user", team_id="test-team") try: client = TestClient(proxy_app) with ( @@ -1184,9 +1528,7 @@ async def test_transcription_sessions_wraps_route_exception( async def _raise_http(*args, **kwargs): raise HTTPException(status_code=403, detail="Model not allowed") - proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( - user_id="test-user" - ) + proxy_app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="test-user") try: client = TestClient(proxy_app, raise_server_exceptions=False) with ( @@ -1287,3 +1629,260 @@ def test_transcription_sessions_rejection_answers_an_openai_typed_error( assert response.status_code == 400 assert (response.json()["error"]["type"], response.json()["error"]["param"]) == ("invalid_request_error", None) + + +_AZURE_REALTIME_DEPLOYMENTS: Final = ( + ("azure-realtime-alias", "azure/qa-realtime-deployment"), + ("azure-transcribe-alias", "azure/qa-whisper-deployment"), + ("azure-translate-alias", "azure/qa-translate-deployment"), +) + + +def _post_through_the_azure_deployments( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: object, + deployments: tuple[tuple[str, str], ...] = _AZURE_REALTIME_DEPLOYMENTS, +) -> tuple[httpx.Response, respx.Route]: + from litellm.proxy import proxy_server + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + model_list: Final = [ # mutable-ok: Router takes a list of deployments + { + "model_name": alias, + "litellm_params": { + "model": deployment, + "api_base": "https://realtime-qa.openai.azure.com", + "api_key": "azure-test-key", + }, + } + for alias, deployment in deployments + ] + monkeypatch.setattr(proxy_server, "llm_router", litellm.Router(model_list=model_list)) + monkeypatch.setattr(proxy_server, "llm_model_list", model_list) + upstream: Final = respx_mock.route(host="realtime-qa.openai.azure.com").mock( + return_value=httpx.Response(200, json={"value": "ek_azure", "expires_at": int(time.time()) + 3600}) + ) + monkeypatch.setitem( + proxy_app.dependency_overrides, + user_api_key_auth, + lambda: UserAPIKeyAuth(user_id="test-user", models=[alias for alias, _ in deployments]), + ) + with ( + patch( # test-quality-ok: endpoint test isolates request metadata enrichment + "litellm.proxy.proxy_server.add_litellm_data_to_request", side_effect=mock_add_litellm_data + ), + patch( # test-quality-ok: endpoint test isolates the process-wide proxy logger + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as logging, + ): + logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + logging.post_call_failure_hook = AsyncMock() + response: Final = TestClient(proxy_app).post( + path, headers={"Authorization": "Bearer sk-test-master-key"}, json=body + ) + return response, upstream + + +def _upstream_session_value(upstream: respx.Route, session_path: tuple[str, ...]) -> object: + return functools.reduce( + lambda node, key: node[key], session_path, json.loads(upstream.calls.last.request.content)["session"] + ) + + +@pytest.mark.parametrize( + ("path", "body", "upstream_model_path", "expected_model"), + ( + ( + "/v1/realtime/client_secrets", + {"session": {"type": "realtime", "model": "azure-realtime-alias"}}, + ("model",), + "qa-realtime-deployment", + ), + ( + "/v1/realtime/client_secrets", + {"model": "azure-realtime-alias"}, + ("model",), + "qa-realtime-deployment", + ), + ( + "/v1/realtime/client_secrets", + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}, + } + }, + ("audio", "input", "transcription", "model"), + "qa-whisper-deployment", + ), + ( + "/v1/realtime/translations/client_secrets", + {"model": "azure-translate-alias"}, + ("model",), + "qa-translate-deployment", + ), + ( + "/v1/realtime/translations/client_secrets", + { + "model": "azure-translate-alias", + "session": {"audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}}}}, + }, + ("audio", "input", "transcription", "model"), + "qa-whisper-deployment", + ), + ), +) +def test_client_secrets_mint_a_secret_for_the_azure_deployment_model( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: dict[str, object], + upstream_model_path: tuple[str, ...], + expected_model: str, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body + ) + + assert response.status_code == 200, response.text + assert _upstream_session_value(upstream, upstream_model_path) == expected_model + + +@pytest.mark.parametrize( + ("body", "session_path", "expected"), + ( + ( + { + "session": { + "type": "realtime", + "model": "azure-realtime-alias", + "audio": {"input": {"turn_detection": None}}, + } + }, + ("audio", "input", "turn_detection"), + None, + ), + ( + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias"}, "turn_detection": None}}, + } + }, + ("audio", "input", "turn_detection"), + None, + ), + ({"session": {"type": "conversation", "model": "azure-realtime-alias"}}, ("type",), "realtime"), + ( + { + "session": { + "type": "transcription", + "audio": {"input": {"transcription": {"model": "azure-transcribe-alias", "delay": "auto"}}}, + } + }, + ("audio", "input", "transcription", "delay"), + "auto", + ), + ), +) +def test_client_secrets_forward_every_session_value_the_caller_set( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + body: Mapping[str, object], + session_path: tuple[str, ...], + expected: object, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + body, + ) + + assert response.status_code == 200, response.text + assert _upstream_session_value(upstream, session_path) == expected + + +def test_client_secrets_without_a_model_let_the_upstream_pick_its_default_through_a_wildcard_deployment( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + {"session": {"type": "realtime"}}, + deployments=(("*", "openai/*"),), + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content)["session"] == {"type": "realtime"} + + +def test_client_secrets_send_the_deployment_model_in_the_flat_transcription_field( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, + mock_add_litellm_data, + mock_pre_call_hook, + respx_mock, + monkeypatch, + "/v1/realtime/client_secrets", + {"session": {"type": "transcription", "input_audio_transcription": {"model": "azure-transcribe-alias"}}}, + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls.last.request.content)["session"] == { + "type": "transcription", + "input_audio_transcription": {"model": "qa-whisper-deployment"}, + } + + +@pytest.mark.parametrize( + ("path", "body"), + ( + ("/v1/realtime/client_secrets", {"session": "not-a-session"}), + ("/v1/realtime/translations/client_secrets", {"session": {"audio": {"input": "not-an-input"}}}), + ("/v1/realtime/transcription_sessions", {"audio": {"input": {"transcription": "not-a-transcription"}}}), + ), +) +def test_realtime_session_routes_answer_400_for_a_malformed_body( + proxy_app: FastAPI, + mock_add_litellm_data: AddLitellmDataToRequest, + mock_pre_call_hook: PreCallHook, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + path: str, + body: Mapping[str, object], +) -> None: + response, upstream = _post_through_the_azure_deployments( + proxy_app, mock_add_litellm_data, mock_pre_call_hook, respx_mock, monkeypatch, path, body + ) + + assert (response.status_code, response.json()["error"]["type"]) == (400, "invalid_request_error") + assert upstream.called is False diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index a4bd47ccb74..c140641a71b 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -1,6 +1,7 @@ import sys from collections.abc import AsyncGenerator, Mapping from contextlib import asynccontextmanager +from dataclasses import replace from types import ModuleType from typing import Final @@ -443,3 +444,45 @@ def test_root_authorization_server_metadata_is_the_gateways_not_the_byok_connect assert root.json()["token_endpoint"] == "https://gateway.example.com/token" assert connector.status_code == 200 assert connector.json()["token_endpoint"] == "https://gateway.example.com/v1/mcp/oauth/token" + + +@pytest.mark.parametrize( + "load_order", + (("/realtime/unrouted", "/mcp/unrouted"), ("/mcp/unrouted", "/realtime/unrouted")), + ids=("realtime_first", "realtime_last"), +) +def test_realtime_and_mcp_features_keep_every_route_once_in_either_load_order( + monkeypatch: pytest.MonkeyPatch, load_order: tuple[str, str] +) -> None: + monkeypatch.delenv(FLAG, raising=False) + + async def mcp_mount() -> Mapping[str, str]: + return {"route": "mcp_mount"} + + mounted_app: Final = FastAPI() + mounted_app.add_api_route("/{path:path}", mcp_mount, methods=["GET"]) + mcp_module: Final = ModuleType("tests.unit.proxy.lazy_fixture_mcp_app") + mcp_module.app = mounted_app # pyright: ignore[reportAttributeAccessIssue] # fixture module built at test time + monkeypatch.setitem(sys.modules, mcp_module.__name__, mcp_module) + features: Final = tuple( + replace(feature, module_path=mcp_module.__name__) if feature.name == "mcp_app" else feature + for feature in LAZY_FEATURES + if feature.name in ("mcp_app", "realtime") + ) + + async def mcp_proxy() -> Mapping[str, str]: + return {"route": "mcp_proxy"} + + app: Final = FastAPI() + attach_lazy_features(app, features) + app.add_api_route("/mcp/proxy", mcp_proxy, methods=["GET"]) + + with TestClient(app) as client: + for path in load_order: + client.get(path) + response: Final = client.get("/mcp/proxy") + + assert loaded_lazy_modules(app) == {feature.module_path for feature in features} + assert response.json() == {"route": "mcp_proxy"} + route_ids: Final = tuple(id(route) for route in app.routes) + assert len(route_ids) == len(frozenset(route_ids)) diff --git a/tests/unit/proxy/test_proxy_server.py b/tests/unit/proxy/test_proxy_server.py index e4af36d99f2..a50db09128f 100644 --- a/tests/unit/proxy/test_proxy_server.py +++ b/tests/unit/proxy/test_proxy_server.py @@ -42,6 +42,7 @@ from fastapi import FastAPI, HTTPException, WebSocket # test /chat/completion request to the proxy from fastapi.testclient import TestClient +from starlette.testclient import WebSocketTestSession from litellm.integrations.custom_logger import CustomLogger from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -3661,6 +3662,91 @@ def test_live_session_pre_call_abort_releases_reservations( route_call.assert_not_awaited() +def _receive_until_closed(websocket: WebSocketTestSession) -> None: + while True: + websocket.receive_text() + + +@pytest.mark.parametrize( + ("litellm_params", "expected_upstream_url"), + ( + pytest.param( + {"model": "openai/gateway-transcribe", "api_base": "https://gateway.example/v1"}, + "wss://gateway.example/v1/realtime?model=gateway-transcribe&intent=transcription", + id="gateway", + ), + pytest.param( + {"model": "xai/grok-voice", "api_base": "https://api.x.ai/v1"}, + "wss://api.x.ai/v1/realtime?model=grok-voice&intent=transcription", + id="xai", + ), + pytest.param( + {"model": "openai/gpt-live-transcribe", "api_base": "https://api.openai.com/v1"}, + "wss://api.openai.com/v1/realtime?intent=transcription", + id="openai", + ), + ), +) +def test_realtime_transcription_websocket_keeps_the_model_upstream_except_on_openai_hosts( + client_no_auth: TestClient, + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, str], + expected_upstream_url: str, +) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + + monkeypatch.setattr( + proxy_server, + "llm_router", + litellm.Router( + model_list=[{"model_name": "team-transcriber", "litellm_params": {**litellm_params, "api_key": "fake"}}] + ), + ) + connect: Final = MagicMock(side_effect=RuntimeError("upstream url captured")) + monkeypatch.setattr("websockets.connect", connect) + + with ( + client_no_auth.websocket_connect( + "/v1/realtime?model=team-transcriber&intent=transcription", + headers={"Authorization": "Bearer test-api-key"}, + ) as websocket, + pytest.raises(WebSocketDisconnect), + ): + _receive_until_closed(websocket) + + assert connect.call_args.args[0] == expected_upstream_url + + +def test_openai_prefixed_translation_websocket_takes_the_openai_websocket_pass_through( + client_no_auth: TestClient, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.websockets import WebSocketDisconnect + + from litellm.proxy import proxy_server + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import _OPENAI_WS_DISABLED_REFUSAL + + route_call: Final = AsyncMock() + monkeypatch.setattr(proxy_server, "route_request", route_call) + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setitem(proxy_server.app.dependency_overrides, user_api_key_auth_websocket, lambda: UserAPIKeyAuth()) + + with client_no_auth.websocket_connect( + "/openai/v1/realtime/translations?model=gpt-realtime-translate", + headers={"Authorization": "Bearer test-api-key"}, + ) as websocket: + refusal: Final = websocket.receive_json() + with pytest.raises(WebSocketDisconnect) as closed: + websocket.receive_json() + + assert refusal["error"]["message"] == _OPENAI_WS_DISABLED_REFUSAL.message + assert closed.value.code == 1008 + route_call.assert_not_awaited() + + @pytest.mark.parametrize("path", ["/openai/v1/live/sessions", "/v1/live/sessions", "/live/sessions"]) def test_live_session_invalid_start_is_rejected_without_routing( path: str, diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 487638d6919..82db8820447 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -30,6 +30,7 @@ from fastapi import FastAPI, HTTPException, Request from fastapi.encoders import jsonable_encoder from fastapi.staticfiles import StaticFiles from fastapi.testclient import TestClient +from starlette.datastructures import URL import litellm import litellm.proxy.proxy_server as proxy_server_module @@ -11117,7 +11118,7 @@ async def test_get_current_spend_redis_error_falls_back_to_in_memory(): def test_realtime_websocket_route_aliases_registered(): - """Realtime sessions reach the proxy via three path aliases stacked on + """Realtime sessions reach the proxy via five path aliases stacked on `realtime_websocket_endpoint`. Dropping any of them silently 405s WebSocket upgrades because the catch-all `/openai/{endpoint:path}` HTTP passthrough only declares HTTP methods. The aliases must also be @@ -11133,7 +11134,13 @@ def test_realtime_websocket_route_aliases_registered(): websocket_paths = {route.path for route in app.routes if isinstance(route, WebSocketRoute)} openai_routes = LiteLLMRoutes.openai_routes.value - for expected in ("/openai/v1/realtime", "/v1/realtime", "/realtime"): + for expected in ( + "/openai/v1/realtime", + "/v1/realtime", + "/realtime", + "/v1/realtime/translations", + "/realtime/translations", + ): assert expected in websocket_paths, ( f"{expected!r} missing from registered WebSocket routes; the " f"realtime endpoint will 405 for clients hitting this path." @@ -11152,7 +11159,7 @@ def _lit6973_fake_realtime_ws() -> MagicMock: ws = MagicMock() ws.headers = {} ws.scope = {"headers": [], "type": "websocket"} - ws.url = "ws://testserver/v1/realtime" + ws.url = URL("ws://testserver/v1/realtime") ws.accept = AsyncMock() ws.send_text = AsyncMock() ws.close = AsyncMock() diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 46777b32319..ad3036560a3 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1,28 +1,40 @@ import asyncio +import functools import json import time from collections.abc import Awaitable, Callable, Mapping +from datetime import datetime, timezone from pathlib import Path from types import TracebackType -from typing import Final, Protocol +from typing import ClassVar, Final, Literal, Protocol from unittest.mock import AsyncMock, MagicMock, patch import httpx import openai import pytest import respx -from websockets.exceptions import ConnectionClosedError +from openai import AsyncOpenAI +from websockets.exceptions import ConnectionClosed, ConnectionClosedError from websockets.frames import Close import litellm +from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.litellm_core_utils.realtime_streaming import TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE, RealTimeStreaming +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.openai.workload_identity import _workload_identity_auth from litellm.models.credentials import CredentialItem +from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy +from litellm.proxy.litellm_pre_call_utils import _apply_resolved_guardrails_to_metadata from litellm.realtime_api import main as realtime_main -from litellm.realtime_api.main import _with_resolved_session_model +from litellm.realtime_api.main import with_resolved_session_model from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.proxy.policy_engine import Policy, PolicyCondition, PolicyMatchContext +from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep +from litellm.types.realtime import RealtimeQueryParams @pytest.fixture @@ -48,19 +60,19 @@ class _CallCapture: def test_resolves_top_level_session_model(): - resolved = _with_resolved_session_model({"model": "alias/gpt-realtime"}, "gpt-realtime") + resolved = with_resolved_session_model({"model": "alias/gpt-realtime"}, "gpt-realtime") assert resolved == {"model": "gpt-realtime"} def test_session_without_model_is_returned_unchanged(): session = {"type": "realtime", "audio": {"input": {}}} - assert _with_resolved_session_model(session, "gpt-realtime") == session + assert with_resolved_session_model(session, "gpt-realtime") == session def test_does_not_clobber_flat_transcription_model(): """The nested transcription model is a different model than the realtime conversation model and must not be overwritten with the routing model.""" - resolved = _with_resolved_session_model( + resolved = with_resolved_session_model( {"model": "gpt-4o-realtime-preview", "input_audio_transcription": {"model": "whisper-1"}}, "gpt-4o-realtime-preview", ) @@ -68,7 +80,7 @@ def test_does_not_clobber_flat_transcription_model(): def test_does_not_clobber_nested_audio_transcription_model(): - resolved = _with_resolved_session_model( + resolved = with_resolved_session_model( { "model": "gpt-4o-realtime-preview", "audio": {"input": {"transcription": {"model": "whisper-1"}}}, @@ -80,7 +92,7 @@ def test_does_not_clobber_nested_audio_transcription_model(): def test_original_session_is_not_mutated(): session = {"model": "alias/gpt-realtime"} - _with_resolved_session_model(session, "gpt-realtime") + with_resolved_session_model(session, "gpt-realtime") assert session == {"model": "alias/gpt-realtime"} @@ -388,6 +400,52 @@ async def test_arealtime_live_session_rejects_streaming_only_pre_call_guardrail( assert live_handler.called is False +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_metadata", + ( + {"user_api_key_metadata": {"disable_global_guardrails": True}}, + {"user_api_key_team_metadata": {"opted_out_global_guardrails": ["default-on-blocker"]}}, + ), +) +async def test_arealtime_key_metadata_cannot_opt_out_of_a_default_on_realtime_guardrail( + monkeypatch: pytest.MonkeyPatch, key_metadata: dict[str, object] +) -> None: + class FakeLiveHandler: + def __init__(self) -> None: + self.called = False + + async def async_live_session(self, **kwargs: object) -> None: + self.called = True + + live_handler: Final = FakeLiveHandler() + monkeypatch.setattr(realtime_main, "openai_live_sessions", live_handler) + monkeypatch.setattr( + litellm, + "callbacks", + [ + CustomGuardrail( + guardrail_name="default-on-blocker", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + ) + ], + ) + + with pytest.raises(ValueError, match="Guardrails are not supported on OpenAI Live sessions"): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-live-1", + websocket=MagicMock(), + api_key="fake-key", + api_base="https://live-gateway.example/v1", + litellm_logging_obj=FakeLogging(), + metadata=key_metadata, + live_session_start={"type": "session.start", "session": {"model": "gpt-live-1"}}, + ) + + assert live_handler.called is False + + @pytest.mark.asyncio async def test_arealtime_live_session_rejects_non_openai_provider(monkeypatch: pytest.MonkeyPatch) -> None: def mock_get_llm_provider( @@ -986,6 +1044,114 @@ async def test_realtime_http_routes_send_a_bare_model_name_to_its_explicit_provi assert gateway.url == f"https://realtime-gateway.example/v1/realtime/{path}" +class _GatewayThatRecordsTheBody: + body: object = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.body = json.loads(request.content) + return httpx.Response(200, json={"client_secret": {"value": "ek_test"}}) + + +@pytest.mark.asyncio +async def test_transcription_session_sends_the_deployment_model_upstream_in_every_transcription_shape() -> None: + gateway: Final = _GatewayThatRecordsTheBody() + await realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session={ + "audio": {"input": {"transcription": {"model": "team-transcriber", "language": "en"}}}, + "input_audio_transcription": {"model": "team-transcriber"}, + "input_audio_format": "pcm16", + }, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + assert gateway.body == { + "audio": {"input": {"transcription": {"model": "gpt-transcribe", "language": "en"}}}, + "input_audio_transcription": {"model": "gpt-transcribe"}, + "input_audio_format": "pcm16", + } + + +def _create_realtime_session_with_an_explicit_null(route: str, client: AsyncHTTPHandler) -> Awaitable[httpx.Response]: + if route == "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + session={ + "type": "realtime", + "model": "gpt-realtime", + "instructions": None, + "tools": None, + "audio": {"input": {"turn_detection": None}}, + }, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=client, + ) + if route == "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + session={"include": None, "audio": {"input": {"noise_reduction": None}}}, + custom_llm_provider="openai", + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=_realtime_http_logging(route), + client=client, + ) + return realtime_main.acreate_realtime_transcription_session.__wrapped__( + model="openai/gpt-transcribe", + transcription_session={"include": None, "audio": {"input": {"turn_detection": None}}}, + api_base="https://realtime-gateway.example/v1", + api_key="fake-key", + litellm_logging_obj=FakeLogging(), + client=client, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "body_path"), + ( + ("client_secrets", ("session", "audio", "input", "turn_detection")), + ("translation_client_secrets", ("session", "audio", "input", "noise_reduction")), + ("transcription_sessions", ("audio", "input", "turn_detection")), + ), +) +async def test_realtime_session_requests_forward_an_explicit_null_upstream( + route: str, body_path: tuple[str, ...] +) -> None: + gateway: Final = _GatewayThatRecordsTheBody() + + await _create_realtime_session_with_an_explicit_null( + route, AsyncHTTPHandler(transport=httpx.MockTransport(gateway)) + ) + + parent: Final = functools.reduce(lambda node, key: node[key], body_path[:-1], gateway.body) + assert body_path[-1] in parent + assert parent[body_path[-1]] is None + session_fields: Final = gateway.body["session"] if body_path[0] == "session" else gateway.body + assert [key for key, value in session_fields.items() if value is None] == [] + + +@pytest.mark.asyncio +async def test_transcription_client_secret_sends_the_deployment_model_as_the_session_model() -> None: + gateway: Final = _GatewayThatRecordsTheSession() + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + session={"type": "transcription", "model": "openai/gpt-live-transcribe"}, + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=_realtime_http_logging("client_secrets"), + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-live-transcribe" + assert gateway.session["audio"] == {"input": {"transcription": {"model": "gpt-live-transcribe"}}} + + class _ClientWebSocketWithHeaders: def __init__(self, headers: tuple[tuple[bytes, bytes], ...]) -> None: self.scope: Final = {"headers": headers} @@ -995,11 +1161,51 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=()) _BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),)) +_GA_DEFAULT_SESSIONS: Final = ( + pytest.param("gpt-realtime-2.1", "realtime", None, id="ga-only-model"), + pytest.param("gpt-realtime-2", "realtime", None, id="ga-only-alias"), + pytest.param("gpt-realtime-2-2026-05-06", "realtime", None, id="ga-only-snapshot"), + # Azure answers beta sessions on these with "The Realtime preview API is no longer available" + # (https://aka.ms/aoai/realtime-ga), checked live 2026-10-28 + pytest.param("gpt-realtime", "realtime", None, id="beta-retired-gpt-realtime"), + pytest.param("gpt-realtime-mini", "realtime", None, id="beta-retired-gpt-realtime-mini"), + pytest.param("gpt-realtime-1.5", "realtime", None, id="beta-retired-gpt-realtime-1.5"), + pytest.param("my-transcribe-deployment", "realtime", {"intent": "transcription"}, id="transcription-intent"), + pytest.param("my-translate-deployment", "translation", None, id="translation-mode"), +) + + +@pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) +def test_azure_realtime_protocol_defaults_to_ga_when_none_is_configured( + local_model_cost_map, model: str, realtime_mode: str, query_params: RealtimeQueryParams | None +) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + None, model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_BETA_CLIENT + ) + + assert protocol.name == "GA" + + +@pytest.mark.parametrize(("model", "realtime_mode", "query_params"), _GA_DEFAULT_SESSIONS) +def test_azure_realtime_protocol_configured_by_the_operator_wins( + local_model_cost_map, model: str, realtime_mode: str, query_params: RealtimeQueryParams | None +) -> None: + from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client + + protocol: Final = azure_realtime_protocol_for_client( + "beta", model=model, realtime_mode=realtime_mode, query_params=query_params, websocket=_GA_CLIENT + ) + + assert protocol.name == "beta" + + async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None: connect: Final = _ConnectThatStopsAfterCapturingTheUrl() with patch("websockets.connect", connect): await realtime_main._arealtime.__wrapped__( - model="azure/gpt-realtime", + model="azure/my-realtime-deployment", websocket=websocket, api_base="https://my-endpoint.openai.azure.com", api_key="fake-key", @@ -1014,7 +1220,7 @@ async def test_arealtime_azure_ga_client_without_beta_header_dials_the_ga_upstre monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) assert ( await _azure_backend_url_dialed_for(_GA_CLIENT) - == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-realtime" + == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment" ) @@ -1022,7 +1228,7 @@ async def test_arealtime_azure_ga_client_without_beta_header_dials_the_ga_upstre async def test_arealtime_azure_beta_header_client_keeps_the_beta_upstream(monkeypatch): monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) assert await _azure_backend_url_dialed_for(_BETA_CLIENT) == ( - "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" ) @@ -1030,7 +1236,7 @@ async def test_arealtime_azure_beta_header_client_keeps_the_beta_upstream(monkey async def test_arealtime_azure_explicit_beta_protocol_wins_over_a_ga_client(monkeypatch): monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) assert await _azure_backend_url_dialed_for(_GA_CLIENT, realtime_protocol="beta") == ( - "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" ) @@ -1038,7 +1244,197 @@ async def test_arealtime_azure_explicit_beta_protocol_wins_over_a_ga_client(monk async def test_arealtime_azure_env_beta_protocol_wins_over_a_ga_client(monkeypatch): monkeypatch.setenv("LITELLM_AZURE_REALTIME_PROTOCOL", "beta") assert await _azure_backend_url_dialed_for(_GA_CLIENT) == ( - "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=gpt-realtime" + "wss://my-endpoint.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + ) + + +class _BetaClientThatStaysConnected: + def __init__(self) -> None: + self.scope: Final = {"headers": ((b"openai-beta", b"realtime=v1"),)} + self.sent: tuple[str, ...] = () + + async def receive_text(self) -> str: + return await asyncio.get_running_loop().create_future() + + async def send_text(self, data: str) -> None: + self.sent = (*self.sent, data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + return None + + +class _ConnectToABackendThatSendsOneFrame: + def __init__(self, frame: Mapping[str, object]) -> None: + self.url: str | None = None + self._backend: Final = MagicMock(recv=AsyncMock(side_effect=[json.dumps(frame), ConnectionClosed(None, None)])) + + def __call__( + self, url: str, *, additional_headers: Mapping[str, str], max_size: int, ssl: object + ) -> "_ConnectToABackendThatSendsOneFrame": + self.url = url + return self + + async def __aenter__(self) -> MagicMock: + return self._backend + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> None: + return None + + +def _realtime_websocket_logging(*failure_callbacks: CustomLogger) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-realtime-websocket", + function_id="test-realtime-websocket", + dynamic_async_failure_callbacks=list(failure_callbacks), + ) + + +_GA_SESSION_CREATED: Final = { + "type": "session.created", + "event_id": "event_1", + "session": { + "type": "realtime", + "id": "sess_1", + "model": "my-realtime-deployment", + "output_modalities": ["text"], + "audio": {"output": {"voice": "marin"}}, + }, +} + + +async def _azure_session_created_seen_by_a_beta_client( + session_kwargs: Mapping[str, object], +) -> tuple[str | None, object]: + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="azure/my-realtime-deployment", + websocket=client, + api_base="https://my-endpoint.openai.azure.com", + api_key="fake-key", + litellm_logging_obj=_realtime_websocket_logging(), + **session_kwargs, + ) + return connect.url, json.loads(client.sent[0]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("session_kwargs", "expected_url"), + ( + pytest.param( + {"query_params": {"intent": "transcription"}}, + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?intent=transcription", + id="transcription-intent", + ), + ), +) +async def test_arealtime_azure_beta_client_on_a_ga_session_it_did_not_need_bridging_for_keeps_ga_session_events( + monkeypatch: pytest.MonkeyPatch, session_kwargs: Mapping[str, object], expected_url: str +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + + assert await _azure_session_created_seen_by_a_beta_client(session_kwargs) == (expected_url, _GA_SESSION_CREATED) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("env_protocol", "session_kwargs"), + ( + pytest.param(None, {"realtime_protocol": "GA"}, id="operator-set-ga"), + pytest.param("GA", {}, id="env-set-ga"), + ), +) +async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_operator_pins_ga( + monkeypatch: pytest.MonkeyPatch, env_protocol: str | None, session_kwargs: Mapping[str, object] +) -> None: + if env_protocol is None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + else: + monkeypatch.setenv("LITELLM_AZURE_REALTIME_PROTOCOL", env_protocol) + + assert await _azure_session_created_seen_by_a_beta_client(session_kwargs) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "base_model_kwargs", + ( + pytest.param({"base_model": "azure/gpt-realtime-2.1"}, id="base-model-param"), + pytest.param({"metadata": {"model_info": {"base_model": "azure/gpt-realtime-2.1"}}}, id="model-info"), + ), +) +async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_deployment_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, base_model_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + + assert await _azure_session_created_seen_by_a_beta_client(base_model_kwargs) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, + ) + + +_GA_SESSION_CREATED_BRIDGED_TO_BETA: Final = { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, +} + +_ROUTER_REQUEST_SHAPES: Final = ( + pytest.param({}, id="sdk-router-call"), + pytest.param({"metadata": {"user_api_key_hash": "hashed-key"}}, id="proxy-request-metadata"), +) + + +def _router_with_an_azure_deployment_aliasing(deployment_model: str, base_model: str) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "team-alias", + "litellm_params": { + "model": deployment_model, + "api_base": "https://my-endpoint.openai.azure.com", + "api_key": "fake-key", + }, + "model_info": {"base_model": base_model}, + } + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _ROUTER_REQUEST_SHAPES) +async def test_router_realtime_bridges_a_beta_client_to_ga_when_model_info_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + router: Final = _router_with_an_azure_deployment_aliasing( + "azure/my-realtime-deployment", "azure/gpt-realtime-2.1-mini" + ) + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + + with patch("websockets.connect", connect): + await router._arealtime(model="team-alias", websocket=client, **request_kwargs) + + assert (connect.url, json.loads(client.sent[0])) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, ) @@ -1130,6 +1526,30 @@ async def test_arealtime_openai_forwards_the_intent_query_param_to_the_upstream_ assert connect.url == "wss://api.openai.com/v1/realtime?model=gpt-realtime&intent=chat" +@pytest.mark.asyncio +@pytest.mark.parametrize("intent", (None, "transcription")) +async def test_arealtime_openai_connects_to_api_base_rather_than_the_callers_client_base_url( + intent: str | None, +) -> None: + connect: Final = _ConnectThatStopsAfterCapturingTheUrl() + query_params: Final[RealtimeQueryParams] = ( + {"model": "openai/gateway-realtime", "intent": intent} if intent else {"model": "openai/gateway-realtime"} + ) + with patch("websockets.connect", connect), patch("websockets.asyncio.client.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gateway-realtime", + websocket=_ClosableGaClientWebSocket(), + api_key="fake-key", + api_base="https://configured-gateway.example/", + client=AsyncOpenAI(api_key="caller-key", base_url="https://caller-client.example/v1"), + query_params=query_params, + litellm_logging_obj=FakeLogging(), + ) + + expected_query: Final = "model=gateway-realtime&intent=transcription" if intent else "model=gateway-realtime" + assert connect.url == f"wss://configured-gateway.example/v1/realtime?{expected_query}" + + @pytest.mark.parametrize( ("model", "api_base", "client_query_params", "expected_backend_url"), [ @@ -1416,26 +1836,82 @@ class _GatewayThatRecordsTheAuthorization: @pytest.mark.asyncio @pytest.mark.parametrize( - "route", - [realtime_main.acreate_realtime_client_secret, realtime_main.acreate_realtime_transcription_session], - ids=["client_secrets", "transcription_sessions"], + ("route", "model"), + [ + (realtime_main.acreate_realtime_client_secret, "gpt-realtime"), + (realtime_main.acreate_realtime_transcription_session, "gpt-realtime"), + (realtime_main.acreate_realtime_translation_client_secret, "gpt-realtime-translate"), + ], + ids=["client_secrets", "transcription_sessions", "translation_client_secrets"], ) @respx.mock async def test_openai_realtime_http_routes_send_the_federated_bearer_for_a_keyless_deployment( - federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]] + federated_deployment: dict[str, str], route: Callable[..., Awaitable[object]], model: str ) -> None: _mock_token_exchange() gateway: Final = _GatewayThatRecordsTheAuthorization() await route.__wrapped__( - model="gpt-realtime", + model=model, custom_llm_provider="openai", - litellm_logging_obj=FakeLogging(), + litellm_logging_obj=Logging( + model=model, + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-federated-realtime", + function_id="test-federated-realtime", + ), client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), **federated_deployment, ) assert gateway.authorization == "Bearer federated-bearer" +@pytest.mark.asyncio +@pytest.mark.parametrize("flow", ("client_secret", "calls")) +async def test_translation_http_routes_preserve_explicit_azure_provider(flow: str) -> None: + expected_path: Final = ( + "/openai/v1/realtime/translations/" + + { + "client_secret": "client_secrets", + "calls": "calls", + }[flow] + ) + + def send_response(request: httpx.Request) -> httpx.Response: + assert request.url.host == "azure-deployment.example" + assert request.url.path == expected_path + return httpx.Response(200, json={"value": "ek-test", "routed_path": request.url.path}) + + params: Final[Mapping[str, object]] = { + "model": "translation-deployment", + "custom_llm_provider": "azure", + "api_base": "https://azure-deployment.example/openai/v1/", + "api_key": "test-key", + "client": AsyncHTTPHandler(transport=httpx.MockTransport(send_response)), + "litellm_logging_obj": Logging( + model="translation-deployment", + messages=[], + stream=False, + call_type="acreate_realtime_translation_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id="test-translation-routing", + function_id="test-translation-routing", + ), + } + response: Final = await ( + realtime_main.acreate_realtime_translation_client_secret.__wrapped__(**params) + if flow == "client_secret" + else realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek-test", sdp_body=b"v=0", **params + ) + ) + + assert response.status_code == 200 + assert response.json() == {"value": "ek-test", "routed_path": expected_path} + + @pytest.mark.asyncio @respx.mock async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_keyless_deployment( @@ -1453,6 +1929,532 @@ async def test_openai_realtime_health_check_sends_the_federated_bearer_for_a_key assert connect.kwargs["additional_headers"] == {"Authorization": "Bearer federated-bearer"} +def _realtime_http_route_call( + route: str, + logging_obj: Logging, + custom_llm_provider: str, + api_base: str, + client: AsyncHTTPHandler | None = None, + metadata: Mapping[str, object] | None = None, +) -> Awaitable[httpx.Response]: + if route == "client_secrets": + return realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + if route == "translation_client_secrets": + return realtime_main.acreate_realtime_translation_client_secret.__wrapped__( + model="gpt-realtime-translate", + custom_llm_provider=custom_llm_provider, + api_key="sk-test", + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + metadata=metadata, + ) + if route == "calls": + return realtime_main.arealtime_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + ) + return realtime_main.arealtime_translation_calls.__wrapped__( + openai_ephemeral_key="ek_test", + sdp_body=b"v=0", + model="gpt-realtime-translate", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + client=client, + litellm_logging_obj=logging_obj, + metadata=metadata, + ) + + +def _realtime_http_logging(route: str) -> Logging: + return Logging( + model="gpt-realtime", + messages=[], + stream=False, + call_type="acreate_realtime_client_secret", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + litellm_call_id=f"test-realtime-{route}", + function_id=f"test-realtime-{route}", + ) + + +_OPENAI_REALTIME_HTTP_ROUTE_PATHS: Final = { + "client_secrets": "/v1/realtime/client_secrets", + "translation_client_secrets": "/v1/realtime/translations/client_secrets", + "calls": "/v1/realtime/calls", + "translation_calls": "/v1/realtime/translations/calls", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", tuple(_OPENAI_REALTIME_HTTP_ROUTE_PATHS)) +async def test_openai_realtime_http_routes_surface_the_upstream_error_body_verbatim(route: str) -> None: + upstream_error: Final = { + "error": { + "message": "Invalid model", + "type": "invalid_request_error", + "param": "session.model", + "code": "invalid_value", + } + } + + with pytest.raises(BaseLLMException) as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler( + transport=httpx.MockTransport(lambda request: httpx.Response(400, json=upstream_error)) + ), + ) + + assert raised.value.status_code == 400 + assert json.loads(raised.value.message) == upstream_error + + +class _GatewayThatRecordsTheSession: + session: Mapping[str, object] | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.session = json.loads(request.content)["session"] + return httpx.Response(200, json={"value": "ek_test"}) + + +@pytest.mark.asyncio +async def test_client_secret_keeps_the_realtime_session_model_when_input_transcription_names_another() -> None: + gateway: Final = _GatewayThatRecordsTheSession() + logging_obj: Final = _realtime_http_logging("client_secrets") + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + model="gpt-realtime", + session={ + "type": "realtime", + "model": "gpt-realtime", + "audio": {"input": {"transcription": {"model": "gpt-realtime-whisper"}}}, + }, + custom_llm_provider="openai", + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=logging_obj, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-realtime" + assert gateway.session["audio"] == {"input": {"transcription": {"model": "gpt-realtime-whisper"}}} + assert logging_obj.model == "gpt-realtime" + + +@pytest.mark.asyncio +async def test_client_secret_session_model_wins_over_a_provider_qualified_top_level_model( + local_model_cost_map: None, +) -> None: + gateway: Final = _GatewayThatRecordsTheSession() + logging_obj: Final = _realtime_http_logging("client_secrets") + + await realtime_main.acreate_realtime_client_secret.__wrapped__( + model="openai/gpt-realtime-2.1", + session={"type": "realtime", "model": "gpt-realtime-2.1-mini"}, + api_key="sk-test", + api_base="https://upstream.example", + litellm_logging_obj=logging_obj, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + ) + + assert gateway.session is not None + assert gateway.session["model"] == "gpt-realtime-2.1-mini" + assert logging_obj.model == "gpt-realtime-2.1-mini" + + +class _UpstreamThatMustNotBeCalled: + called: bool = False + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.called = True + return httpx.Response(200, json={"value": "ek_test"}) + + +class _RecordingClientWebSocket: + def __init__(self) -> None: + self.scope: Final = {"headers": ()} + self.sent: tuple[str, ...] = () + self.close_code: int | None = None + + async def send_text(self, data: str) -> None: + self.sent = (*self.sent, data) + + async def close(self, code: int = 1000, reason: str = "") -> None: + self.close_code = code + + +def _block_every_request_with_a_guardrail( + monkeypatch: pytest.MonkeyPatch, event_hook: GuardrailEventHooks = GuardrailEventHooks.pre_call +) -> None: + monkeypatch.setattr( + litellm, + "callbacks", + [CustomGuardrail(guardrail_name="translation-blocker", event_hook=event_hook, default_on=True)], + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_http_routes_refuse_a_guarded_request_before_any_upstream_call( + route: str, + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + upstream: Final = _UpstreamThatMustNotBeCalled() + + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + ) + + assert raised.value.status_code == 400 + assert upstream.called is False + + +_DEFAULT_ON_GUARDRAIL_OPT_OUTS: Final = ( + pytest.param({"user_api_key_metadata": {"disable_global_guardrails": True}}, id="key-disables-global"), + pytest.param( + {"user_api_key_team_metadata": {"opted_out_global_guardrails": ["translation-blocker"]}}, id="team-opts-out" + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "path"), + (("translation_client_secrets", "client_secrets"), ("translation_calls", "calls")), +) +@pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_http_routes_serve_a_key_opted_out_of_the_default_on_guardrail( + route: str, + path: str, + key_metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + gateway: Final = _GatewayThatRecordsTheUrl() + + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + metadata=key_metadata, + ) + + assert gateway.url == f"https://upstream.example/v1/realtime/translations/{path}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls")) +async def test_translation_http_routes_refuse_a_provider_other_than_openai_and_azure_before_any_upstream_call( + route: str, +) -> None: + upstream: Final = _UpstreamThatMustNotBeCalled() + + with pytest.raises(litellm.BadRequestError, match="supports only the azure, openai providers, got xai") as raised: + await _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="xai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + ) + + assert raised.value.status_code == 400 + assert upstream.called is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_websocket_closes_1008_for_a_guarded_request_before_the_upstream_connect( + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + ) + + assert connect.url is None + assert websocket.close_code == 1008 + assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ + TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("key_metadata", _DEFAULT_ON_GUARDRAIL_OPT_OUTS) +@pytest.mark.parametrize("event_hook", (GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call)) +async def test_translation_websocket_opens_for_a_key_opted_out_of_the_default_on_guardrail( + key_metadata: Mapping[str, object], + monkeypatch: pytest.MonkeyPatch, + event_hook: GuardrailEventHooks, +) -> None: + _block_every_request_with_a_guardrail(monkeypatch, event_hook) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + metadata=key_metadata, + ) + + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + + +class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[object, ...] = () + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.failures = (*self.failures, kwargs.get("exception")) + + +@pytest.mark.asyncio +async def test_translation_websocket_refusal_logs_the_session_as_a_failure(monkeypatch: pytest.MonkeyPatch) -> None: + _block_every_request_with_a_guardrail(monkeypatch) + recorder: Final = _FailureRecorder() + logging_obj: Final = _realtime_websocket_logging(recorder) + + with patch("websockets.connect", _CapturingConnect()): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=_RecordingClientWebSocket(), + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=logging_obj, + ) + + assert [(type(failure), getattr(failure, "status_code", None)) for failure in recorder.failures] == [ + (litellm.BadRequestError, 400) + ] + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in str(recorder.failures[0]) + assert logging_obj.model_call_details[REALTIME_SESSION_FAILURE_LOGGED_KEY] is True + + +class _NativeLifecycleGuardrail(CustomGuardrail): + use_native_lifecycle_hooks: ClassVar[bool] = True + + +@pytest.mark.asyncio +async def test_translation_websocket_opens_under_a_native_pre_call_guardrail( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + litellm, + "callbacks", + [ + _NativeLifecycleGuardrail( + guardrail_name="runner-skipped", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + ], + ) + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=FakeLogging(), + ) + + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls", "websocket")) +@pytest.mark.parametrize( + ("key_metadata", "blocked"), + ( + pytest.param({"user_api_key_metadata": {"disable_global_guardrails": True}}, False, id="key-disables-global"), + pytest.param({}, True, id="no-opt-out"), + ), +) +async def test_translation_lets_only_an_opted_out_key_past_a_legacy_string_callback_guardrail( + route: str, + key_metadata: Mapping[str, object], + blocked: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + initialize_callbacks_on_proxy( + value=["lakera_prompt_injection"], premium_user=False, config_file_path="", litellm_settings={} + ) + if route == "websocket": + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + metadata=key_metadata, + ) + if blocked: + assert connect.url is None + assert websocket.close_code == 1008 + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in websocket.sent[0] + else: + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + assert all(TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE not in event for event in websocket.sent) + return + gateway: Final = _GatewayThatRecordsTheUrl() + call: Final = _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + metadata=key_metadata, + ) + if blocked: + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE): + await call + assert gateway.url is None + else: + await call + assert gateway.url is not None + + +@pytest.mark.asyncio +async def test_translation_websocket_closes_1008_for_a_provider_other_than_openai_and_azure() -> None: + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="xai/grok-voice", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + ) + + assert connect.url is None + assert websocket.close_code == 1008 + assert [json.loads(event)["error"]["message"] for event in websocket.sent] == [ + "Realtime translation supports only the azure, azure_ai, openai providers, got xai" + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("route", ("translation_client_secrets", "translation_calls", "websocket")) +@pytest.mark.parametrize("mode", ("pre_call", "post_call")) +@pytest.mark.parametrize("metadata_bucket", ("metadata", "litellm_metadata")) +@pytest.mark.parametrize("selected,condition_matches", ((True, True), (True, False), (False, True))) +async def test_translation_rejects_only_applicable_policy_pipelines( + route: str, + mode: Literal["pre_call", "post_call"], + metadata_bucket: str, + selected: bool, + condition_matches: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + request_data: Final[dict[str, object]] = {"metadata": {}, "litellm_metadata": {}} + _apply_resolved_guardrails_to_metadata( + data=request_data, + metadata_variable_name=metadata_bucket, + context=PolicyMatchContext(model="translation-deployment"), + policy_names=["translation-policy"] if selected else [], + policies={ + "translation-policy": Policy( + condition=PolicyCondition(model="translation-deployment" if condition_matches else "unrelated-model"), + pipeline=GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="speech-policy", on_fail="block")]), + ) + }, + ) + metadata: Final = realtime_main._build_litellm_metadata(request_data) + blocked: Final = selected and condition_matches + if route == "websocket": + websocket: Final = _RecordingClientWebSocket() + connect: Final = _CapturingConnect() + with patch("websockets.connect", connect): + await realtime_main._arealtime.__wrapped__( + model="openai/gpt-realtime-translate", + websocket=websocket, + api_key="fake-key", + realtime_mode="translation", + litellm_logging_obj=_realtime_websocket_logging(), + litellm_metadata=metadata, + ) + if blocked: + assert connect.url is None + assert websocket.close_code == 1008 + assert TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE in websocket.sent[0] + else: + assert connect.url == "wss://api.openai.com/v1/realtime/translations?model=gpt-realtime-translate" + return + upstream: Final = _UpstreamThatMustNotBeCalled() + call: Final = _realtime_http_route_call( + route, + _realtime_http_logging(route), + custom_llm_provider="openai", + api_base="https://upstream.example", + client=AsyncHTTPHandler(transport=httpx.MockTransport(upstream)), + metadata=metadata, + ) + if blocked: + with pytest.raises(litellm.BadRequestError, match=TRANSLATION_GUARDRAILS_UNSUPPORTED_MESSAGE) as raised: + await call + assert raised.value.status_code == 400 + assert upstream.called is False + else: + response: Final = await call + assert response.status_code == 200 + assert upstream.called is True + + @pytest.mark.asyncio async def test_arealtime_routes_soniox_with_the_deployment_options(monkeypatch): from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig diff --git a/tests/unit/test_check_openapi_docs.py b/tests/unit/test_check_openapi_docs.py index d7122541416..5d8803c4118 100644 --- a/tests/unit/test_check_openapi_docs.py +++ b/tests/unit/test_check_openapi_docs.py @@ -1,7 +1,10 @@ """Tests for scripts/check_openapi_docs.py, the scan behind scripts/openapi_docs_gate.py.""" +from collections.abc import Mapping from typing import Final +import pytest + import check_openapi_docs as checker _BODY: Final = {"content": {"application/json": {"schema": {"type": "object"}}}} @@ -9,8 +12,23 @@ _RESPONSES: Final = {"200": {"content": {"application/json": {"schema": {"$ref": _DOCUMENTED: Final = {"description": "Does a thing", "requestBody": _BODY, "responses": _RESPONSES} -def test_a_fully_documented_post_has_no_findings() -> None: - assert checker.violations({"paths": {"/v1/thing": {"post": _DOCUMENTED}}}) == () +def _post_with_media_types(request_media_type: str, response_media_type: str) -> Mapping[str, object]: + return { + **_DOCUMENTED, + "requestBody": {"content": {request_media_type: {"schema": {"type": "string"}}}}, + "responses": {"200": {"content": {response_media_type: {"schema": {"type": "string"}}}}}, + } + + +@pytest.mark.parametrize("media_type", ("application/json", "application/sdp")) +def test_a_fully_documented_post_has_no_findings(media_type: str) -> None: + operation: Final = _post_with_media_types(media_type, media_type) + assert checker.violations({"paths": {"/v1/thing": {"post": operation}}}) == () + + +def test_a_schema_under_another_media_type_still_counts_as_undocumented() -> None: + operation: Final = _post_with_media_types("multipart/form-data", "audio/wav") + assert {v.code for v in checker.violations({"paths": {"/v1/thing": {"post": operation}}})} == {"OA001", "OA003"} def test_each_missing_piece_is_its_own_rule() -> None: diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index aa759f2c4d9..e0ac07b3870 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -1451,6 +1451,15 @@ def test_transcription_usage_cost_returns_zero_for_unknown_type(): assert _transcription_usage_cost({}, {}) == 0.0 +@pytest.mark.parametrize("malformed_seconds", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_transcription_ignores_invalid_provider_duration(malformed_seconds: float | int | bool) -> None: + from litellm.cost_calculator import _get_transcription_usage_duration + + response: Final = SimpleNamespace(usage={"type": "duration", "seconds": malformed_seconds}) + + assert _get_transcription_usage_duration(response) is None + + def test_realtime_transcription_prices_each_clip_at_the_transcription_model_set_when_it_ran(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) @@ -7474,6 +7483,39 @@ def test_transcription_cost_falls_back_to_provider_billed_seconds( assert cost == pytest.approx(0.01 * expected_seconds) +@pytest.mark.parametrize( + ("provider", "measured_seconds", "expected_seconds"), + [("openai", 1.0, 1.0), ("azure", 1.0, 1.0), ("azure", None, 7.0)], + ids=["openai_measured_upload_wins", "azure_measured_upload_wins", "unmeasured_upload_bills_provider_seconds"], +) +def test_transcription_priced_by_provider_duration_still_bills_the_measured_upload( + monkeypatch: pytest.MonkeyPatch, provider: str, measured_seconds: float | None, expected_seconds: float +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "gpt-fake-transcribe", + { + "input_cost_per_second": 0.01, + "litellm_provider": provider, + "mode": "audio_transcription", + "provider_specific_entry": {"transcription_usage_duration": 1}, + }, + ) + measured: Final = {} if measured_seconds is None else {"audio_transcription_duration": measured_seconds} + transcription: Final = convert_to_model_response_object( + response_object={"text": "What is the weather in Paris?", "usage": {"type": "duration", "seconds": 7}}, + model_response_object=TranscriptionResponse(), + hidden_params={"model": "gpt-fake-transcribe", "custom_llm_provider": provider, **measured}, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, model=f"{provider}/gpt-fake-transcribe", call_type="atranscription" + ) + + assert cost == pytest.approx(0.01 * expected_seconds) + + def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setitem( litellm.model_cost, @@ -7498,6 +7540,77 @@ def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypat assert cost == pytest.approx(0.000075 * 2) +@pytest.mark.parametrize( + "deployment_id,deployment_rates,expected_cost", + [ + ("per-second-only", {"input_cost_per_second": 0.0001}, 30 * 0.0001), + ("zero-token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 0.0}, 0.0), + ("token-rate", {"input_cost_per_second": 0.0001, "input_cost_per_token": 1e-6}, 500 * 1e-6), + ], +) +def test_transcription_deployment_bills_seconds_only_without_a_declared_token_rate( + monkeypatch: pytest.MonkeyPatch, + deployment_id: str, + deployment_rates: Mapping[str, float], + expected_cost: float, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + deployment_id, + {**deployment_rates, "litellm_provider": "openai", "mode": "audio_transcription"}, + ) + transcription: Final = convert_to_model_response_object( + response_object={ + "text": "What is the weather in Paris?", + "usage": { + "type": "tokens", + "input_tokens": 500, + "output_tokens": 100, + "total_tokens": 600, + "input_token_details": {"audio_tokens": 500, "text_tokens": 0}, + }, + }, + model_response_object=TranscriptionResponse(), + hidden_params={ + "model": "gpt-4o-transcribe", + "custom_llm_provider": "openai", + "audio_transcription_duration": 30.0, + }, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, + model="openai/gpt-4o-transcribe", + call_type="atranscription", + custom_pricing=True, + router_model_id=deployment_id, + ) + + assert cost == pytest.approx(expected_cost) + + +@pytest.mark.parametrize("model", ["gpt-4o-transcribe", "gpt-4o-mini-transcribe"]) +def test_azure_transcription_without_usage_bills_only_tokens_while_openai_bills_the_measured_audio( + _local_model_cost_map: None, model: str +) -> None: + def cost_of_ten_seconds_without_usage(provider: str) -> float: + transcript: Final = TranscriptionResponse(text="Good") + transcript.set_audio_transcription_duration(10.0) + return completion_cost( + completion_response=transcript, + model=f"{provider}/{model}", + custom_llm_provider=provider, + call_type="atranscription", + ) + + assert cost_of_ten_seconds_without_usage("openai") > 0 + assert cost_of_ten_seconds_without_usage("azure") == 0.0, ( + "Azure lists only token prices for this model, https://azure.microsoft.com/en-us/pricing/details/azure-openai/ " + "(per the 2026-10-28 review)" + ) + + def test_gpt_image_2_azure_cost_tracking(): azure_image_generation_response: Final = { "created": 1758585600, @@ -10441,6 +10554,301 @@ def test_responses_ws_logging_object_carries_the_completed_responses_id_tier_and assert ResponsesWebSocketTokenUsageProcessor.create_logging_object([{"type": "error", "error": {}}]).id is None +@pytest.mark.parametrize("provider", ("openai", "azure")) +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_cached_multimodal_token_cost(_local_model_cost_map, provider: str, model: str): + model_name: Final = f"azure/{model}" if provider == "azure" else model + rates: Final = litellm.model_cost[model_name] + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.created", "session": {"model": model}}, + { + "type": "response.done", + "response": { + "usage": { + "input_tokens": 1000, + "output_tokens": 300, + "total_tokens": 1300, + "input_token_details": { + "text_tokens": 400, + "audio_tokens": 400, + "image_tokens": 200, + "cached_tokens": 300, + "cached_tokens_details": {"text_tokens": 100, "audio_tokens": 150, "image_tokens": 50}, + }, + "output_token_details": {"text_tokens": 100, "audio_tokens": 100, "reasoning_tokens": 100}, + } + }, + }, + ] + combined: Final = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(events) + actual: Final = handle_realtime_stream_cost_calculation( + results=events, + combined_usage_object=combined, + custom_llm_provider=provider, + litellm_model_name=model_name, + ) + expected: Final = ( + 300 * rates["input_cost_per_token"] + + 250 * rates["input_cost_per_audio_token"] + + 150 * rates["input_cost_per_image_token"] + + 100 * rates["cache_read_input_token_cost"] + + 150 * rates["cache_read_input_audio_token_cost"] + + 50 * rates["cache_read_input_image_token_cost"] + + 200 * rates["output_cost_per_token"] + + 100 * rates["output_cost_per_audio_token"] + ) + + assert actual == pytest.approx(expected) + + +def test_realtime_translation_duration_cost(_local_model_cost_map): + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "gpt-realtime-translate" + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": 1.5}} + ] + actual: Final = handle_realtime_translation_cost_calculation( + results=events, + custom_llm_provider="openai", + litellm_model_name=model, + ) + + assert actual == pytest.approx(2 * litellm.model_cost[model]["input_cost_per_second"]) + + +def test_realtime_translation_bills_session_audio_once_on_openai_and_azure(_local_model_cost_map) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + one_minute_session: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}} + ] + openai_cost: Final = handle_realtime_translation_cost_calculation( + results=one_minute_session, custom_llm_provider="openai", litellm_model_name="gpt-realtime-translate" + ) + azure_cost: Final = handle_realtime_translation_cost_calculation( + results=one_minute_session, custom_llm_provider="azure", litellm_model_name="gpt-realtime-translate" + ) + + assert openai_cost == pytest.approx(60 * litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"]) + assert openai_cost == pytest.approx(azure_cost) + + +@pytest.mark.parametrize("malformed_duration", [-1.0, float("nan"), float("inf"), 10**1000, True]) +def test_realtime_translation_duration_cost_ignores_malformed_provider_usage( + _local_model_cost_map, malformed_duration: float | int | bool +) -> None: + from litellm.cost_calculator import handle_realtime_translation_cost_calculation + + model: Final = "translation-deployment-priced-both-directions" + litellm.register_model( + model_cost={ + model: { + "litellm_provider": "openai", + "mode": "realtime", + "input_cost_per_second": 0.001, + "output_cost_per_second": 0.002, + } + } + ) + input_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": malformed_duration, "output_seconds": 1.5}, + } + ] + output_invalid_events: Final[OpenAIRealtimeStreamList] = [ + { + "type": "session.closed", + "usage": {"type": "duration", "input_seconds": 2.0, "output_seconds": malformed_duration}, + } + ] + + input_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=input_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + output_invalid_cost: Final = handle_realtime_translation_cost_calculation( + results=output_invalid_events, custom_llm_provider="openai", litellm_model_name=model + ) + + assert input_invalid_cost == pytest.approx(1.5 * litellm.model_cost[model]["output_cost_per_second"]) + assert output_invalid_cost == pytest.approx(2.0 * litellm.model_cost[model]["input_cost_per_second"]) + + +def _translation_session_cost( + deployment_id: str, + session_model: str | None = "gpt-realtime-translate", + model: str = "gpt-realtime-translate", + base_model: str | None = None, +) -> float: + session_created: Final[OpenAIRealtimeStreamList] = ( + [{"type": "session.created", "session": {"model": session_model}}] if session_model is not None else [] + ) + one_minute_session: Final = LiteLLMRealtimeStreamLoggingObject( + usage=Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), + results=[ + *session_created, + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 60.0, "output_seconds": 60.0}}, + ], + ) + return completion_cost( + completion_response=one_minute_session, + model=model, + call_type=CallTypes.arealtime.value, + custom_llm_provider="openai", + custom_pricing=True, + router_model_id=deployment_id, + base_model=base_model, + ) + + +@pytest.mark.parametrize("session_model", ["gpt-realtime-translate", None]) +@pytest.mark.parametrize( + "deployment_model,base_model", + [("gpt-realtime-translate", None), ("my-translator", "gpt-realtime-translate")], +) +@pytest.mark.parametrize( + "deployment_id,deployment_rates,bills_public_input_rate", + [ + ("translate-output-priced", {"output_cost_per_second": 0.001}, True), + ("translate-free-input", {"input_cost_per_second": 0.0, "output_cost_per_second": 0.001}, False), + ], +) +def test_realtime_translation_deployment_per_second_rates_fall_back_to_public_rates_per_key( + _local_model_cost_map: None, + deployment_id: str, + deployment_rates: Mapping[str, float], + bills_public_input_rate: bool, + deployment_model: str, + base_model: str | None, + session_model: str | None, +) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": f"openai/{deployment_model}", + "api_key": "sk-test", + **deployment_rates, + }, + "model_info": {"id": deployment_id, **({"base_model": base_model} if base_model else {})}, + } + ] + ) + public_input_rate: Final = litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"] + + assert public_input_rate > 0 + assert _translation_session_cost( + deployment_id, session_model=session_model, model=deployment_model, base_model=base_model + ) == pytest.approx(60 * 0.001 + (60 * public_input_rate if bills_public_input_rate else 0.0)) + + +def test_realtime_translation_custom_deployment_with_one_rate_bills_the_session_model_for_the_other( + _local_model_cost_map: None, +) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/my-translator", + "api_key": "sk-test", + "output_cost_per_second": 0.001, + }, + "model_info": {"id": "custom-translate-output-priced"}, + } + ] + ) + public_input_rate: Final = litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"] + + assert public_input_rate > 0 + assert _translation_session_cost( + "custom-translate-output-priced", model="my-translator" + ) == pytest.approx(60 * 0.001 + 60 * public_input_rate) + + +def test_realtime_translation_rate_ignores_a_session_model_the_deployment_does_not_run( + _local_model_cost_map: None, +) -> None: + litellm.register_model( + model_cost={ + "translation-session-echo": { + "litellm_provider": "openai", + "mode": "realtime", + "input_cost_per_second": 0.5, + } + } + ) + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/gpt-realtime-translate", + "api_key": "sk-test", + "output_cost_per_second": 0.001, + }, + "model_info": {"id": "translate-output-priced"}, + } + ] + ) + + assert _translation_session_cost( + "translate-output-priced", session_model="translation-session-echo" + ) == pytest.approx(_translation_session_cost("translate-output-priced", session_model=None)) + + +def test_realtime_translation_deployment_without_per_second_rates_bills_public_rates(_local_model_cost_map) -> None: + litellm.Router( + model_list=[ + { + "model_name": "translate", + "litellm_params": { + "model": "openai/gpt-realtime-translate", + "api_key": "sk-test", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + "model_info": {"id": "translate-token-priced"}, + } + ] + ) + public_entry: Final = litellm.model_cost["gpt-realtime-translate"] + public_cost: Final = 60 * (public_entry.get("input_cost_per_second") or 0.0) + 60 * ( + public_entry.get("output_cost_per_second") or 0.0 + ) + + assert public_cost > 0 + assert _translation_session_cost("translate-token-priced") == pytest.approx(public_cost) + + +def test_realtime_translation_spend_usage_carries_the_billed_input_seconds(_local_model_cost_map) -> None: + logging_obj: Final = Logging( + model="gpt-realtime-translate", + messages=[], + stream=False, + call_type=CallTypes.arealtime.value, + start_time=datetime.datetime.now(), + litellm_call_id="translation-spend-usage", + function_id="translation-spend-usage", + ) + logging_obj.update_environment_variables( + model="gpt-realtime-translate", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) + events: Final[OpenAIRealtimeStreamList] = [ + {"type": "session.closed", "usage": {"type": "duration", "input_seconds": 30.0, "output_seconds": 42.5}} + ] + + logging_obj.success_handler(result=events, start_time=datetime.datetime.now(), end_time=datetime.datetime.now()) + + payload: Final = logging_obj.model_call_details["standard_logging_object"] + assert payload["metadata"]["usage_object"]["seconds"] == pytest.approx(30.0) + assert payload["response_cost"] == pytest.approx( + 30.0 * litellm.model_cost["gpt-realtime-translate"]["input_cost_per_second"] + ) + + @pytest.mark.parametrize("profile_prefix", ("us", "global")) @pytest.mark.parametrize( "model_name", diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 95c5958c748..47629aab326 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -6184,6 +6184,79 @@ async def test_groq_atranscription_honors_base_url_alias(respx_mock: respx.MockR assert response.text == "hello" +def test_transcription_keeps_the_prompt_passed_by_position(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + + litellm.transcription( + "groq/whisper-large-v3", GROQ_WAV_FILE, "en", "Four score", base_url=GROQ_INTERNAL_BASE, api_key="fake-key" + ) + + sent: Final = route.calls.last.request.content + assert b'name="language"\r\n\r\nen\r\n' in sent + assert b'name="prompt"\r\n\r\nFour score\r\n' in sent + + +@pytest.mark.parametrize( + "provider_body,expected_upload_measurements", + [({"text": "hello", "duration": 2.5}, 0), ({"text": "hello"}, 1)], + ids=["provider_reports_duration", "provider_omits_duration"], +) +def test_transcription_measures_the_upload_only_when_the_provider_omits_the_duration( + respx_mock: respx.MockRouter, provider_body: Mapping[str, object], expected_upload_measurements: int +): + respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock( + return_value=httpx.Response(200, json=provider_body) + ) + + with patch.object( + litellm_main, "calculate_request_duration", wraps=litellm_main.calculate_request_duration + ) as measure_upload: + response: Final = litellm.transcription( + model="groq/whisper-large-v3", file=GROQ_WAV_FILE, base_url=GROQ_INTERNAL_BASE, api_key="fake-key" + ) + + assert response.text == "hello" + assert measure_upload.call_count == expected_upload_measurements + + +@pytest.mark.parametrize("model_name", ("whisper-1", "gpt-4o-transcribe", "gpt-transcribe")) +def test_openai_compatible_transcription_with_an_openai_model_name_reaches_the_server( + respx_mock: respx.MockRouter, model_name: str +): + route: Final = respx_mock.post("https://vllm.internal/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "served by vllm"}) + ) + + response: Final = litellm.transcription( + model=f"hosted_vllm/{model_name}", + file=GROQ_WAV_FILE, + api_base="https://vllm.internal/v1", + api_key="fake-key", + response_format="verbose_json", + ) + + assert route.called + assert response.text == "served by vllm" + + +def test_openai_gpt_transcribe_still_rejects_non_json_response_formats(respx_mock: respx.MockRouter): + route: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "should not be sent"}) + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format 'json' or 'text'"): + litellm.transcription( + model="openai/gpt-transcribe", + file=GROQ_WAV_FILE, + api_key="fake-key", + response_format="srt", + ) + + assert not route.called + + def test_groq_speech_honors_base_url_alias(respx_mock: respx.MockRouter): route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/speech").mock( return_value=httpx.Response(200, content=b"mp3-bytes") diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 759eee51ba4..e9e7f17f80d 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -866,6 +866,67 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): ] +@pytest.mark.parametrize("model", ("gpt-realtime-2.1", "gpt-realtime-2.1-mini")) +def test_realtime_family_cache_image_rate_tracks_azure(prices: dict, model: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["cache_read_input_image_token_cost"] > 0 + assert azure["cache_read_input_image_token_cost"] == openai["cache_read_input_image_token_cost"] + assert azure["input_cost_per_image_token"] >= azure["cache_read_input_image_token_cost"] + + +@pytest.mark.parametrize( + "model,mode", + ( + ("gpt-realtime-translate", "realtime"), + ("gpt-live-transcribe", "audio_transcription"), + ("gpt-transcribe", "audio_transcription"), + ), +) +def test_azure_realtime_specialized_models_follow_openai_modes(prices: dict, model: str, mode: str): + openai: Final = prices[model] + azure: Final = prices[f"azure/{model}"] + + assert openai["mode"] == azure["mode"] == mode + assert azure["supports_audio_input"] is True + assert set(openai.get("supported_endpoints", ())) <= set(azure.get("supported_endpoints", ())) + + +_DATED_SNAPSHOT: Final = re.compile(r"(?P.+)-\d{4}-\d{2}-\d{2}") + + +def _azure_realtime_snapshots_with_an_alias(prices: Mapping[str, dict]) -> tuple[tuple[str, str], ...]: + dated: Final = ( + (name, _DATED_SNAPSHOT.fullmatch(name)) + for name, entry in prices.items() + if name.startswith("azure/") and "/v1/realtime" in entry.get("supported_endpoints", ()) + ) + return tuple((name, match["alias"]) for name, match in dated if match is not None and match["alias"] in prices) + + +def _realtime_ga_only(entry: dict) -> object: + return (entry.get("provider_specific_entry") or {}).get("realtime_ga_only") + + +def test_dated_azure_realtime_models_select_the_same_protocol_as_their_alias(prices: dict): + snapshots: Final = _azure_realtime_snapshots_with_an_alias(prices) + disagreeing: Final = tuple( + (snapshot, alias) + for snapshot, alias in snapshots + if _realtime_ga_only(prices[snapshot]) != _realtime_ga_only(prices[alias]) + ) + + assert snapshots + assert disagreeing == () + + +def test_model_prices_backup_is_synchronized(prices: dict): + backup: Final = json.loads(BACKUP_PRICES_PATH.read_text()) + + assert backup == prices + + def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]: prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) return MappingProxyType( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 010c2ee6f92..b95ea8509ba 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -108,6 +108,7 @@ from litellm.utils import ( client, get_non_default_completion_params, get_optional_params_image_gen, + get_optional_params_transcription, get_prompt_cache_min_tokens, get_supported_openai_params, get_token_count, @@ -1269,6 +1270,9 @@ _COST_MAP_ROW_SCHEMA: Final = { "/v1/images/generations", "/v1/realtime", "/v1/realtime/transcription_sessions", + "/v1/realtime/translations", + "/v1/realtime/translations/client_secrets", + "/v1/realtime/translations/calls", "/v1/images/variations", "/v1/images/edits", "/v1/batch", @@ -11095,6 +11099,49 @@ def test_get_provider_audio_transcription_config_resolves_for_every_provider() - assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig) + +@pytest.mark.parametrize( + ("custom_llm_provider", "model"), + [ + ("groq", "whisper-large-v3"), + ("deepgram", "nova-3"), + ("mistral", "voxtral-mini-latest"), + ("elevenlabs", "scribe_v1"), + ("fireworks_ai", "whisper-v3"), + ], +) +def test_openai_sdk_transcription_params_pass_through_to_other_providers(custom_llm_provider: str, model: str) -> None: + hints: Final = {"stream": True, "languages": ["en"], "keywords": ["LiteLLM"]} + + params: Final = get_optional_params_transcription(model=model, custom_llm_provider=custom_llm_provider, **hints) + + forwarded: Final = {**params.get("extra_body", {}), **params} + assert {key: forwarded.get(key) for key in hints} == hints + + +@respx.mock +def test_drop_params_keeps_openai_sdk_transcription_params_out_of_a_deepgram_request() -> None: + route: Final = respx.post(url__startswith="https://api.deepgram.com/v1/listen").mock( + return_value=httpx.Response( + 200, + json={"metadata": {"duration": 1.0}, "results": {"channels": [{"alternatives": [{"transcript": "hi"}]}]}}, + ) + ) + + litellm.transcription( + model="deepgram/nova-3", + file=("sample.wav", b"audio"), + api_key="dg-test", + punctuate=True, + stream=True, + languages=["en"], + keywords=["LiteLLM"], + drop_params=True, + ) + + assert dict(route.calls.last.request.url.params) == {"model": "nova-3", "punctuate": "true"} + + @pytest.mark.parametrize( ("model", "optional_params", "expected_response_format"), [ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 79680f096e3..66fe9b17a82 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11098,7 +11098,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_3"]; + get: operations["websocket_realtime_websocket_endpoint_get_5"]; put?: never; post?: never; delete?: never; @@ -11116,7 +11116,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_openai_v1_realtime_calls_post"]; delete?: never; options?: never; @@ -11133,7 +11136,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_openai_v1_realtime_client_secrets_post"]; delete?: never; options?: never; @@ -14075,7 +14081,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get"]; + get: operations["websocket_realtime_websocket_endpoint_get_3"]; put?: never; post?: never; delete?: never; @@ -14093,7 +14099,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_realtime_calls_post"]; delete?: never; options?: never; @@ -14110,7 +14119,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_realtime_client_secrets_post"]; delete?: never; options?: never; @@ -14142,6 +14154,66 @@ export interface paths { patch?: never; trace?: never; }; + "/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ + post: operations["proxy_realtime_calls_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ + post: operations["create_realtime_client_secret_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/register": { parameters: { query?: never; @@ -21893,7 +21965,7 @@ export interface paths { * WebSocket: realtime_websocket_endpoint * @description WebSocket connection endpoint */ - get: operations["websocket_realtime_websocket_endpoint_get_2"]; + get: operations["websocket_realtime_websocket_endpoint_get_4"]; put?: never; post?: never; delete?: never; @@ -21911,7 +21983,10 @@ export interface paths { }; get?: never; put?: never; - /** Proxy Realtime Calls */ + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ post: operations["proxy_realtime_calls_v1_realtime_calls_post"]; delete?: never; options?: never; @@ -21928,7 +22003,10 @@ export interface paths { }; get?: never; put?: never; - /** Create Realtime Client Secret */ + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ post: operations["create_realtime_client_secret_v1_realtime_client_secrets_post"]; delete?: never; options?: never; @@ -21960,6 +22038,66 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/realtime/translations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * WebSocket: realtime_websocket_endpoint + * @description WebSocket connection endpoint + */ + get: operations["websocket_realtime_websocket_endpoint_get_2"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/calls": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Proxy Realtime Calls + * @description Exchange a WebRTC SDP offer for an answer using the ephemeral token from client_secrets + */ + post: operations["proxy_realtime_calls_v1_realtime_translations_calls_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/realtime/translations/client_secrets": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Create Realtime Client Secret + * @description Create a short-lived client token for a Realtime conversation, transcription, or translation session + */ + post: operations["create_realtime_client_secret_v1_realtime_translations_client_secrets_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/rerank": { parameters: { query?: never; @@ -28770,7 +28908,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "acreate_realtime_client_secret" | "arealtime_calls" | "acreate_realtime_transcription_session" | "acreate_realtime_translation_client_secret" | "arealtime_translation_calls" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "get_responses" | "aget_responses" | "delete_responses" | "adelete_responses" | "cancel_responses" | "acancel_responses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -48495,6 +48633,54 @@ export interface components { */ success_definition: string; }; + /** TranscriptionDetectedLanguage */ + TranscriptionDetectedLanguage: { + /** Code */ + code: string; + }; + /** TranscriptionResponse */ + TranscriptionResponse: { + /** Languages */ + languages?: components["schemas"]["TranscriptionDetectedLanguage"][] | null; + /** Text */ + text?: string | null; + /** Usage */ + usage?: components["schemas"]["TranscriptionUsageDurationObject"] | components["schemas"]["TranscriptionUsageTokensObject"] | null; + } & { + [key: string]: unknown; + }; + /** TranscriptionUsageDurationObject */ + TranscriptionUsageDurationObject: { + /** Seconds */ + seconds: number; + /** + * Type + * @constant + */ + type: "duration"; + }; + /** TranscriptionUsageInputTokenDetailsObject */ + TranscriptionUsageInputTokenDetailsObject: { + /** Audio Tokens */ + audio_tokens: number; + /** Text Tokens */ + text_tokens: number; + }; + /** TranscriptionUsageTokensObject */ + TranscriptionUsageTokensObject: { + input_token_details?: components["schemas"]["TranscriptionUsageInputTokenDetailsObject"] | null; + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + /** Total Tokens */ + total_tokens: number; + /** + * Type + * @constant + */ + type: "tokens"; + }; /** TransformRequestBody */ TransformRequestBody: { call_type: components["schemas"]["CallTypes"]; @@ -53416,7 +53602,8 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["TranscriptionResponse"]; + "text/event-stream": string; }; }; /** @description Validation Error */ @@ -67728,7 +67915,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_3: { + websocket_realtime_websocket_endpoint_get_5: { parameters: { query?: never; header?: never; @@ -71094,7 +71281,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get: { + websocket_realtime_websocket_endpoint_get_3: { parameters: { query?: never; header?: never; @@ -71172,6 +71359,173 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/sdp": string; + }; + }; + responses: { + /** @description WebRTC SDP answer */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + create_realtime_client_secret_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "gpt-realtime-translate", + * "session": { + * "model": "gpt-realtime-translate", + * "type": "translation" + * } + * } + */ + "application/json": { + expires_after?: { + /** + * Anchor + * @default created_at + */ + anchor?: string | null; + /** Seconds */ + seconds?: number | null; + } | null; + /** Model */ + model?: string | null; + session?: ({ + audio?: ({ + input?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Noise Reduction */ + noise_reduction?: { + [key: string]: unknown; + } | null; + transcription?: ({ + /** Delay */ + delay?: string | null; + /** Keywords */ + keywords?: string[] | null; + /** Language */ + language?: string | null; + /** Languages */ + languages?: string[] | null; + /** Model */ + model?: string | null; + /** Prompt */ + prompt?: string | null; + } & { + [key: string]: unknown; + }) | null; + /** Turn Detection */ + turn_detection?: { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + output?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Language */ + language?: string | null; + /** Voice */ + voice?: string | { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + } & { + [key: string]: unknown; + }) | null; + /** Include */ + include?: string[] | null; + /** Instructions */ + instructions?: string | null; + /** Max Output Tokens */ + max_output_tokens?: number | string | null; + /** Model */ + model?: string | null; + /** Output Modalities */ + output_modalities?: string[] | null; + /** Prompt */ + prompt?: { + [key: string]: unknown; + } | null; + /** Tool Choice */ + tool_choice?: unknown | null; + /** Tools */ + tools?: { + [key: string]: unknown; + }[] | null; + /** Tracing */ + tracing?: unknown | null; + /** Truncation */ + truncation?: unknown | null; + /** Type */ + type?: string | null; + } & { + [key: string]: unknown; + }) | null; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; register_client_register_post: { parameters: { query?: { @@ -78625,7 +78979,8 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["TranscriptionResponse"]; + "text/event-stream": string; }; }; /** @description Validation Error */ @@ -82221,7 +82576,7 @@ export interface operations { }; }; }; - websocket_realtime_websocket_endpoint_get_2: { + websocket_realtime_websocket_endpoint_get_4: { parameters: { query?: never; header?: never; @@ -82299,6 +82654,173 @@ export interface operations { }; }; }; + websocket_realtime_websocket_endpoint_get_2: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description WebSocket Protocol Switched */ + 101: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + }; + }; + proxy_realtime_calls_v1_realtime_translations_calls_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/sdp": string; + }; + }; + responses: { + /** @description WebRTC SDP answer */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "text/plain": string; + }; + }; + }; + }; + create_realtime_client_secret_v1_realtime_translations_client_secrets_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "model": "gpt-realtime-translate", + * "session": { + * "model": "gpt-realtime-translate", + * "type": "translation" + * } + * } + */ + "application/json": { + expires_after?: { + /** + * Anchor + * @default created_at + */ + anchor?: string | null; + /** Seconds */ + seconds?: number | null; + } | null; + /** Model */ + model?: string | null; + session?: ({ + audio?: ({ + input?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Noise Reduction */ + noise_reduction?: { + [key: string]: unknown; + } | null; + transcription?: ({ + /** Delay */ + delay?: string | null; + /** Keywords */ + keywords?: string[] | null; + /** Language */ + language?: string | null; + /** Languages */ + languages?: string[] | null; + /** Model */ + model?: string | null; + /** Prompt */ + prompt?: string | null; + } & { + [key: string]: unknown; + }) | null; + /** Turn Detection */ + turn_detection?: { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + output?: ({ + /** Format */ + format?: string | { + [key: string]: unknown; + } | null; + /** Language */ + language?: string | null; + /** Voice */ + voice?: string | { + [key: string]: unknown; + } | null; + } & { + [key: string]: unknown; + }) | null; + } & { + [key: string]: unknown; + }) | null; + /** Include */ + include?: string[] | null; + /** Instructions */ + instructions?: string | null; + /** Max Output Tokens */ + max_output_tokens?: number | string | null; + /** Model */ + model?: string | null; + /** Output Modalities */ + output_modalities?: string[] | null; + /** Prompt */ + prompt?: { + [key: string]: unknown; + } | null; + /** Tool Choice */ + tool_choice?: unknown | null; + /** Tools */ + tools?: { + [key: string]: unknown; + }[] | null; + /** Tracing */ + tracing?: unknown | null; + /** Truncation */ + truncation?: unknown | null; + /** Type */ + type?: string | null; + } & { + [key: string]: unknown; + }) | null; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["RealtimeClientSecretResponse"]; + }; + }; + }; + }; rerank_v1_rerank_post: { parameters: { query?: never; From 5b546259d568317d3689053d72faa88d04982f7b Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:55:51 -0800 Subject: [PATCH 18/34] fix(cache_control)!: keep one deployment's prompt-cache opt-in from reaching siblings on the same model (#49685) * fix(cache_control): read the routed deployment's own prompt-cache flag, not a sibling's shared cost-map entry Two deployments that share one model string share one key in the live litellm.model_cost map, and router registration writes each deployment's supports_prompt_cache_breakpoint into that key. The cache control hook read the live map, so a flagless deployment picked up a sibling's opt-in and sent prompt_cache_options and prompt_cache_breakpoint upstream, which the provider rejects with a 400 When a call carries the routed deployment's model_info, the prompt cache checks now use its declared supports_prompt_cache_breakpoint, and fall back to the loaded catalog instead of the live map. This covers the hook on completion, acompletion, Responses and the /v1/messages route, the azure_ai chat transform, the OpenAI Responses transform under drop_params, and the /v1/messages to Responses adapter * fix(prompt caching): read each deployment's own breakpoint flag in the openai and bedrock readers A deployment's own supports_prompt_cache_breakpoint now picks OpenAI's prompt_cache_breakpoint only where the request goes out OpenAI-shaped on a provider other than openai, such as azure_ai OpenAI models and bedrock_mantle. Bedrock on every route and providers that build Anthropic messages keep cache_control, which Bedrock Converse turns into cachePoint The openai chat reader now passes the routed model_info and checks base_model. The Bedrock cache point reader takes the deployment's flag first and, for a routed call with no flag, reads the catalog as loaded instead of the live map that sibling deployments write into. model_info is threaded from the converse transform, the factory message and tool builders, and the responses bridge * fix(prompt caching): keep gateways on cache_control and read register_model opt-ins on routed calls A declared supports_prompt_cache_breakpoint picks OpenAI's breakpoint only on providers hosting OpenAI's API or for a model the cost map lists under the provider, so OpenRouter, LiteLLM proxy, and vLLM deployments keep cache_control as before. Routed calls read the loaded catalog plus the user's own register_model entries, never router registrations. * fix(prompt caching): honor a deployment's own supports_prompt_caching on Bedrock and its declared litellm_provider on gateways * fix(prompt caching): keep wildcard cache flags off matched models, read deployment flags for auto caching and base_model aliases * test(anthropic): cover auto caching on an unmapped Claude deployment * fix(cache_control): keep the merge base's reads where the per-deployment flag changed them A register_model entry under a provider-prefixed name reaches routed calls again, since the overlay now resolves the bare and prefixed names the way runtime registrations are stored. The hook and the openai chat reader no longer judge base_model, as at the merge base, so a client base_model sent under allow_client_pricing_override cannot pick OpenAI's breakpoint and get a 400. A Bedrock deployment flag that is not a real true reads as false again, as at the merge base. An azure_ai wildcard's supports_prompt_cache_breakpoint speaks for each model it matches again, as at the merge base. Other wildcards still leave both cache flags to the catalog. The s10/s11 Bedrock wire test asserted that a flagless deployment inherits its sibling's flag on a shared ARN, which is the leak this PR removes. It now asserts that each deployment keeps its own flag and that a flagless unmapped ARN keeps its cache points after the delete. * fix(cache_control): let the later register_model entry win between a bare and a provider-prefixed name Routed calls read register_model entries through an overlay that merged the bare name after the provider-prefixed one whatever order they were registered in, so a later opt-out under one name lost to an earlier opt-in under the other. Each registration now moves its key to the end of the kept registrations, and the overlay merges both names oldest first, which matches what SDK calls read from the live model map. Also lists the wildcard cache flag helper in the router coverage script, since the hook's wildcard tests drive it rather than calling it by name. * fix(cache_control): keep register_model entries visible while a cost map reload replays them The previous commit moved a re-registered name to the end of the saved registrations by removing and re-adding it, and the overlay walked that dict to order a bare and a provider-prefixed name. A reload replays every saved entry, so a request on another thread could briefly miss an entry or fail on a dict that changed size mid-walk. Entries are written in place again, a per-name sequence records which registration came last, and both the overlay and the reload replay order by it, so the later registration still wins after a reload. * fix(cache_control): read a string or numeric supports_prompt_caching as the bool it spells and find register_model entries under any route prefix A deployment flag stored as "true", 1, "false" or 0 (POST /model/new and quoted YAML keep strings) now means that bool on the Bedrock cachePoint reader and Anthropic auto caching, and any other value falls to the catalog. register_model records which names wrote to each catalog row, so an opt-out under bedrock/ or bedrock/converse/ reaches routed calls like one under anthropic/. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../transformation.py | 10 +- .../anthropic_cache_control_hook.py | 207 ++++-- litellm/litellm_core_utils/core_helpers.py | 10 +- .../pattern_match_deployments.py | 16 + .../prompt_templates/factory.py | 52 +- litellm/llms/anthropic/common_utils.py | 26 +- .../messages/responses_adapters/handler.py | 7 +- litellm/llms/azure_ai/chat/transformation.py | 20 +- .../bedrock/chat/converse_transformation.py | 39 +- litellm/llms/bedrock/common_utils.py | 41 +- .../llms/openai/chat/gpt_transformation.py | 19 +- .../llms/openai/responses/transformation.py | 2 + litellm/main.py | 4 + .../transformation.py | 23 +- litellm/responses/main.py | 2 + litellm/utils.py | 60 +- .../router_code_coverage.py | 1 + .../test_bedrock_kimi_k3_cache_point_wire.py | 10 +- ...responses_transformation_transformation.py | 32 + .../test_anthropic_cache_control_hook.py | 633 ++++++++++++++++++ .../anthropic/test_anthropic_common_utils.py | 140 ++++ .../test_azure_chat_gpt_transformation.py | 2 +- .../llms/bedrock/test_bedrock_common_utils.py | 184 +++++ .../chat/test_openai_gpt_transformation.py | 42 ++ .../test_litellm_completion_responses.py | 25 + tests/unit/test_utils.py | 20 + 26 files changed, 1511 insertions(+), 116 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 0a500173c5b..c03419436d6 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -34,7 +34,7 @@ from litellm.completion_extras.litellm_responses_transformation.history import ( replay_items, requested_phase, ) -from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint +from litellm.integrations.anthropic_cache_control_hook import routed_model_info, supports_openai_prompt_cache_breakpoint from litellm.litellm_core_utils.allowed_tools_tool_choice import ( parse_chat_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, @@ -1166,10 +1166,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): optional_params = self._extract_extra_body_params(optional_params) provider: Final = _served_provider(litellm_params) base_model: Final = litellm_params.get("base_model") - supports_prompt_cache_breakpoint: Final = supports_openai_prompt_cache_breakpoint(model, provider) or ( - isinstance(base_model, str) - and bool(base_model) - and supports_openai_prompt_cache_breakpoint(base_model, provider) + model_info: Final = routed_model_info(litellm_params.get("model_info")) + named_models: Final = (model, base_model) if isinstance(base_model, str) and base_model else (model,) + supports_prompt_cache_breakpoint: Final = any( + supports_openai_prompt_cache_breakpoint(name, provider, model_info) for name in named_models ) converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api( messages, diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 01be455e229..4302b24ca33 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -14,6 +14,7 @@ import itertools import os import re from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence +from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast from urllib.parse import urlparse @@ -23,7 +24,6 @@ from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient -from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.prompt_templates.common_utils import ( cache_control_as_prompt_cache_breakpoint, with_prompt_cache_breakpoint, @@ -85,8 +85,40 @@ _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) _CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object]) _ATTRIBUTE_VALUE: Final = TypeAdapter(object) +_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues +CostMap: TypeAlias = Mapping[str, Mapping[str, object]] + + +def routed_model_info(value: object) -> Mapping[str, object] | None: + try: + return _MODEL_INFO_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _live_cost_map_unless(model_cost: CostMap | None) -> CostMap: + import litellm + + if model_cost is not None: + return model_cost + return cast(CostMap, litellm.model_cost) # cast-ok: the live map holds catalog entries + + +def declared_prompt_cache_breakpoint(model_info: Mapping[str, object] | None) -> bool | None: + declared: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint") + return declared if isinstance(declared, bool) else None + + +def routed_cost_map() -> CostMap: + from litellm.utils import catalog_with_runtime_registrations + + return catalog_with_runtime_registrations() + + +def _catalog_unless_unrouted(model_info: Mapping[str, object] | None) -> CostMap | None: + return None if model_info is None else routed_cost_map() def _validated_object_mapping(value: object) -> dict[object, object] | None: @@ -314,19 +346,27 @@ def _served_provider_for_model(model: str, api_base: object = None) -> str | Non return provider -def supports_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None = None) -> bool: - return openai_prompt_cache_breakpoint_support(model, custom_llm_provider) is True +def supports_openai_prompt_cache_breakpoint( + model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: + return openai_prompt_cache_breakpoint_support(model, custom_llm_provider, model_info) is True -def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str | None = None) -> bool | None: +def openai_prompt_cache_breakpoint_support( + model: str, custom_llm_provider: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool | None: + declared: Final = declared_prompt_cache_breakpoint(model_info) + if declared is not None: + return declared + model_cost: Final = _catalog_unless_unrouted(model_info) hosted_flag: Final = ( None if custom_llm_provider is None - else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model) + else _hosted_openai_dialect_flag(model, custom_llm_provider, _served_provider_for_model, model_cost) ) if hosted_flag is not None: return hosted_flag - model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) + model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model, model_cost) if model_map_flag is not None: return model_map_flag fine_tuned_base: Final = _FINE_TUNED_BASE_PATTERN.match(model.rsplit("/", 1)[-1]) @@ -347,13 +387,18 @@ class _DeclaredBaseModels(BaseModel): return tuple(name for name in candidates if isinstance(name, str) and name) +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + return routed_model_info(_DeclaredBaseModels.model_validate(litellm_params).model_info) + + def deployment_prompt_cache_breakpoint_support( model: str, custom_llm_provider: str, litellm_params: object ) -> bool | None: """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first.""" declared: Final = _DeclaredBaseModels.model_validate(litellm_params) + model_info: Final = routed_model_info(declared.model_info) supports: Final = ( - openai_prompt_cache_breakpoint_support(name, custom_llm_provider) + openai_prompt_cache_breakpoint_support(name, custom_llm_provider, model_info) for name in dict.fromkeys((*declared.names(), model)) ) return next((support for support in supports if support is not None), None) @@ -370,11 +415,10 @@ def _named_model_breakpoint_support(model: str) -> bool | None: return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION -def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: - import litellm - - entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) - flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) +def _model_map_prompt_cache_breakpoint_flag(model: str, model_cost: CostMap | None = None) -> bool | None: + cost_map: Final = _live_cost_map_unless(model_cost) + entries: Final = (cost_map.get(key) for key in (model, model.rsplit("/", 1)[-1])) + flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, Mapping)) return next((flag is True for flag in flags if flag is not None), None) @@ -390,7 +434,7 @@ def _hosted_openai_dialect_flag( model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None], - model_cost: Mapping[str, Mapping[str, object]] | None = None, + model_cost: CostMap | None = None, ) -> bool | None: """ Explicit opt-in for an OpenAI-shaped deployment served by another provider. @@ -407,13 +451,7 @@ def _hosted_openai_dialect_flag( row of the same name, so an exact entry only speaks for a deployment when it is keyed for that deployment's provider. """ - import litellm - - cost_map: Final = ( - cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: the live map holds catalog entries - if model_cost is None - else model_cost - ) + cost_map: Final = _live_cost_map_unless(model_cost) exact_entry: Final = cost_map.get(model) exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, Mapping) else None if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)): @@ -423,10 +461,16 @@ def _hosted_openai_dialect_flag( return None if isinstance(exact_entry, Mapping) and exact_provider == provider: return _hosted_entry_flag(exact_entry, lambda: provider) + entries: Final = (cost_map.get(candidate) for candidate in _provider_cost_map_keys(model, provider)) + flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping)) + return next((flag for flag in flags if flag is not None), None) + + +def _provider_cost_map_keys(model: str, provider: str) -> tuple[str, ...]: from litellm.utils import get_potential_model_names names: Final = get_potential_model_names(model, provider) - candidates: Final = ( + return ( names["combined_model_name"], names["region_free_combined_model_name"], names["split_model"], @@ -434,9 +478,60 @@ def _hosted_openai_dialect_flag( names["stripped_model_name"], names["provider_prefixed_model_name"], ) - entries: Final = (cost_map.get(candidate) for candidate in candidates) - flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, Mapping)) - return next((flag for flag in flags if flag is not None), None) + + +def _cost_map_lists_model_for_provider(model: str, provider: str, model_cost: CostMap) -> bool: + entries: Final = (model_cost.get(key) for key in (model, *_provider_cost_map_keys(model, provider))) + return any(isinstance(entry, Mapping) and entry.get("litellm_provider") == provider for entry in entries) + + +_PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT: Final = frozenset(("bedrock", "bedrock_converse")) +_PROVIDERS_HOSTING_OPENAIS_API: Final = frozenset(("azure", "azure_ai", "bedrock_mantle")) + + +def _declared_hosted_flag( + declared: bool, model: str, provider: str | None, model_cost: CostMap, deployment_provider: object +) -> bool | None: + """ + A deployment's own ``supports_prompt_cache_breakpoint`` picks OpenAI's breakpoint only where the + request goes out OpenAI-shaped. Bedrock reads the same flag on every route as whether Converse takes + a ``cachePoint``, and a provider building Anthropic messages takes ``cache_control``, so both keep + the Anthropic mark. A gateway or self-hosted server (OpenRouter, a LiteLLM proxy, vLLM) may pass + the request on to Anthropic, so there the flag only picks OpenAI's breakpoint for a model the cost + map, or the deployment's own ``litellm_provider``, lists under that provider. + """ + if provider is None or provider == "openai": + return None + return ( + declared + and provider not in _PROVIDERS_READING_THE_FLAG_AS_CACHE_POINT + and ( + provider in _PROVIDERS_HOSTING_OPENAIS_API + or deployment_provider == provider + or _cost_map_lists_model_for_provider(model, provider, model_cost) + ) + and _sends_openai_shaped_chat(model, provider) + ) + + +@lru_cache(maxsize=512) +def _sends_openai_shaped_chat(model: str, custom_llm_provider: str) -> bool: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + from litellm.llms.openai.openai import OpenAIConfig + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + try: + served_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=served_model, provider=LlmProviders(provider) + ) + except (BadRequestError, ValueError): + return False + return isinstance(chat_config, (OpenAIGPTConfig, OpenAIConfig, AzureOpenAIConfig)) def targets_openai_api(api_base: object) -> bool: @@ -677,7 +772,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) remaining_points: Final = tuple(point for point in injection_points if point.get("location") != "message") - custom_llm_provider: Final = _custom_llm_provider(non_default_params, injection_points[0]) + request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict + custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0]) stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") openai_dialect: Final = ( stamped_dialect @@ -685,8 +781,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): else AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( model, custom_llm_provider, - non_default_params.get("api_base") or non_default_params.get("base_url"), - non_default_params.get("prompt_cache_options"), + request_params.get("api_base") or request_params.get("base_url"), + request_params.get("prompt_cache_options"), + routed_model_info(request_params.get("model_info")), ) ) # A provisional message list defers every role-targeted point to the pass holding @@ -757,32 +854,35 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, api_base: object = None, prompt_cache_options: object = None, + model_info: Mapping[str, object] | None = None, ) -> bool: if model is None: return False - hosted_flag: Final = _hosted_openai_dialect_flag( - model, custom_llm_provider, lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base) + declared: Final = declared_prompt_cache_breakpoint(model_info) + hosted_flag: Final = ( + _hosted_openai_dialect_flag( + model, + custom_llm_provider, + lambda name: AnthropicCacheControlHook._resolve_provider(name, api_base), + _catalog_unless_unrouted(model_info), + ) + if declared is None + else _declared_hosted_flag( + declared, + model, + custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model, api_base), + routed_cost_map(), + None if model_info is None else model_info.get("litellm_provider"), + ) ) if hosted_flag is not None: return hosted_flag - if not supports_openai_prompt_cache_breakpoint(model): + if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info): return False if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False return prompt_cache_options is not None or targets_openai_api(api_base) - @staticmethod - def catalog_targets_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str) -> bool: - return ( - _hosted_openai_dialect_flag( - model, - custom_llm_provider, - AnthropicCacheControlHook._resolve_provider, - GetModelCostMap.loaded_model_cost_map(), - ) - is True - ) - @staticmethod def _resolve_provider(model: str, api_base: object = None) -> str | None: return _served_provider_for_model(model, api_base) @@ -1181,6 +1281,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, api_base: object, prompt_cache_options: object, + model_info: Mapping[str, object] | None = None, ) -> Sequence[Mapping[str, object]]: """Carry onto the points what the prompt-management hook never receives. @@ -1192,7 +1293,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): across requests. """ with_dialect: Final = AnthropicCacheControlHook._stamped_with_dialect( - points, model, custom_llm_provider, api_base, prompt_cache_options + points, model, custom_llm_provider, api_base, prompt_cache_options, model_info ) stamped_earlier: Final = any(EXTERNAL_BREAKPOINTS_STAMP in point for point in with_dialect) if external_breakpoints == 0 and not stamped_earlier: @@ -1206,14 +1307,17 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider: str | None, api_base: object, prompt_cache_options: object, + model_info: Mapping[str, object] | None = None, ) -> Sequence[Mapping[str, object]]: - if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider): + if model_info is None and not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint( + model, custom_llm_provider + ): return points return AnthropicCacheControlHook._stamped( points, "_litellm_openai_dialect", AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider, api_base, prompt_cache_options + model, custom_llm_provider, api_base, prompt_cache_options, model_info ), ) @@ -1272,6 +1376,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): cache_control: object = None, request_kwargs: object = None, on_messages_route: bool = False, + model_info: Mapping[str, object] | None = None, ) -> list[CacheControlInjectionPoint]: """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on. @@ -1289,7 +1394,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True: return [] - if not supports_anthropic_cache_control(model, custom_llm_provider): + if not supports_anthropic_cache_control(model, custom_llm_provider, model_info): return [] if AnthropicCacheControlHook._request_has_cache_control( @@ -1367,6 +1472,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): tools: list | None = None, enable_prompt_caching: bool | None = None, api_base: object = None, + model_info: Mapping[str, object] | None = None, ) -> None: """For /chat/completions: resolve the injection points the request should carry. @@ -1393,6 +1499,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider, api_base, non_default_params.get("prompt_cache_options"), + model_info, ) non_default_params["cache_control_injection_points"] = ( AnthropicCacheControlHook._stamped(stamped, CUSTOM_LLM_PROVIDER_STAMP, custom_llm_provider) @@ -1409,6 +1516,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): enable_prompt_caching=enable_prompt_caching, cache_control=non_default_params.get("cache_control"), request_kwargs=non_default_params, + model_info=model_info, ) if points: non_default_params["cache_control_injection_points"] = points @@ -1527,6 +1635,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): cache_control=cache_control, request_kwargs=kwargs, on_messages_route=True, + model_info=routed_model_info(kwargs.get("model_info")), ) if model is not None else () @@ -1535,7 +1644,11 @@ class AnthropicCacheControlHook(CustomPromptManagement): return messages, system openai_dialect: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options") + model, + custom_llm_provider, + api_base, + kwargs.get("prompt_cache_options"), + routed_model_info(kwargs.get("model_info")), ) breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 6fd6e91a627..43c586e7245 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -97,7 +97,7 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non return float(seconds / denominator) -_DROP_PARAMS_BOOL: Final = TypeAdapter(bool) +_LAX_BOOL: Final = TypeAdapter(bool) def as_str_mapping(value: object) -> Mapping[str, object] | None: @@ -160,15 +160,19 @@ def attribute_items(source: object, name: str) -> Iterable[object]: return iter_items(value) if value else () -def normalize_drop_params(value: object) -> bool | None: +def lax_bool(value: object) -> bool | None: if value is None or isinstance(value, bool): return value try: - return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value) + return _LAX_BOOL.validate_python(value.strip() if isinstance(value, str) else value) except ValidationError: return None +def normalize_drop_params(value: object) -> bool | None: + return lax_bool(value) + + def drops_unsupported_params(drop_params: object) -> bool: return litellm.drop_params is True or normalize_drop_params(drop_params) is True diff --git a/litellm/litellm_core_utils/pattern_match_deployments.py b/litellm/litellm_core_utils/pattern_match_deployments.py index 2b8d1e9c11b..a9091348c50 100644 --- a/litellm/litellm_core_utils/pattern_match_deployments.py +++ b/litellm/litellm_core_utils/pattern_match_deployments.py @@ -5,6 +5,7 @@ Class to handle llm wildcard routing and regex pattern matching import copy import itertools import re +from collections.abc import Mapping from re import Match from typing import Final @@ -22,6 +23,16 @@ class _RoutedDeployment(BaseModel): _ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...]) +_PATTERN_WIDE_CACHE_FLAGS: Final = frozenset(("supports_prompt_cache_breakpoint", "supports_prompt_caching")) +_AZURE_AI_PATTERN_CACHE_FLAGS: Final = frozenset(("supports_prompt_caching",)) + + +def _cache_flags_left_to_the_catalog(litellm_params: Mapping[str, object]) -> frozenset[str]: + model: Final = litellm_params.get("model") + provider: Final = litellm_params.get("custom_llm_provider") or inferred_provider( + model if isinstance(model, str) else None + ) + return _AZURE_AI_PATTERN_CACHE_FLAGS if provider == "azure_ai" else _PATTERN_WIDE_CACHE_FLAGS class PatternUtils: @@ -133,6 +144,11 @@ class PatternMatchRouter: matched_pattern=matched_pattern, litellm_deployment_litellm_model=deployment["litellm_params"]["model"], ) + if "*" in deployment["litellm_params"]["model"] and isinstance(new_deployment.get("model_info"), dict): + left_to_the_catalog = _cache_flags_left_to_the_catalog(deployment["litellm_params"]) + new_deployment["model_info"] = { + key: value for key, value in new_deployment["model_info"].items() if key not in left_to_the_catalog + } new_deployments.append(new_deployment) return new_deployments diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index b3c8833811f..4c3907dc52f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4254,6 +4254,7 @@ def _convert_to_bedrock_tool_call_invoke( tool_calls: list, model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockContentBlock]: """ OpenAI tool invokes: @@ -4338,6 +4339,7 @@ def _convert_to_bedrock_tool_call_invoke( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: _parts_list.append(_cache_point_block) @@ -4358,6 +4360,7 @@ def _convert_to_bedrock_tool_call_invoke( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if cache_point_block is not None: _parts_list.append(cache_point_block) @@ -5029,6 +5032,7 @@ class BedrockConverseMessagesProcessor: deployment_model: str | None = None, base_model: str | None = None, drop_params: bool = False, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockMessageBlock]: from litellm.llms.bedrock.common_utils import bedrock_converse_accepts_redacted_thinking @@ -5091,6 +5095,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -5098,7 +5103,11 @@ class BedrockConverseMessagesProcessor: elif message_content and isinstance(message_content, str): _part = BedrockContentBlock(text=message_content) _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - message_block, block_type="content_block", model=model, deployment_model=deployment_model + message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) user_content.append(_part) if _cache_point_block is not None: @@ -5149,6 +5158,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if cache_point_block is not None: tool_content.append(cache_point_block) @@ -5214,6 +5224,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None and not ( drop_params and _as_thinking_block(element) is not None @@ -5231,6 +5242,7 @@ class BedrockConverseMessagesProcessor: block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) @@ -5239,7 +5251,7 @@ class BedrockConverseMessagesProcessor: if _tool_calls: assistant_content.extend( _convert_to_bedrock_tool_call_invoke( - _tool_calls, model=model, deployment_model=deployment_model + _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info ) ) @@ -5434,6 +5446,7 @@ def _bedrock_converse_messages_pt( deployment_model: str | None = None, base_model: str | None = None, drop_params: bool = False, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockMessageBlock]: """ Converts given messages from OpenAI format to Bedrock format @@ -5502,6 +5515,7 @@ def _bedrock_converse_messages_pt( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: _parts.append(_cache_point_block) @@ -5509,7 +5523,11 @@ def _bedrock_converse_messages_pt( elif message_content and isinstance(message_content, str): _part = BedrockContentBlock(text=message_content) _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - message_block, block_type="content_block", model=model, deployment_model=deployment_model + message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) user_content.append(_part) if _cache_point_block is not None: @@ -5562,6 +5580,7 @@ def _bedrock_converse_messages_pt( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if cache_point_block is not None: tool_content.append(cache_point_block) @@ -5628,6 +5647,7 @@ def _bedrock_converse_messages_pt( block_type="content_block", model=model, deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None and not ( drop_params and _as_thinking_block(element) is not None @@ -5640,14 +5660,20 @@ def _bedrock_converse_messages_pt( assistant_content.append(BedrockContentBlock(text=_assistant_content)) # Add cache point block for assistant string content _cache_point_block = litellm.AmazonConverseConfig().get_cache_point_block( - assistant_message_block, block_type="content_block", model=model, deployment_model=deployment_model + assistant_message_block, + block_type="content_block", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) if _cache_point_block is not None: assistant_content.append(_cache_point_block) _tool_calls = assistant_message_block.get("tool_calls", []) if _tool_calls: assistant_content.extend( - _convert_to_bedrock_tool_call_invoke(_tool_calls, model=model, deployment_model=deployment_model) + _convert_to_bedrock_tool_call_invoke( + _tool_calls, model=model, deployment_model=deployment_model, model_info=model_info + ) ) msg_i += 1 @@ -5703,7 +5729,10 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str: def add_cache_point_tool_block( - tool: dict, model: str | None = None, deployment_model: str | None = None + tool: dict, + model: str | None = None, + deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> BedrockToolBlock | None: from litellm.llms.bedrock.common_utils import ( bedrock_cache_point_ttl_supported, @@ -5711,7 +5740,7 @@ def add_cache_point_tool_block( ) cache_control: Final = tool.get("cache_control", None) - if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model): + if cache_control is not None and bedrock_model_accepts_cache_points(model, deployment_model, model_info): cache_point: Final = cache_control.get("type", "ephemeral") if cache_point == "ephemeral": cache_point_block: Final[CachePointBlock] = {"type": "default"} @@ -5748,7 +5777,10 @@ def _is_bedrock_tool_block(tool: dict) -> bool: def _bedrock_tools_pt( - tools: list, model: str | None = None, deployment_model: str | None = None + tools: list, + model: str | None = None, + deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> list[BedrockToolBlock]: """ OpenAI tools looks like: @@ -5869,7 +5901,9 @@ def _bedrock_tools_pt( tool_block_list.append(tool_block) ## ADD CACHE POINT TOOL BLOCK ## - cache_point_tool_block = add_cache_point_tool_block(tool, model=model, deployment_model=deployment_model) + cache_point_tool_block = add_cache_point_tool_block( + tool, model=model, deployment_model=deployment_model, model_info=model_info + ) if cache_point_tool_block is not None: tool_block_list.append(cache_point_tool_block) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index c9df903cbd6..bfd1341711a 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -28,7 +28,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, ) from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, lax_bool from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, is_encrypted_reasoning_block, @@ -307,9 +307,10 @@ def count_tokens_feature_betas( ) -def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool: +def supports_anthropic_cache_control( + model: str, custom_llm_provider: str | None, model_info: Mapping[str, object] | None = None +) -> bool: from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider - from litellm.utils import supports_prompt_caching try: provider: Final = custom_llm_provider if custom_llm_provider is not None else get_llm_provider(model=model)[1] @@ -318,10 +319,27 @@ def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None return ( provider in ("anthropic", "bedrock", "vertex_ai", "azure_ai") and "claude" in model.lower() - and supports_prompt_caching(model=model, custom_llm_provider=provider) + and _deployment_supports_prompt_caching(model, provider, model_info) ) +def _deployment_supports_prompt_caching(model: str, provider: str, model_info: Mapping[str, object] | None) -> bool: + from litellm.utils import catalog_with_runtime_registrations, get_model_info, supports_prompt_caching + + declared: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching")) + if declared is not None: + return declared + live: Final = supports_prompt_caching(model=model, custom_llm_provider=provider) + if model_info is None: + return live + try: + key: Final = get_model_info(model=model, custom_llm_provider=provider).get("key") + except Exception: # noqa: BLE001 # an unmapped model keeps the live answer + return live + catalog_entry: Final = catalog_with_runtime_registrations().get(key) + return live if catalog_entry is None else catalog_entry.get("supports_prompt_caching") is True + + def _tool_config_point_control(points: object) -> Mapping[object, object] | None: point_fields: Final = tuple(validated_object_mapping(point) for point in validated_object_list(points) or ()) tool_config_point: Final = next( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 8b638f2c03a..ec09987837c 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -13,6 +13,7 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + routed_model_info, supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.asyncify import ( @@ -187,6 +188,7 @@ def _reads_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[str, ob provider if isinstance(provider, str) else None, forwarded_kwargs.get("api_base") or forwarded_kwargs.get("base_url"), forwarded_kwargs.get("prompt_cache_options"), + routed_model_info(forwarded_kwargs.get("model_info")), ) @@ -194,10 +196,11 @@ def _model_takes_prompt_cache_breakpoint(model: str, forwarded_kwargs: Mapping[s provider: Final = forwarded_kwargs.get("custom_llm_provider") provider_name: Final = provider if isinstance(provider, str) else None base_model: Final = forwarded_kwargs.get("base_model") - return supports_openai_prompt_cache_breakpoint(model, provider_name) or ( + model_info: Final = routed_model_info(forwarded_kwargs.get("model_info")) + return supports_openai_prompt_cache_breakpoint(model, provider_name, model_info) or ( isinstance(base_model, str) and bool(base_model) - and supports_openai_prompt_cache_breakpoint(base_model, provider_name) + and supports_openai_prompt_cache_breakpoint(base_model, provider_name, model_info) ) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 5214d31fb2e..487958d6d55 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -10,7 +10,11 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_logger -from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + declared_prompt_cache_breakpoint, + routed_model_info, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( content_parts, convert_content_list_to_str, @@ -125,18 +129,14 @@ def _with_breakpoints_for_foundry(message: AllMessageValues, drop_params: bool) def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, object]) -> bool: + model_info: Final = routed_model_info(litellm_params.get("model_info")) + declared: Final = declared_prompt_cache_breakpoint(model_info) + if declared is not None: + return declared base_model: Final = litellm_params.get("base_model") named_models: Final = (base_model, model) if isinstance(base_model, str) and base_model else (model,) - model_info: Final = litellm_params.get("model_info") - if not isinstance(model_info, Mapping): - return any( - AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai") for name in named_models - ) - deployment_flag: Final = _FIELDS.validate_python(model_info).get("supports_prompt_cache_breakpoint") - if isinstance(deployment_flag, bool): - return deployment_flag return any( - AnthropicCacheControlHook.catalog_targets_openai_prompt_cache_breakpoint(name, "azure_ai") + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint(name, "azure_ai", model_info=model_info) for name in named_models ) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 2d185068815..f2985237554 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1420,6 +1420,7 @@ class AmazonConverseConfig(BaseConfig): block_type: Literal["system"], model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> SystemContentBlock | None: pass @@ -1433,6 +1434,7 @@ class AmazonConverseConfig(BaseConfig): block_type: Literal["content_block"], model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> ContentBlock | None: pass @@ -1445,9 +1447,10 @@ class AmazonConverseConfig(BaseConfig): block_type: Literal["system", "content_block"], model: str | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> SystemContentBlock | ContentBlock | None: cache_control: Final = message_block.get("cache_control", None) - if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model): + if cache_control is None or not bedrock_model_accepts_cache_points(model, deployment_model, model_info): return None cache_point: Final = self._build_cache_point_block(cache_control, model, deployment_model) @@ -1593,7 +1596,11 @@ class AmazonConverseConfig(BaseConfig): return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control) def _transform_system_message( - self, messages: list[AllMessageValues], model: str | None = None, deployment_model: str | None = None + self, + messages: list[AllMessageValues], + model: str | None = None, + deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> tuple[list[AllMessageValues], list[SystemContentBlock]]: leading_count: Final = next( (i for i, m in enumerate(messages) if not is_system_message(m)), @@ -1609,7 +1616,7 @@ class AmazonConverseConfig(BaseConfig): if isinstance(content, str) and content: system_content_blocks.append(SystemContentBlock(text=content)) cache_block = self.get_cache_point_block( - message, block_type="system", model=model, deployment_model=deployment_model + message, block_type="system", model=model, deployment_model=deployment_model, model_info=model_info ) if cache_block: system_content_blocks.append(cache_block) @@ -1618,7 +1625,11 @@ class AmazonConverseConfig(BaseConfig): if m.get("type") == "text" and m.get("text"): system_content_blocks.append(SystemContentBlock(text=m["text"])) cache_block = self.get_cache_point_block( - m, block_type="system", model=model, deployment_model=deployment_model + m, + block_type="system", + model=model, + deployment_model=deployment_model, + model_info=model_info, ) if cache_block: system_content_blocks.append(cache_block) @@ -1816,6 +1827,7 @@ class AmazonConverseConfig(BaseConfig): additional_request_params: dict, litellm_params: Mapping[str, object] | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> tuple[list[ToolBlock], list]: """Process tools and collect anthropic_beta values.""" bedrock_tools: list[ToolBlock] = [] @@ -1877,6 +1889,7 @@ class AmazonConverseConfig(BaseConfig): _tools_the_model_accepts(regular_tools, model, litellm_params), model=model, deployment_model=deployment_model, + model_info=model_info, ) # Add computer use tools and anthropic_beta if needed (only when computer use tools are present) @@ -1946,6 +1959,7 @@ class AmazonConverseConfig(BaseConfig): _tools_the_model_accepts(_TOOLS_AS_SENT.validate_python(filtered_tools), model, litellm_params), model=model, deployment_model=deployment_model, + model_info=model_info, ) # Append pre-formatted tools (systemTool etc.) after transformation @@ -2021,6 +2035,7 @@ class AmazonConverseConfig(BaseConfig): drop_params: bool = False, litellm_params: Mapping[str, object] | None = None, deployment_model: str | None = None, + model_info: Mapping[str, object] | None = None, ) -> CommonRequestObject: # Drop thinking param if thinking is enabled but thinking_blocks are missing # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use" @@ -2059,7 +2074,7 @@ class AmazonConverseConfig(BaseConfig): # Process tools and collect beta values bedrock_tools, anthropic_beta_list = self._process_tools_and_beta( - original_tools, model, headers, additional_request_params, litellm_params, deployment_model + original_tools, model, headers, additional_request_params, litellm_params, deployment_model, model_info ) requested_thinking: Final = additional_request_params.get("thinking") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # _prepare_request_params hands back a bare dict body_betas: Final = additional_request_params.get("anthropic_beta") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # same bare dict @@ -2077,7 +2092,7 @@ class AmazonConverseConfig(BaseConfig): if ( cache_injection_points and len(bedrock_tools) > 0 - and bedrock_model_accepts_cache_points(model, deployment_model) + and bedrock_model_accepts_cache_points(model, deployment_model, model_info) ): for point in cache_injection_points: if point.get("location") == "tool_config": @@ -2163,8 +2178,11 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict[str, object], headers: dict | None = None, ) -> RequestObject: + from litellm.integrations.anthropic_cache_control_hook import routed_model_info + base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) + model_info: Final = routed_model_info(litellm_params.get("model_info")) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( @@ -2173,6 +2191,7 @@ class AmazonConverseConfig(BaseConfig): ), model=capability_model, deployment_model=model, + model_info=model_info, ) # Convert last user message to guarded_text if guardrailConfig is present @@ -2184,6 +2203,7 @@ class AmazonConverseConfig(BaseConfig): _data: Final[CommonRequestObject] = self._transform_request_helper( model=capability_model, deployment_model=model, + model_info=model_info, system_content_blocks=system_content_blocks, optional_params=optional_params, messages=messages, @@ -2198,6 +2218,7 @@ class AmazonConverseConfig(BaseConfig): messages=messages, model=capability_model, deployment_model=model, + model_info=model_info, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), @@ -2250,8 +2271,11 @@ class AmazonConverseConfig(BaseConfig): litellm_params: dict[str, object], headers: dict | None = None, ) -> RequestObject: + from litellm.integrations.anthropic_cache_control_hook import routed_model_info + base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) + model_info: Final = routed_model_info(litellm_params.get("model_info")) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( @@ -2260,6 +2284,7 @@ class AmazonConverseConfig(BaseConfig): ), model=capability_model, deployment_model=model, + model_info=model_info, ) # Convert last user message to guarded_text if guardrailConfig is present @@ -2270,6 +2295,7 @@ class AmazonConverseConfig(BaseConfig): _data: Final[CommonRequestObject] = self._transform_request_helper( model=capability_model, deployment_model=model, + model_info=model_info, system_content_blocks=system_content_blocks, optional_params=optional_params, messages=messages, @@ -2285,6 +2311,7 @@ class AmazonConverseConfig(BaseConfig): messages=messages, model=capability_model, deployment_model=model, + model_info=model_info, llm_provider="bedrock_converse", user_continue_message=litellm_params.pop("user_continue_message", None), base_model=self._deployment_base_model(_DEPLOYMENT_PARAMS.validate_python(litellm_params)), diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 3126450b287..1f10b73505b 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -33,6 +33,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix +from litellm.litellm_core_utils.core_helpers import lax_bool from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count from litellm.litellm_core_utils.url_utils import ensure_api_route @@ -1515,7 +1516,9 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: ) -def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str | None = None) -> bool: +def bedrock_model_accepts_cache_points( + model: str | None, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: """ Whether Converse ``cachePoint`` blocks may be sent to this model. @@ -1535,43 +1538,61 @@ def bedrock_model_accepts_cache_points(model: str | None, deployment_model: str never silently degrade. ``litellm.utils.supports_prompt_caching`` is not reusable here: it returns False for unmapped models, the opposite polarity. ``deployment_model`` is the model as configured when ``model`` is its ``base_model``: its own declaration wins. + A routed call passes its deployment's ``model_info``, whose breakpoint flag wins outright and + whose ``supports_prompt_caching`` wins over the catalog's for a known model. Other flags come + from the catalog as loaded, since the registered keys are shared by every deployment of the + same model string and carry whichever sibling registered last. """ if model is None: return True deployment_declared: Final = ( - None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model) + None if deployment_model is None else bedrock_declared_cache_point_support(deployment_model, model_info) ) declared: Final = ( - deployment_declared if deployment_declared is not None else bedrock_declared_cache_point_support(model) + deployment_declared + if deployment_declared is not None + else bedrock_declared_cache_point_support(model, model_info) ) return declared is not False -def bedrock_declared_cache_point_support(model: str) -> bool | None: +def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None: if _OPENAI_FAMILY_MODEL_RE.search(model): return False + deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint") + if deployment_flag is not None: + return deployment_flag is True + catalog_entry: Final = _catalog_entry_reader(model_info) map_keys: Final = (model, get_bedrock_base_model(model)) registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/")) explicit_marker_support: Final = next( ( entry.get("supports_prompt_cache_breakpoint") is True for key in (*registered_keys, *map_keys) - if (entry := litellm.model_cost.get(key)) is not None - and entry.get("supports_prompt_cache_breakpoint") is not None + if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None ), None, ) if explicit_marker_support is not None: return explicit_marker_support - if not any(key in litellm.model_cost for key in map_keys): + if not any(catalog_entry(key) is not None for key in map_keys): return None + deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching")) + if deployment_caching is not None: + return deployment_caching return any( - entry.get("supports_prompt_caching") is True - for key in map_keys - if (entry := litellm.model_cost.get(key)) is not None + entry.get("supports_prompt_caching") is True for key in map_keys if (entry := catalog_entry(key)) is not None ) +def _catalog_entry_reader(model_info: Mapping[str, object] | None) -> Callable[[str], Mapping[str, object] | None]: + from litellm.utils import catalog_with_runtime_registrations + + if model_info is not None: + return catalog_with_runtime_registrations().get + return _model_cost_entry + + def bedrock_supports_tool_search(model: str) -> bool: """ Whether Bedrock InvokeModel admits the ``tool_search_tool_*`` tool types on ``model``. diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 4bfc9f28939..0479ac5c36c 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -556,18 +556,23 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]: """ A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on - api.openai.com, or one opted in through ``prompt_cache_options`` or the cost - map) gets each Anthropic ``cache_control`` mark rewritten in place, and under - drop_params each content part's ``prompt_cache_breakpoint`` cut down to what chat - completions accepts. Any other endpoint keeps or strips the marks as before, and - under drop_params also loses every ``prompt_cache_breakpoint`` it cannot read. + api.openai.com, or one opted in through ``prompt_cache_options``, its own + ``model_info``, or the cost map) gets each + Anthropic ``cache_control`` mark rewritten in place, and under drop_params each + content part's ``prompt_cache_breakpoint`` cut down to what chat completions + accepts. Any other endpoint keeps or strips the marks as before, and under + drop_params also loses every ``prompt_cache_breakpoint`` it cannot read. """ - from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, routed_model_info custom_llm_provider: Final = _litellm_params_str(litellm_params, "custom_llm_provider") api_base: Final = _litellm_params_str(litellm_params, "api_base") reads_breakpoints: Final = AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( - model, custom_llm_provider, api_base, prompt_cache_options + model, + custom_llm_provider, + api_base, + prompt_cache_options, + routed_model_info(litellm_params.get("model_info")), ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): return ( diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9f4a4fd6083..6bbf3598bd8 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + deployment_model_info, deployment_prompt_cache_breakpoint_support, targets_openai_api, ) @@ -767,6 +768,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): custom_llm_provider=_provider_name(self.custom_llm_provider), api_base=host.api_base or host.base_url, prompt_cache_options=prompt_cache_options, + model_info=deployment_model_info(litellm_params), ) @staticmethod diff --git a/litellm/main.py b/litellm/main.py index fc18aaa356d..45177f4223e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -755,6 +755,7 @@ async def acompletion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + routed_model_info, ) from litellm.types.llms.openai import AllMessageValues @@ -766,6 +767,7 @@ async def acompletion( tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs api_base=base_url or kwargs.get("api_base"), + model_info=routed_model_info(caller_kwargs.get("model_info")), ) prompt_id: Final = cast(str | None, kwargs.get("prompt_id", None)) # cast-ok: read from untyped kwargs @@ -5827,6 +5829,7 @@ def completion( from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + routed_model_info, ) from litellm.types.llms.openai import AllMessageValues @@ -5838,6 +5841,7 @@ def completion( tools=tools, enable_prompt_caching=cast(bool | None, kwargs.get("enable_prompt_caching")), # cast-ok: untyped kwargs api_base=base_url or kwargs.get("api_base"), + model_info=routed_model_info(cast(object, model_info)), # cast-ok: untyped kwargs ) if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 0f4ed0611be..3e605e926f7 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -52,6 +52,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( cache_control_count, dropped_cache_control, newest_marks_within_cache_control_limit, + routed_model_info, whole_messages_within_cache_control_limit, with_cache_control, ) @@ -782,13 +783,15 @@ def _replays_compaction_block(input: str | ResponseInputParam, context_managemen _PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL: Final = frozenset(("anthropic", "dashscope", "minimax", "sap", "zai")) -def _bedrock_target_keeps_item_marks(model: str, drop_params: bool) -> bool: - declared: Final = bedrock_declared_cache_point_support(model) +def _bedrock_target_keeps_item_marks(model: str, drop_params: bool, model_info: Mapping[str, object] | None) -> bool: + declared: Final = bedrock_declared_cache_point_support(model, model_info) return declared is True or (declared is None and not drop_params) -def _target_keeps_item_marks(model: str, custom_llm_provider: str | None, drop_params: bool) -> bool: - if supports_anthropic_cache_control(model, custom_llm_provider): +def _target_keeps_item_marks( + model: str, custom_llm_provider: str | None, drop_params: bool, model_info: Mapping[str, object] | None +) -> bool: + if supports_anthropic_cache_control(model, custom_llm_provider, model_info): return True if custom_llm_provider in _PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL: return True @@ -798,7 +801,7 @@ def _target_keeps_item_marks(model: str, custom_llm_provider: str | None, drop_p return is_snowflake_claude_model(model) if custom_llm_provider == "databricks": return "claude" in model.lower() - return custom_llm_provider == "bedrock" and _bedrock_target_keeps_item_marks(model, drop_params) + return custom_llm_provider == "bedrock" and _bedrock_target_keeps_item_marks(model, drop_params, model_info) def _refuses_marks_on_tool_blocks(model: str, custom_llm_provider: str | None) -> bool: @@ -1234,7 +1237,10 @@ class LiteLLMCompletionResponsesConfig: request_kwargs: Final = _STR_KEY_DICT_ADAPTER.validate_python({**kwargs, "drop_params": drop_params}) context_management: Final = responses_api_request.get("context_management") keeps_item_marks: Final = _target_keeps_item_marks( - model, custom_llm_provider, drops_unsupported_params(request_kwargs.get("drop_params")) + model, + custom_llm_provider, + drops_unsupported_params(request_kwargs.get("drop_params")), + routed_model_info(request_kwargs.get("model_info")), ) request_input: Final = with_compaction_items_as_params(input) native_compaction: Final = _replays_compaction_block( @@ -1456,7 +1462,10 @@ class LiteLLMCompletionResponsesConfig: model: Final = str(request_fields.get("model") or "") provider: Final = requested_provider if isinstance(requested_provider, str) else None keeps_item_marks: Final = _target_keeps_item_marks( - model, provider, drops_unsupported_params(request_kwargs.get("drop_params")) + model, + provider, + drops_unsupported_params(request_kwargs.get("drop_params")), + routed_model_info(request_kwargs.get("model_info")), ) lookup: Final = session_lookup or partial( ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index ad1d35bbe17..3de63538f9a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -27,6 +27,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( CARRY_UNMATCHED_MESSAGE_POINTS, AnthropicCacheControlHook, configured_injection_points, + routed_model_info, ) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind @@ -882,6 +883,7 @@ def _stamp_injection_points_for_the_hook( custom_llm_provider, kwargs.get("api_base") or kwargs.get("base_url"), kwargs.get("prompt_cache_options"), + routed_model_info(kwargs.get("model_info")), ) diff --git a/litellm/utils.py b/litellm/utils.py index 645947dcdba..108c6fae982 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4226,6 +4226,54 @@ def _register_on_current_map( _runtime_registered_model_cost: Final[dict[str, dict[str, object]]] = {} # mutable-ok: replayed on reload +_runtime_registration_order: Final[dict[str, int]] = {} # mutable-ok: tells which of a row's registered names came last +_runtime_registered_names: Final[dict[str, tuple[str, ...]]] = {} # mutable-ok: names registered onto each live row +_REGISTRATION_TICKS: Final = itertools.count() + + +@dataclass(frozen=True, slots=True) +class _RegistrationsOverCatalog(Mapping[str, Mapping[str, object]]): + catalog: Mapping[str, Mapping[str, object]] + registrations: Mapping[str, Mapping[str, object]] + registration_order: Mapping[str, int] + registered_names: Mapping[str, tuple[str, ...]] + + def __getitem__(self, key: str) -> Mapping[str, object]: + catalog_entry: Final = self.catalog.get(key) + names: Final = dict.fromkeys((*self.registered_names.get(key, ()), key)) + present: Final = tuple(name for name in names if name in self.registrations) + if not present: + return self.catalog[key] + oldest_first: Final = sorted(present, key=lambda name: self.registration_order.get(name, -1)) + registered: Final = (self.registrations[name] for name in oldest_first) + return MappingProxyType( + dict(itertools.chain.from_iterable(entry.items() for entry in (catalog_entry or {}, *registered))) + ) + + def __iter__(self) -> Iterator[str]: + return iter({**dict.fromkeys(self.catalog), **dict.fromkeys(self.registrations)}) + + def __len__(self) -> int: + return len(self.catalog.keys() | self.registrations.keys()) + + +def _record_registered_name(written_key: str, registered_key: str) -> None: + names: Final = _runtime_registered_names.get(written_key, ()) + if registered_key not in names: + _runtime_registered_names[written_key] = (*names, registered_key) + + +def catalog_with_runtime_registrations() -> Mapping[str, Mapping[str, object]]: + """The loaded catalog plus ``register_model`` calls, without what routers register for their deployments.""" + catalog: Final = GetModelCostMap.loaded_model_cost_map() + if not _runtime_registered_model_cost: + return catalog + return _RegistrationsOverCatalog( + catalog, + MappingProxyType(_runtime_registered_model_cost), + MappingProxyType(_runtime_registration_order), + MappingProxyType(_runtime_registered_names), + ) def runtime_model_cost_registrations( @@ -4291,7 +4339,14 @@ def reapply_runtime_model_cost_registrations() -> None: if _LiveDeploymentReplay.callback is not None: _LiveDeploymentReplay.callback() if _runtime_registered_model_cost: - register_model(model_cost=dict(_runtime_registered_model_cost)) + register_model( + model_cost=dict( + sorted( + _runtime_registered_model_cost.items(), + key=lambda registration: _runtime_registration_order.get(registration[0], -1), + ) + ) + ) def cost_map_omits_token_price(*keys: object) -> bool: @@ -4365,9 +4420,12 @@ def register_model( dict(_runtime_registered_model_cost.get(_registered_key, {})), _RATE_TABLE.validate_python(_registered_value), ) + _runtime_registration_order[_registered_key] = next(_REGISTRATION_TICKS) for key, value in loaded_model_cost.items(): write = _register_on_current_map(str(key), value, custom_llm_provider, warning_display_name) + if persist_across_reloads: + _record_registered_name(write.key, str(key)) # Invalidate case-insensitive lookup map since model_cost was modified _invalidate_model_cost_lowercase_map() diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 1b1ee484afc..57ae9f1905f 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -138,6 +138,7 @@ ignored_function_names = [ "_resolve_pre_call_check_input_tokens", # Tested through the token-count failure pre-call check tests in test_router.py "_params_neither_dropped_nor_allowed", # Tested through the allowed-or-dropped param pre-call check tests in test_router.py "_deployment_rejects_params", # Tested through the unsupported-param pre-call check tests in test_router.py + "_cache_flags_left_to_the_catalog", # Tested through the wildcard prompt cache flag tests in test_anthropic_cache_control_hook.py "_deployment_lacks_params", # Tested through the unsupported-param pre-call check tests in test_router.py "_pre_call_check_requested_chat_params", # Tested through the unsupported-param pre-call check tests in test_router.py "_deployment_supports_param", # Tested through the deployment-capability pre-call check tests in test_router.py diff --git a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py index a6714979c8e..05e08c5fc27 100644 --- a/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py +++ b/tests/integration/providers/test_bedrock_kimi_k3_cache_point_wire.py @@ -1251,17 +1251,19 @@ def test_s01_to_s06_an_odd_typed_flag_is_read_as_false_never_as_true( @pytest.mark.timeout(600) -def test_s10_s11_two_deployments_of_one_arn_share_the_flag_and_a_delete_leaves_it_in_place(gateway: Gateway) -> None: +def test_s10_s11_two_deployments_of_one_arn_keep_their_own_flags_and_a_delete_changes_neither( + gateway: Gateway, +) -> None: arn: Final = _arn("shared") with wire_server(_peer()) as wire, gateway.scenario() as scenario: flagged: Final = _deployment( gateway, scenario, wire, f"bedrock/{arn}", {"supports_prompt_cache_breakpoint": False}, cleanup=False ) plain: Final = _deployment(gateway, scenario, wire, f"bedrock/{arn}") - for name in (flagged, plain): + for name, cache_points in ((flagged, 0), (plain, 2)): outcome, received = _observe_at(gateway, wire, "chat", _chat_body(name, _prompt(), _SYSTEM_AND_USER)) _assert_answered(outcome) - assert received.cache_points == 0, (name, received.body) + assert received.cache_points == cache_points, (name, received.body) _success_row(outcome.response_id) flagged_identity: Final = _model_id(flagged) gateway.post("/model/delete", {"id": flagged_identity}) @@ -1276,7 +1278,7 @@ def test_s10_s11_two_deployments_of_one_arn_share_the_flag_and_a_delete_leaves_i wire.drain() after, after_received = _observe_at(gateway, wire, "chat", _chat_body(plain, _prompt(), _SYSTEM_AND_USER)) _assert_answered(after) - assert after_received.cache_points == 0, after_received.body + assert after_received.cache_points == 2, after_received.body _success_row(after.response_id) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 9e40e128952..9313e3d7039 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -5482,6 +5482,38 @@ def test_transform_request_strips_the_breakpoint_when_the_served_providers_row_o ] +@pytest.mark.parametrize( + ("sibling_flag", "model_info", "keeps_the_breakpoint"), + [ + (True, {"id": "flagless"}, False), + (True, {"id": "opted-out", "supports_prompt_cache_breakpoint": False}, False), + (False, {"id": "opted-in", "supports_prompt_cache_breakpoint": True}, True), + ], + ids=["flagless", "opted-out", "opted-in"], +) +def test_transform_request_reads_the_routed_deployments_own_flag_not_a_siblings_on_the_shared_key( + monkeypatch: pytest.MonkeyPatch, + sibling_flag: bool, + model_info: Mapping[str, object], + keeps_the_breakpoint: bool, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "bedrock_mantle/qa-mantle-shared", + {**_MANTLE_GPT_ROW, "supports_prompt_cache_breakpoint": sibling_flag}, + ) + + system_content: Final = _bridged_input_for_served_provider( + "qa-mantle-shared", "bedrock_mantle", model_info=model_info + )[0]["content"] + + assert system_content == [ + {**_MARKED_SYSTEM_PART, "type": "input_text"} + if keeps_the_breakpoint + else {"type": "input_text", "text": "Stable prefix"} + ] + + def test_transform_request_strips_the_breakpoint_for_a_served_provider_without_a_row_of_its_own( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 0a6f37e4c15..c040d8e5bac 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -8,14 +8,18 @@ import subprocess import sys import textwrap from collections.abc import Mapping +from types import MappingProxyType from typing import Final, List, Optional, Tuple, cast from unittest.mock import MagicMock, patch import pytest +import respx +from httpx import Response from openai.types.chat import ChatCompletionMessage from pydantic import BaseModel, ConfigDict import litellm +from litellm import Router from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import ( CARRY_UNMATCHED_MESSAGE_POINTS, @@ -5055,6 +5059,635 @@ def test_cache_control_hook_injects_next_to_a_malformed_message_instead_of_raisi ] +_SHARED_FOUNDRY_MODEL: Final = "azure_ai/qa-shared-gpt-deployment" +_SIBLING_BREAKPOINT_FLAGS: Final = MappingProxyType( + { + "opted-in": MappingProxyType({"supports_prompt_cache_breakpoint": True}), + "opted-out": MappingProxyType({"supports_prompt_cache_breakpoint": False}), + } +) +_FOUNDRY_CHAT_ROUTE: Final = ( + "/models/chat/completions", + MappingProxyType( + { + "id": "chatcmpl-azure-ai", + "object": "chat.completion", + "created": 1700000000, + "model": "qa-shared-gpt-deployment", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + } + ), +) +_FOUNDRY_RESPONSES_ROUTE: Final = ( + "/openai/v1/responses", + MappingProxyType( + { + "id": "resp_azure_ai", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "qa-shared-gpt-deployment", + "output": [ + { + "type": "message", + "id": "msg_azure_ai", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + } + ), +) + + +def _foundry_siblings_sharing_one_model_string( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + registration_order: tuple[str, ...], + upstream: tuple[str, Mapping[str, object]], +) -> tuple[Router, Mapping[str, respx.Route]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + path, reply = upstream + routes: Final = MappingProxyType( + { + host: respx_mock.post(host=f"{host}.services.ai.azure.com", path=path).mock( + return_value=Response(200, json=dict(reply)) + ) + for host in registration_order + } + ) + router: Final = Router( + model_list=[ + { + "model_name": host, + "litellm_params": { + "model": _SHARED_FOUNDRY_MODEL, + "api_base": f"https://{host}.services.ai.azure.com/models", + "api_key": "azure-ai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": f"{host}-id", **_SIBLING_BREAKPOINT_FLAGS.get(host, {})}, + } + for host in registration_order + ], + num_retries=0, + ) + return router, routes + + +async def _send_unmarked_system_prompt(router: Router, host: str, entrypoint: str) -> None: + chat_messages: Final = [{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + match entrypoint: + case "completion": + router.completion(model=host, messages=chat_messages) + case "acompletion": + await router.acompletion(model=host, messages=chat_messages) + case "aanthropic_messages": + await router.aanthropic_messages( + model=host, system="stable", messages=[{"role": "user", "content": "hi"}], max_tokens=16 + ) + case "aresponses": + await router.aresponses(model=host, input=chat_messages) + + +@pytest.mark.parametrize( + ("entrypoint", "upstream"), + [ + ("completion", _FOUNDRY_CHAT_ROUTE), + ("acompletion", _FOUNDRY_CHAT_ROUTE), + ("aanthropic_messages", _FOUNDRY_CHAT_ROUTE), + ("aresponses", _FOUNDRY_RESPONSES_ROUTE), + ], + ids=["completion", "acompletion", "aanthropic_messages", "aresponses"], +) +@pytest.mark.parametrize( + "registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")] +) +async def test_foundry_deployment_reads_only_its_own_prompt_cache_flag_not_a_siblings_sharing_its_model_string( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + registration_order: tuple[str, ...], + entrypoint: str, + upstream: tuple[str, Mapping[str, object]], +) -> None: + router, routes = _foundry_siblings_sharing_one_model_string(respx_mock, monkeypatch, registration_order, upstream) + + for host in registration_order: + await _send_unmarked_system_prompt(router, host, entrypoint) + + def sent_prompt_caching(host: str) -> tuple[object, bool]: + body: Final = routes[host].calls.last.request.content + return json.loads(body).get("prompt_cache_options"), b"prompt_cache_breakpoint" in body + + assert {host: sent_prompt_caching(host) for host in registration_order} == { + "opted-in": ({"mode": "implicit"}, True), + "opted-out": (None, False), + "unflagged": (None, False), + } + + +@pytest.mark.parametrize( + ("model", "sibling_flag", "expected"), + [("qa-openai-shared", True, False), ("gpt-99.0-qa-shared", False, True)], + ids=["sibling-opt-in-does-not-turn-it-on", "sibling-opt-out-does-not-turn-it-off"], +) +def test_openai_deployment_reads_the_catalog_not_a_siblings_flag_on_the_shared_key( + monkeypatch: pytest.MonkeyPatch, model: str, sibling_flag: bool, expected: bool +) -> None: + sibling_entry: Final = { + "litellm_provider": "openai", + "mode": "chat", + "supports_prompt_cache_breakpoint": sibling_flag, + } + monkeypatch.setitem(litellm.model_cost, model, sibling_entry) + + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "openai", "https://api.openai.com/v1", model_info={"id": "flagless-deployment"} + ) + is expected + ) + + +@pytest.mark.parametrize( + ("model", "declared", "api_base", "expected"), + [ + ("qa-openai-shared", True, "https://api.openai.com/v1", True), + ("qa-openai-shared", True, "https://compatible-host.example.com/v1", False), + ("gpt-99.0-qa-shared", False, "https://api.openai.com/v1", False), + ], + ids=["opt-in-on-openai", "opt-in-on-a-compatible-host-stays-off", "opt-out-beats-the-version-rule"], +) +def test_openai_deployment_declared_flag_answers_support_while_the_host_still_decides( + model: str, declared: bool, api_base: str, expected: bool +) -> None: + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "openai", api_base, model_info={"supports_prompt_cache_breakpoint": declared} + ) + is expected + ) + + +@pytest.mark.parametrize( + "registration_order", [("opted-in", "opted-out", "unflagged"), ("unflagged", "opted-out", "opted-in")] +) +async def test_foundry_responses_drop_params_keeps_a_client_breakpoint_by_the_deployments_own_flag( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, ...] +) -> None: + router, routes = _foundry_siblings_sharing_one_model_string( + respx_mock, monkeypatch, registration_order, _FOUNDRY_RESPONSES_ROUTE + ) + client_marked_system_prompt: Final = { + "role": "system", + "content": [{"type": "input_text", "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + + for host in registration_order: + await router.aresponses( + model=host, input=[client_marked_system_prompt, {"role": "user", "content": "hi"}], drop_params=True + ) + + assert { + host: b"prompt_cache_breakpoint" in routes[host].calls.last.request.content for host in registration_order + } == { + "opted-in": True, + "opted-out": False, + "unflagged": True, + } + + +@pytest.mark.parametrize( + ("entrypoint", "part_type", "reply"), + [("acompletion", "text", _FOUNDRY_CHAT_ROUTE[1]), ("aresponses", "input_text", _FOUNDRY_RESPONSES_ROUTE[1])], + ids=["chat", "responses"], +) +@pytest.mark.parametrize("registration_order", [("opted-in", "unflagged"), ("unflagged", "opted-in")]) +async def test_azure_drop_params_keeps_a_client_breakpoint_by_the_deployments_own_flag_not_a_siblings( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + registration_order: tuple[str, ...], + entrypoint: str, + part_type: str, + reply: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + routes: Final = MappingProxyType( + { + host: respx_mock.post(host=f"{host}.openai.azure.com").mock(return_value=Response(200, json=dict(reply))) + for host in registration_order + } + ) + router: Final = Router( + model_list=[ + { + "model_name": host, + "litellm_params": { + "model": "azure/gpt-4.1-qa-shared", + "api_base": f"https://{host}.openai.azure.com", + "api_key": "azure-test-key", + "api_version": "2025-04-01-preview", + }, + "model_info": {"id": f"{host}-id", **_SIBLING_BREAKPOINT_FLAGS.get(host, {})}, + } + for host in registration_order + ], + num_retries=0, + ) + marked_system_prompt: Final = { + "role": "system", + "content": [{"type": part_type, "text": "stable", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + request: Final = [marked_system_prompt, {"role": "user", "content": "hi"}] + + for host in registration_order: + match entrypoint: + case "acompletion": + await router.acompletion(model=host, messages=request, drop_params=True) + case "aresponses": + await router.aresponses(model=host, input=request, drop_params=True) + + assert { + host: b"prompt_cache_breakpoint" in routes[host].calls.last.request.content for host in registration_order + } == {"opted-in": True, "unflagged": False} + + +@pytest.mark.parametrize("registration_order", [("opted-in", "unflagged"), ("unflagged", "opted-in")]) +async def test_openai_messages_bridge_sends_a_breakpoint_by_the_deployments_own_flag_not_a_siblings( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, ...] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + _, reply = _FOUNDRY_RESPONSES_ROUTE + responses_route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=Response(200, json=dict(reply)) + ) + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": "openai/qa-shared-gpt-deployment", "api_key": "openai-test-key"}, + "model_info": {"id": f"{name}-id", **_SIBLING_BREAKPOINT_FLAGS.get(name, {})}, + } + for name in registration_order + ], + num_retries=0, + ) + + async def sends_a_breakpoint(name: str) -> bool: + await router.aanthropic_messages( + model=name, + system=[{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}], + messages=[{"role": "user", "content": "hi"}], + max_tokens=16, + ) + return b"prompt_cache_breakpoint" in responses_route.calls.last.request.content + + assert {name: await sends_a_breakpoint(name) for name in registration_order} == { + "opted-in": True, + "unflagged": False, + } + + +_OPENAI_ROUTES: Final = MappingProxyType( + { + "acompletion": ("https://api.openai.com/v1/chat/completions", _FOUNDRY_CHAT_ROUTE[1]), + "aresponses": ("https://api.openai.com/v1/responses", _FOUNDRY_RESPONSES_ROUTE[1]), + "aanthropic_messages": ("https://api.openai.com/v1/responses", _FOUNDRY_RESPONSES_ROUTE[1]), + } +) + + +def _openai_route(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str) -> respx.Route: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + url, reply = _OPENAI_ROUTES[entrypoint] + return respx_mock.post(url).mock(return_value=Response(200, json=dict(reply))) + + +@pytest.mark.parametrize("declared", [True, False]) +async def test_a_wildcard_deployments_prompt_cache_flag_leaves_each_matched_model_to_the_catalog( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, declared: bool +) -> None: + chat: Final = _openai_route(respx_mock, monkeypatch, "acompletion") + router: Final = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "openai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "wildcard-id", "supports_prompt_cache_breakpoint": declared}, + } + ], + num_retries=0, + ) + + async def sends_prompt_cache_options(model: str) -> bool: + await _send_unmarked_system_prompt(router, model, "acompletion") + return b"prompt_cache_options" in chat.calls.last.request.content + + assert {model: await sends_prompt_cache_options(model) for model in ("openai/gpt-4.1-qa", "openai/gpt-99.0-qa")} == { + "openai/gpt-4.1-qa": False, + "openai/gpt-99.0-qa": True, + } + + +async def test_an_azure_ai_wildcard_deployments_breakpoint_flag_still_speaks_for_each_matched_model( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + path, reply = _FOUNDRY_CHAT_ROUTE + chat: Final = respx_mock.post(host="qa-wildcard.services.ai.azure.com", path=path).mock( + return_value=Response(200, json=dict(reply)) + ) + router: Final = Router( + model_list=[ + { + "model_name": "azure_ai/*", + "litellm_params": { + "model": "azure_ai/*", + "api_base": "https://qa-wildcard.services.ai.azure.com/models", + "api_key": "azure-ai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "wildcard-id", "supports_prompt_cache_breakpoint": True}, + } + ], + num_retries=0, + ) + + await _send_unmarked_system_prompt(router, "azure_ai/qa-unlisted-alias", "acompletion") + + body: Final = chat.calls.last.request.content + assert (json.loads(body).get("prompt_cache_options"), b"prompt_cache_breakpoint" in body) == ( + {"mode": "implicit"}, + True, + ) + + +@pytest.mark.parametrize("entrypoint", ["acompletion", "aresponses", "aanthropic_messages"]) +@pytest.mark.parametrize("declared_in", ["litellm_params", "model_info"]) +async def test_a_base_model_that_takes_breakpoints_leaves_an_openai_deployments_request_as_it_was( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, declared_in: str +) -> None: + upstream: Final = _openai_route(respx_mock, monkeypatch, entrypoint) + + def deployment(name: str, base_model: Mapping[str, str]) -> dict[str, object]: + return { + "model_name": name, + "litellm_params": { + "model": "openai/gpt-4.1-qa", + "api_key": "openai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + **(base_model if declared_in == "litellm_params" else {}), + }, + "model_info": {"id": f"{name}-id", **(base_model if declared_in == "model_info" else {})}, + } + + router: Final = Router( + model_list=[deployment("plain", {}), deployment("priced-as-breakpoint-model", {"base_model": "gpt-99.0-qa"})], + num_retries=0, + ) + + async def sent_body(name: str) -> object: + await _send_unmarked_system_prompt(router, name, entrypoint) + return json.loads(upstream.calls.last.request.content) + + assert await sent_body("priced-as-breakpoint-model") == await sent_body("plain") + + +def test_a_routed_call_honors_a_register_model_breakpoint_opt_out_made_under_the_provider_prefixed_name() -> None: + from litellm.utils import catalog_with_runtime_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "openai" and entry.get("supports_prompt_cache_breakpoint") is True + ) + litellm.register_model({f"openai/{model}": {"supports_prompt_cache_breakpoint": False}}) + + assert supports_openai_prompt_cache_breakpoint(model, "openai", {"id": "plain-id"}) is False + + +_APPLICATION_INFERENCE_PROFILE: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/qa" + + +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected"), + [ + ("us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("converse/us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("invoke/us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("converse_like/us.anthropic.claude-sonnet-5-5", "bedrock", False), + ("openai/gpt-oss-120b", "bedrock", False), + ("moonshotai.kimi-k3", "bedrock", False), + (_APPLICATION_INFERENCE_PROFILE, "bedrock", False), + ("claude-sonnet-5-5", "anthropic", False), + ("claude-sonnet-5-5", "azure_ai", False), + ("claude-sonnet-5-5", "vertex_ai", False), + ("gemini-3-pro", "gemini", False), + ("qa-custom-model", "qa-custom-handler", False), + ("qa-shared-gpt-deployment", "azure_ai", True), + ("my-custom-deployment", "azure", True), + ("qa-mantle-model", "bedrock_mantle", True), + ("anthropic/qa-gateway-claude", "openrouter", False), + ("qa-downstream-claude", "litellm_proxy", False), + ("qa-self-hosted-model", "hosted_vllm", False), + ], +) +def test_a_declared_breakpoint_flag_picks_openais_dialect_only_where_the_request_goes_out_openai_shaped( + model: str, custom_llm_provider: str, expected: bool +) -> None: + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, custom_llm_provider, model_info={"supports_prompt_cache_breakpoint": True} + ) + is expected + ) + + +def test_a_gateway_deployment_declaring_the_flag_picks_openais_dialect_for_a_model_the_cost_map_lists_there() -> None: + litellm.register_model({"openrouter/qa-listed-gpt": {"litellm_provider": "openrouter", "mode": "chat"}}) + + assert [ + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "openrouter", model_info={"supports_prompt_cache_breakpoint": True} + ) + for model in ("qa-listed-gpt", "qa-unlisted-gpt") + ] == [True, False] + + +def test_a_gateway_deployment_declaring_the_flag_picks_openais_dialect_when_its_model_info_lists_that_provider() -> None: + assert [ + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + "qa-self-hosted-gpt", + "hosted_vllm", + model_info={"supports_prompt_cache_breakpoint": True, "litellm_provider": listed_provider}, + ) + for listed_provider in ("hosted_vllm", "openrouter") + ] == [True, False] + +def test_a_custom_handler_provider_declaring_the_flag_keeps_the_anthropic_mark() -> None: + litellm.register_model( + {"qa-custom-handler/qa-model": {"litellm_provider": "qa-custom-handler", "mode": "chat"}} + ) + + assert ( + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + "qa-model", "qa-custom-handler", model_info={"supports_prompt_cache_breakpoint": True} + ) + is False + ) + + +def test_a_routed_deployment_without_its_own_flag_reads_a_register_model_opt_in_but_not_a_siblings() -> None: + litellm.register_model( + {"azure_ai/qa-registered-gpt": {"litellm_provider": "azure_ai", "supports_prompt_cache_breakpoint": True}} + ) + Router( + model_list=[ + { + "model_name": "qa-opted-in-sibling", + "litellm_params": {"model": "azure_ai/qa-sibling-gpt", "api_base": "https://qa.services.ai.azure.com"}, + "model_info": {"id": "qa-opted-in-sibling-id", "supports_prompt_cache_breakpoint": True}, + } + ] + ) + + assert [ + AnthropicCacheControlHook.targets_openai_prompt_cache_breakpoint( + model, "azure_ai", model_info={"id": "qa-flagless-id"} + ) + for model in ("qa-registered-gpt", "qa-sibling-gpt") + ] == [True, False] + + +@pytest.mark.parametrize( + ("entrypoint", "injection_points"), + [ + ("completion", [{"location": "message", "role": "system"}]), + ("acompletion", [{"location": "message", "role": "system"}]), + ("aresponses", [{"location": "message", "role": "system"}]), + ("aresponses-client-marked", None), + ], +) +async def test_an_azure_custom_deployment_declaring_the_breakpoint_flag_gets_openais_breakpoint( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + entrypoint: str, + injection_points: list[dict[str, str]] | None, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + azure: Final = respx_mock.post(host="qa-azure.openai.azure.com").mock( + side_effect=lambda request: Response( + 200, + json=dict( + _FOUNDRY_RESPONSES_ROUTE[1] if request.url.path.endswith("/responses") else _FOUNDRY_CHAT_ROUTE[1] + ), + ) + ) + deployment_params: Final = { + "model": "azure/my-custom-deployment", + "api_base": "https://qa-azure.openai.azure.com", + "api_key": "azure-test-key", + "api_version": "2025-04-01-preview", + } + router: Final = Router( + model_list=[ + { + "model_name": "custom-azure", + "litellm_params": ( + deployment_params + if injection_points is None + else {**deployment_params, "cache_control_injection_points": injection_points} + ), + "model_info": {"id": "custom-azure-id", "supports_prompt_cache_breakpoint": True}, + } + ], + num_retries=0, + ) + client_marked_system_prompt: Final = { + "role": "system", + "content": [{"type": "input_text", "text": "stable", "cache_control": {"type": "ephemeral"}}], + } + + match entrypoint: + case "aresponses-client-marked": + await router.aresponses( + model="custom-azure", input=[client_marked_system_prompt, {"role": "user", "content": "hi"}] + ) + case _: + await _send_unmarked_system_prompt(router, "custom-azure", entrypoint) + + body: Final = azure.calls.last.request.content + assert (b"prompt_cache_breakpoint" in body, b"cache_control" in body) == (True, False) + + +_BEDROCK_CONVERSE_REPLY: Final = MappingProxyType( + { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 1, "totalTokens": 6}, + } +) +_BEDROCK_INVOKE_REPLY: Final = MappingProxyType( + { + "id": "msg_bedrock", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 1}, + } +) + + +@pytest.mark.parametrize( + ("entrypoint", "expected_system"), + [ + ("completion", [{"text": "stable"}, {"cachePoint": {"type": "default"}}]), + ("acompletion", [{"text": "stable"}, {"cachePoint": {"type": "default"}}]), + ("aanthropic_messages", [{"type": "text", "text": "stable", "cache_control": {"type": "ephemeral"}}]), + ], +) +async def test_a_bedrock_claude_deployment_declaring_the_breakpoint_flag_keeps_bedrocks_own_cache_marks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, entrypoint: str, expected_system: list[object] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + bedrock: Final = respx_mock.post(host="bedrock-runtime.us-east-1.amazonaws.com").mock( + side_effect=lambda request: Response( + 200, + json=dict(_BEDROCK_INVOKE_REPLY if request.url.path.endswith("/invoke") else _BEDROCK_CONVERSE_REPLY), + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "claude-on-bedrock", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-sonnet-5-5", + "aws_access_key_id": "bedrock-test-key", + "aws_secret_access_key": "bedrock-test-secret", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "claude-on-bedrock-id", "supports_prompt_cache_breakpoint": True}, + } + ], + num_retries=0, + ) + + await _send_unmarked_system_prompt(router, "claude-on-bedrock", entrypoint) + + body: Final = bedrock.calls.last.request.content + assert (json.loads(body)["system"], b"prompt_cache" in body) == (expected_system, False) + + def _search_conversation_with(follow_up: dict[str, object]) -> list[AllMessageValues]: return [ cast(AllMessageValues, {"role": "user", "content": "Weather in Paris?"}), diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index cf0b8a39865..0ee5d52d50d 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -10,6 +10,7 @@ Verifies that: - ANTHROPIC_API_KEY / ANTHROPIC_API_BASE take precedence over their aliases. """ +import itertools import json import logging import os @@ -4728,3 +4729,142 @@ def test_tools_without_strict_drops_only_the_strict_key_and_counts_the_strict_to assert tools_without_strict(tools) == ToolsWithoutStrict( tools=({"name": "a"}, {"name": "b"}, {"name": "c"}), dropped_count=1 ) + + +@pytest.mark.parametrize("registration_order", [("caching-off", "unflagged"), ("unflagged", "caching-off")]) +def test_auto_injection_reads_a_deployments_own_prompt_caching_flag_not_a_siblings( + registration_order: tuple[str, ...], +) -> None: + from litellm import Router + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + + model: Final = "anthropic.claude-qa-cache-model" + litellm.register_model({model: {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_caching": True}}) + model_infos: Final = { + "caching-off": {"id": "caching-off-id", "supports_prompt_caching": False}, + "unflagged": {"id": "unflagged-id"}, + } + Router( + model_list=[ + { + "model_name": name, + "litellm_params": {"model": f"bedrock/{model}", "aws_region_name": "us-east-1"}, + "model_info": model_infos[name], + } + for name in registration_order + ] + ) + + assert {name: supports_anthropic_cache_control(model, "bedrock", model_infos[name]) for name in registration_order} == { + "caching-off": False, + "unflagged": True, + } + + +def test_auto_injection_leaves_an_unmapped_claude_deployment_uncached_instead_of_raising() -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + + assert supports_anthropic_cache_control("claude-qa-unmapped-model", "anthropic", {"id": "unmapped-id"}) is False + + +def test_auto_injection_honors_a_register_model_opt_out_made_under_the_provider_prefixed_name() -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "anthropic" and entry.get("supports_prompt_caching") is True + ) + litellm.register_model({f"anthropic/{model}": {"supports_prompt_caching": False}}) + + assert supports_anthropic_cache_control(model, "anthropic", {"id": "plain-id"}) is False + + +@pytest.mark.parametrize("route_prefix", ["bedrock/", "bedrock/converse/"]) +def test_auto_injection_honors_a_register_model_opt_out_made_under_a_bedrock_route_prefixed_name(route_prefix: str) -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "bedrock_converse" + and entry.get("supports_prompt_caching") is True + and key.startswith("us.anthropic.claude") + ) + litellm.register_model({f"{route_prefix}{model}": {"supports_prompt_caching": False}}) + + assert ( + supports_anthropic_cache_control(model, "bedrock", {"id": "plain-id"}), + supports_anthropic_cache_control(model, "bedrock"), + ) == (False, False) + + +@pytest.mark.parametrize( + ("declared", "catalog_caching", "expected"), + [("true", False, True), (1, False, True), ("false", True, False), (0, True, False), ("maybe", True, True)], +) +def test_auto_injection_reads_a_string_or_numeric_deployment_prompt_caching_flag_as_the_bool_it_spells( + declared: object, catalog_caching: bool, expected: bool +) -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + + model: Final = f"claude-qa-lax-flag-model-{catalog_caching}" + litellm.register_model( + {model: {"litellm_provider": "anthropic", "mode": "chat", "supports_prompt_caching": catalog_caching}} + ) + + assert ( + supports_anthropic_cache_control(model, "anthropic", {"id": "lax-id", "supports_prompt_caching": declared}) + is expected + ) + + +@pytest.mark.parametrize("reload_cost_map", [False, True], ids=["before-reload", "after-reload"]) +@pytest.mark.parametrize( + ("registrations", "expected"), + [ + ((("", True), ("anthropic/", False)), False), + ((("anthropic/", False), ("", True)), True), + ((("", False), ("anthropic/", True), ("", False)), False), + ], + ids=["prefixed-opt-out-last", "bare-opt-in-last", "bare-opt-out-registered-again-last"], +) +def test_the_later_of_a_bare_and_a_prefixed_register_model_entry_wins_for_routed_and_sdk_calls( + registrations: tuple[tuple[str, bool], ...], expected: bool, reload_cost_map: bool +) -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations, reapply_runtime_model_cost_registrations + + model: Final = next( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "anthropic" and entry.get("supports_prompt_caching") is True + ) + for prefix, flag in registrations: + litellm.register_model({f"{prefix}{model}": {"supports_prompt_caching": flag}}) + if reload_cost_map: + reapply_runtime_model_cost_registrations() + + assert ( + supports_anthropic_cache_control(model, "anthropic", {"id": "plain-id"}), + supports_anthropic_cache_control(model, "anthropic"), + ) == (expected, expected) + + +def test_a_register_model_opt_out_for_one_claude_model_leaves_every_other_routed_models_catalog_flag() -> None: + from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control + from litellm.utils import catalog_with_runtime_registrations + + opted_out, untouched = itertools.islice( + ( + key + for key, entry in catalog_with_runtime_registrations().items() + if entry.get("litellm_provider") == "anthropic" and entry.get("supports_prompt_caching") is True + ), + 2, + ) + litellm.register_model({f"anthropic/{opted_out}": {"supports_prompt_caching": False}}) + + assert supports_anthropic_cache_control(untouched, "anthropic", {"id": "plain-id"}) is True diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index dd29ee83bc6..f93968eb2e6 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -917,7 +917,7 @@ def test_azure_sends_part_breakpoints_as_written_without_drop_params( [ ("unit-breakpoint-ignorer", {"drop_params": True}), ("unit-breakpoint-reader", {"drop_params": True, "base_model": "azure/unit-breakpoint-ignorer"}), - ("my-azure-alias", {"drop_params": True, "model_info": {"base_model": "azure/unit-breakpoint-ignorer"}}), + ("my-azure-alias", {"drop_params": True, "model_info": {"base_model": "azure/gpt-4.1"}}), ], ids=("known-non-reader", "base-model-known-non-reader", "model-info-base-model-known-non-reader"), ) diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 38c69e0ce27..80715105b7f 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -1,5 +1,7 @@ import asyncio, importlib, litellm, pytest +import respx +from httpx import Response from typing import Final @@ -581,6 +583,188 @@ def test_deployment_model_info_breakpoint_flag_covers_an_unmapped_arn(local_mode assert bedrock_model_accepts_cache_points(unflagged_arn) is True + +_CONVERSE_REPLY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 5, "outputTokens": 1, "totalTokens": 6}, +} +_CACHE_MARK: Final = {"type": "ephemeral"} +_MARKED_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}, + "cache_control": _CACHE_MARK, +} +_COMPUTER_TOOL: Final = { + "type": "computer_20250124", + "name": "computer", + "display_width_px": 1024, + "display_height_px": 768, +} +_MARKED_TOOL_CONVERSATION: Final = ( + {"role": "system", "content": "stable"}, + {"role": "user", "content": [{"type": "text", "text": "Weather in Paris and Rome?", "cache_control": _CACHE_MARK}]}, + { + "role": "assistant", + "content": [{"type": "text", "text": "Checking both.", "cache_control": _CACHE_MARK}], + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}{"city": "Rome"}'}, + "cache_control": _CACHE_MARK, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "Sunny", "cache_control": _CACHE_MARK}]}, + {"role": "tool", "tool_call_id": "call_1_1", "content": "Cloudy"}, + {"role": "assistant", "content": "Paris is sunny, Rome is cloudy.", "cache_control": _CACHE_MARK}, + {"role": "user", "content": "Thanks"}, +) + + +@pytest.mark.parametrize( + ("model", "flagged", "declared", "tools"), + [ + pytest.param("moonshotai.kimi-k3", "opted-in", True, (_MARKED_WEATHER_TOOL,), id="kimi-opted-in"), + pytest.param( + "us.anthropic.claude-sonnet-5-5", + "opted-out", + False, + (_COMPUTER_TOOL, _MARKED_WEATHER_TOOL), + id="claude-opted-out-next-to-computer-use", + ), + ], +) +@pytest.mark.parametrize("entrypoint", ["completion", "acompletion"]) +@pytest.mark.parametrize("flagless_registers_first", [False, True]) +async def test_a_bedrock_deployment_sends_cache_points_by_its_own_flag_not_a_siblings_on_the_same_model( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + local_model_cost_map: None, + model: str, + flagged: str, + declared: bool, + tools: tuple[dict[str, object], ...], + entrypoint: str, + flagless_registers_first: bool, +) -> None: + from litellm import Router + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + converse: Final = respx_mock.post(f"https://bedrock-runtime.us-east-1.amazonaws.com/model/{model}/converse").mock( + return_value=Response(200, json=_CONVERSE_REPLY) + ) + flags: Final = {flagged: {"supports_prompt_cache_breakpoint": declared}} + registration_order: Final = ("flagless", flagged) if flagless_registers_first else (flagged, "flagless") + router: Final = Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": f"bedrock/{model}", + "aws_access_key_id": "bedrock-test-key", + "aws_secret_access_key": "bedrock-test-secret", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": f"{name}-id", **flags.get(name, {})}, + } + for name in registration_order + ], + num_retries=0, + ) + messages: Final = [dict(message) for message in _MARKED_TOOL_CONVERSATION] + + async def sends_a_cache_point(name: str) -> bool: + match entrypoint: + case "completion": + router.completion(model=name, messages=messages, tools=list(tools)) + case "acompletion": + await router.acompletion(model=name, messages=messages, tools=list(tools)) + return b"cachePoint" in converse.calls.last.request.content + + assert {name: await sends_a_cache_point(name) for name in registration_order} == { + flagged: declared, + "flagless": not declared, + } + +async def test_a_routed_bedrock_deployment_sends_cache_points_when_register_model_opted_its_model_in( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + from litellm import Router + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + converse: Final = respx_mock.post( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/moonshotai.kimi-k3/converse" + ).mock(return_value=Response(200, json=_CONVERSE_REPLY)) + litellm.register_model({"bedrock/moonshotai.kimi-k3": {"supports_prompt_cache_breakpoint": True}}) + router: Final = Router( + model_list=[ + { + "model_name": "flagless", + "litellm_params": { + "model": "bedrock/moonshotai.kimi-k3", + "aws_access_key_id": "bedrock-test-key", + "aws_secret_access_key": "bedrock-test-secret", + "aws_region_name": "us-east-1", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": "flagless-id"}, + } + ], + num_retries=0, + ) + + await router.acompletion(model="flagless", messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}]) + + assert b"cachePoint" in converse.calls.last.request.content + + +@pytest.mark.parametrize( + ("catalog_caching", "deployment_caching", "expected"), + [(None, True, True), (True, False, False), (True, None, True), (None, None, False)], +) +def test_a_routed_deployments_own_prompt_caching_flag_decides_cache_points_for_a_catalogued_model( + catalog_caching: bool | None, deployment_caching: bool | None, expected: bool, local_model_cost_map: None +) -> None: + from litellm.llms.bedrock.common_utils import bedrock_model_accepts_cache_points + + model: Final = f"qa-catalogued-model-{catalog_caching}-{deployment_caching}" + catalog_flag: Final = {} if catalog_caching is None else {"supports_prompt_caching": catalog_caching} + deployment_flag: Final = {} if deployment_caching is None else {"supports_prompt_caching": deployment_caching} + litellm.register_model({model: {"litellm_provider": "bedrock", "mode": "chat", **catalog_flag}}) + + assert bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", **deployment_flag}) is expected + + +@pytest.mark.parametrize( + ("declared", "catalog_caching", "expected"), + [ + ("true", False, True), + (1, False, True), + (" TRUE ", False, True), + ("false", True, False), + (0, True, False), + ("maybe", True, True), + ("maybe", False, False), + ], +) +def test_a_deployments_prompt_caching_flag_written_as_a_string_or_number_decides_cache_points_as_the_bool_it_spells( + declared: object, catalog_caching: bool, expected: bool, local_model_cost_map: None +) -> None: + from litellm.llms.bedrock.common_utils import bedrock_model_accepts_cache_points + + model: Final = f"qa-catalogued-model-lax-{catalog_caching}" + litellm.register_model( + {model: {"litellm_provider": "bedrock", "mode": "chat", "supports_prompt_caching": catalog_caching}} + ) + + assert ( + bedrock_model_accepts_cache_points(model, model_info={"id": "qa-deployment", "supports_prompt_caching": declared}) + is expected + ) + def test_merge_bedrock_aws_request_params_strips_caller_identity_when_deployment_has_static_credentials(): from litellm.llms.bedrock.common_utils import merge_bedrock_aws_request_params diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 4aa93848c79..263c2f7618d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2173,3 +2173,45 @@ async def test_openai_request_rewrites_media_parts_without_touching_the_callers_ {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}}, ] assert messages == messages_before + + +_OPENAI_DEPLOYMENT_FLAGS: Final = {"opted-in": {"supports_prompt_cache_breakpoint": True}} + + +@pytest.mark.parametrize("registration_order", [("opted-in", "flagless"), ("flagless", "opted-in")]) +async def test_chat_sends_a_breakpoint_by_the_deployments_own_flag_not_a_siblings_on_the_same_model( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, registration_order: tuple[str, str] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + catalog_row_without_a_flag: Final = {"litellm_provider": "openai", "mode": "chat"} + monkeypatch.setitem(litellm.model_cost, "qa-openai-shared", catalog_row_without_a_flag) + chat: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": name, + "litellm_params": { + "model": "openai/qa-openai-shared", + "api_base": "https://api.openai.com/v1", + "api_key": "openai-test-key", + "cache_control_injection_points": [{"location": "message", "role": "system"}], + }, + "model_info": {"id": f"{name}-id", **_OPENAI_DEPLOYMENT_FLAGS.get(name, {})}, + } + for name in registration_order + ], + num_retries=0, + ) + + async def sends_a_breakpoint(name: str) -> bool: + await router.acompletion( + model=name, messages=[{"role": "system", "content": "stable"}, {"role": "user", "content": "hi"}] + ) + return b"prompt_cache_breakpoint" in chat.calls.last.request.content + + assert {name: await sends_a_breakpoint(name) for name in registration_order} == { + "opted-in": True, + "flagless": False, + } diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index d5619515491..012926c6cb3 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6564,6 +6564,31 @@ def test_an_application_inference_profile_gets_no_item_marks_under_drop_params() assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 0 + +def test_a_bedrock_deployment_keeps_item_marks_by_its_own_flag_not_a_siblings_on_the_same_model( + local_model_cost_map: None, +): + from litellm import Router + + Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": {"model": "bedrock/moonshotai.kimi-k3", "aws_region_name": "us-east-1"}, + "model_info": {"id": "opted-in-id", "supports_prompt_cache_breakpoint": True}, + } + ] + ) + marked_system: Final = {"role": "system", "content": "House rules", "cache_control": _EPHEMERAL} + + def bridged_system(model_info: Mapping[str, object]) -> Mapping[str, object]: + return _bridged_request(marked_system, model="bedrock/moonshotai.kimi-k3", model_info=model_info)[1] + + assert ( + bridged_system({"id": "flagless-id"}), + bridged_system({"id": "opted-in-id", "supports_prompt_cache_breakpoint": True}), + ) == ({"role": "system", "content": "House rules"}, marked_system) + def test_the_shared_input_transform_keeps_every_item_mark_for_a_request_it_sends(): messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=cast(ResponseInputParam, list(_marked_turns(5))), # cast-ok: cache_control is not an input TypedDict key diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index b95ea8509ba..c1c2593c4ef 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -11433,6 +11433,26 @@ def test_vertex_llama3_model_info_resolves_without_the_meta_prefix(): assert prefixed_info["key"] == f"vertex_ai/{model}", model +def test_the_catalog_with_register_model_entries_lists_each_model_once_and_leaves_router_registrations_out() -> None: + from litellm.utils import catalog_with_runtime_registrations + + catalog: Final = catalog_with_runtime_registrations() + catalog_size: Final = len(catalog) + catalogued_model: Final = next(iter(catalog)) + catalogued_entry: Final = dict(catalog[catalogued_model]) + litellm.register_model({catalogued_model: {"supports_prompt_cache_breakpoint": True}}) + litellm.register_model({"qa-registered-only-model": {"litellm_provider": "openai", "mode": "chat"}}) + litellm.register_model({"qa-router-deployment-id": {"mode": "chat"}}, persist_across_reloads=False) + + merged: Final = catalog_with_runtime_registrations() + listed: Final = list(merged) + + assert len(listed) == len(merged) == catalog_size + 1 + assert "qa-registered-only-model" in listed + assert "qa-router-deployment-id" not in listed + assert dict(merged[catalogued_model]) == {**catalogued_entry, "supports_prompt_cache_breakpoint": True} + + @pytest.mark.parametrize( "key", ("openai.test-converse", "bedrock/openai.test-converse", "bedrock/converse/openai.test-converse") ) From 864b98acec1ff746fea2bddcd820855968b6f844 Mon Sep 17 00:00:00 2001 From: norkelbyte <348963847+norkelbyte@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:56:23 -0800 Subject: [PATCH 19/34] fix(bedrock)!: fail a Nova Sonic realtime session whose Bedrock stream drops instead of closing 1000 (#49170) * fix(bedrock): fail a Nova Sonic realtime session whose Bedrock stream drops instead of closing 1000 The Bedrock SDK ends the output stream the same way for a dropped connection as for a session that sent sessionEnd, so the handler treated a dropped stream as a clean finish. The client got a 1000 close with no error event and the router logged 200 OK with no retry An output end now counts as clean only after the client left and the handler began closing the Bedrock input. An earlier end raises a 503 BedrockError, so the client gets an error event and a 1011 close, the failure handlers run, partial spend is still logged, and a retry of a committed session is still refused * fix(bedrock): fail a dropped Nova Sonic session right away and log it as failed When Bedrock's output side ends mid-session, by a drop or a modeled stream error, the client side no longer spends the close timeout sending close messages into the dead input stream, so the error reaches the app right after the drop. A session the app ends itself still sends Bedrock its close messages and closes the input The replay refusal now leads with the committed cause, so the error event and the cut 1011 close reason both start with it. It stays a 400 and is tagged as a failure outside the selected deployment, so router retries and fallbacks of a dropped session no longer add extra failure counts A clean realtime session now counts once toward its deployment's cooldown success counter, so cooldown reads the real failure rate instead of 100 percent. Realtime tpm and rpm counting is unchanged The spend row of a session the provider ended now reads failure with its error information, while the events already forwarded are still billed. A session the app ended keeps logging success * fix(bedrock): agree on a dropped realtime session's failure across loggers and the spend row A realtime session's success event now reports status failure, with the same error_information the spend row stores, when the session recorded a provider failure in its metadata. Usage already streamed stays billed and the router still counts one cooldown failure The spend row builder now runs every row's error_information through sanitize_error_information_for_spend_logs (DB size cap plus prompt redaction), so the realtime failure rows get the same treatment as the proxy failure hook, which no longer sanitizes on its own to avoid truncating twice Drop the selection-failure tag on the committed-session refusal, since the shared logging object already keeps a retried refusal out of cooldown, and pin the router's realtime success counting with an OpenAI realtime deployment * fix(bedrock): count realtime router successes only for sessions that logged one, and log first-turn drops The router counts a realtime success only when the session's success handlers ran, so a refused handshake or an upstream close before any frame no longer offsets its failure. The success event's payload status stays success again, so each logger gets one failure record per dropped session, while the spend row still reads failure from the metadata the Bedrock handler records. A committed Bedrock session that fails before any logged event now gets one zero-cost Logs row. After an output failure the input stream is closed in the background, so the SDK body writer finishes without delaying the client's error. Every input send failure reports the same 503 stream-ended cause as an output drop * fix(realtime): count a router success only for a session that ended cleanly A realtime session the upstream closes with 1011 or 1006 after its first frame now marks itself as ended abnormally where that close is seen, in the shared relay and in the OpenAI Live handler, so the router no longer counts it as a cooldown success. The success callbacks still run, so the proxy keeps the budget and parallel-slot settlement it had A Bedrock session that fails before Bedrock sends any usage now logs no success event, as on base. The spend-row failure fields go into a new metadata mapping instead of the request's own dict, validated with a pydantic TypeAdapter instead of an unchecked TypeIs guard. The handler tests drop a wall-clock bound and type their helpers * fix(bedrock): log a dropped realtime session's usage from its own logging state The failure event replaced the shared standard_logging_object while the queued success dispatch was still running its callbacks, so loggers received $0 and 0 tokens for a billed dropped session. * fix(proxy): settle a rate-limit reservation once when a request logs a failure and a success A dropped realtime session fires a failure event and then a success event for the same call. The v3 limiter refunded the TPM reservation on the failure and released it again on the success, so the key's TPM counter went negative. The success path now skips a reservation the failure already released and marks it settled for a later failure. The realtime handler test callbacks are typed * fix(rate-limit): settle each token reservation once when a request logs both a failure and a success A dropped realtime session logs a success for its usage and then a failure. After a success the limiter now clears every reserved amount (TPM, PTU, ITPM, OTPM), so a later failure refunds nothing twice. A new reservation written in the same context after a released one now starts clean, so reserve, fail, reserve, succeed charges the second request's usage once instead of 78 TPM or 28 ITPM for 27 and 20 used * fix(rate-limit): take a token hold before awaiting so concurrent success and failure callbacks settle it once * fix(rate-limit): track the ITPM/OTPM hold's release apart from TPM so a nested pre-call keeps the request's project IO charge * test(rate-limit): pin the split hold-release flags across nested MCP pre-calls * fix(realtime): count a relay that fails on its own as an abnormal end * test(realtime): pin the relay failure exit with a client frame the provider rejects --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 1 + .../litellm_core_utils/realtime_streaming.py | 20 +- litellm/llms/bedrock/realtime/handler.py | 164 +++++- litellm/llms/openai/live/handler.py | 20 +- .../hooks/parallel_request_limiter_v3.py | 92 ++- .../proxy/hooks/proxy_track_cost_callback.py | 10 +- .../spend_tracking/spend_tracking_utils.py | 10 +- litellm/router_python_core.py | 19 + .../realtime/test_bedrock_realtime_handler.py | 546 +++++++++++++++++- tests/unit/llms/openai/live/test_handler.py | 92 ++- .../hooks/test_parallel_request_limiter_v3.py | 368 ++++++++++++ .../hooks/test_proxy_track_cost_callback.py | 38 +- tests/unit/proxy/hooks/test_tpm_concurrent.py | 2 +- .../test_spend_tracking_utils.py | 40 ++ .../test_router/test_router_helper_utils.py | 247 +++++++- 15 files changed, 1551 insertions(+), 118 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 96caa162604..97631b9997a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -451,6 +451,7 @@ OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset( {"response.completed", "response.incomplete", "response.failed"} ) REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged" +REALTIME_SESSION_ENDED_ABNORMALLY_KEY: Final = "realtime_session_ended_abnormally" # SSL/TLS cipher configuration for faster handshakes # Strategy: Strongly prefer fast modern ciphers, but allow fallback to commonly supported ones diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index e9e9a5a75b9..8e6e71a31d3 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -21,6 +21,7 @@ from litellm.constants import ( REALTIME_LAZY_SETUP_CONTENT_GRACE_SECONDS, REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) @@ -65,6 +66,10 @@ class BackendClose: code: int reason: str + @property + def ended_cleanly(self) -> bool: + return self.code == 1000 + @property def message(self) -> str: if not self.reason: @@ -75,6 +80,7 @@ class BackendClose: class ClientLoopExit(Enum): CLIENT_DISCONNECTED = auto() BACKEND_CLOSED = auto() + RELAY_FAILED = auto() def backend_close_from(error: "ConnectionClosed") -> BackendClose: @@ -2526,7 +2532,7 @@ class RealTimeStreaming: return ClientLoopExit.BACKEND_CLOSED except Exception as e: verbose_logger.debug("Error in client ack messages: %s", e) - return ClientLoopExit.CLIENT_DISCONNECTED + return ClientLoopExit.RELAY_FAILED async def bidirectional_forward(self) -> None: if self._is_translation_session and translation_guardrails_apply(self.request_data): @@ -2539,9 +2545,15 @@ class RealTimeStreaming: client_task: Final = asyncio.create_task(self.client_ack_messages()) try: await asyncio.wait((forward_task, client_task), return_when=asyncio.FIRST_COMPLETED) - if client_task.done() and client_task.result() is ClientLoopExit.CLIENT_DISCONNECTED: + client_exit: Final = client_task.result() if client_task.done() else None + if client_exit is ClientLoopExit.RELAY_FAILED and self.logging_obj: + self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True + if client_exit in (ClientLoopExit.CLIENT_DISCONNECTED, ClientLoopExit.RELAY_FAILED): return - await self._close_client(await forward_task) + close: Final = await forward_task + if not close.ended_cleanly and self.logging_obj: + self.logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True + await self._close_client(close) finally: forward_task.cancel() client_task.cancel() @@ -2551,7 +2563,7 @@ class RealTimeStreaming: redacted_message: Final = redact_internal_details_from_client_message(close.message) redacted_reason: Final = redact_internal_details_from_client_message(close.reason) try: - if close.code != 1000: + if not close.ended_cleanly: await self.websocket.send_text(realtime_error_event(redacted_message, error_type="server_error")) await self.websocket.close( code=client_close_code(close.code), diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 00622a1e287..49939399ec8 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -9,7 +9,7 @@ import contextlib import importlib.metadata import json from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, NoReturn, Protocol, runtime_checkable @@ -34,12 +34,15 @@ from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateR from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput +from litellm.types.utils import StandardLoggingPayloadErrorInformation from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError from .transformation import BedrockRealtimeConfig, requested_modalities _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) +_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata") _EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({}) @@ -61,6 +64,10 @@ _BEDROCK_STREAM_ERROR_STATUS: Final[Mapping[str, int]] = MappingProxyType( ) +def _stream_ended_before_session_closed() -> BedrockError: + return BedrockError(status_code=503, message="Bedrock realtime stream ended before the session closed") + + def _as_bedrock_error(error: BaseException) -> BaseException: status_code: Final = _BEDROCK_STREAM_ERROR_STATUS.get(type(error).__name__) if status_code is None: @@ -165,6 +172,12 @@ class _BridgeOutcome: client_disconnected: bool +@dataclass(frozen=True, slots=True) +class SessionEndSignals: + input_closing: asyncio.Event = field(default_factory=asyncio.Event) + output_failed: asyncio.Event = field(default_factory=asyncio.Event) + + async def _client_messages(client_ws: RealtimeClientWebSocket, initial_message: str | None) -> AsyncIterator[str]: if initial_message is not None: yield initial_message @@ -188,8 +201,8 @@ def raise_if_session_committed(scope: Mapping[str, object] | None) -> None: raise BedrockError( status_code=400, message=( - "Bedrock realtime session already committed to a provider stream; it cannot be replayed" - + (f". The committed stream failed with: {committed_failure}" if committed_failure else "") + (f"{committed_failure}. " if committed_failure else "") + + "Bedrock realtime session already committed to a provider stream; it cannot be replayed" ), ) @@ -201,11 +214,48 @@ def _pending_opening_message(scope: Mapping[str, object]) -> str | None: return pending if isinstance(pending, str) else None +def _spend_log_metadata(litellm_params: Mapping[str, object]) -> tuple[str, Mapping[str, object]] | None: + return next( + ( + (key, metadata) + for key in _SPEND_LOG_METADATA_KEYS + if (metadata := _REQUEST_MAPPING_ADAPTER.validate_python(litellm_params.get(key))) + ), + None, + ) + + +def _record_failure_on_spend_log(logging_obj: LiteLLMLogging, error: BaseException) -> None: + """The spend row of a session the provider ended reads failure while it still bills the forwarded events.""" + litellm_params: Final = _REQUEST_MAPPING_ADAPTER.validate_python( + logging_obj.model_call_details.get("litellm_params") + ) + if litellm_params is None: + return + spend_log_metadata: Final = _spend_log_metadata(litellm_params) + if spend_log_metadata is None: + return + status_code: Final[object] = getattr(error, "status_code", None) + error_information: Final[StandardLoggingPayloadErrorInformation] = { + "error_code": "" if status_code is None else str(status_code), + "error_class": type(error).__name__, + "error_message": redact_string(str(error)), + "llm_provider": "bedrock", + } + metadata_key, metadata = spend_log_metadata + logging_obj.model_call_details["litellm_params"] = { # rebind-ok: the spend row reads the logging object + **litellm_params, + metadata_key: {**metadata, "status": "failure", "error_information": error_information}, + } + + def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseException) -> NoReturn: error: Final = _as_bedrock_error(failure) verbose_proxy_logger.error("Bedrock Realtime: provider stream failed: %s", redact_string(str(error))) if _is_session_committed(scope): scope[BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY] = redact_string(str(error)) + if error is failure: + raise failure raise error from failure @@ -267,6 +317,25 @@ async def _reject_session_update( return True +async def _close_bedrock_input( + bedrock_stream: BedrockBidirectionalStream, + close_messages: Sequence[str], + send_to_bedrock: Callable[[str], Awaitable[None]], + timeout: float, +) -> None: + async def flush_and_close() -> None: + for close_message in close_messages: + with contextlib.suppress(Exception): + await send_to_bedrock(close_message) + with contextlib.suppress(Exception): + await bedrock_stream.input_stream.close() + + try: + await asyncio.wait_for(flush_and_close(), timeout) + except (asyncio.TimeoutError, TimeoutError): + verbose_proxy_logger.debug("Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", timeout) + + class BedrockRealtime(BaseAWSLLM): """Handler for Bedrock Nova Sonic realtime speech-to-speech API.""" @@ -278,6 +347,18 @@ class BedrockRealtime(BaseAWSLLM): super().__init__() self._sdk_version_lookup: Final = sdk_version_lookup self._close_timeout: Final = close_timeout + self._failed_input_closes: Final[set[asyncio.Task[None]]] = set() # mutable-ok: strong refs to running closes + + def _close_failed_input_in_background( + self, + bedrock_stream: BedrockBidirectionalStream, + send_to_bedrock: Callable[[str], Awaitable[None]], + ) -> None: + close_task: Final = asyncio.create_task( + _close_bedrock_input(bedrock_stream, (), send_to_bedrock, self._close_timeout) + ) + self._failed_input_closes.add(close_task) + close_task.add_done_callback(self._failed_input_closes.discard) async def async_realtime( self, @@ -433,23 +514,26 @@ class BedrockRealtime(BaseAWSLLM): if _should_log_event(leftover_event) ), ) + if outcome.provider_failure is not None and outcome.client_disconnected: + verbose_proxy_logger.debug( + "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure + ) + provider_failure: Final = None if outcome.client_disconnected else outcome.provider_failure if logged_events: + if provider_failure is not None: + _record_failure_on_spend_log(logging_obj, _as_bedrock_error(provider_failure)) + usage_logging_obj: Final = logging_obj if provider_failure is None else logging_obj.attempt_scoped_copy() GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - logging_obj.dispatch_success_handlers( + usage_logging_obj.dispatch_success_handlers( list(logged_events), prefer_async_handlers=True, ) ) logging_obj.model_call_details[REALTIME_SESSION_SUCCESS_LOGGED_KEY] = True - if outcome.provider_failure is None: + if provider_failure is None: return - if outcome.client_disconnected: - verbose_proxy_logger.debug( - "Bedrock Realtime: stream failed after the client disconnected: %s", outcome.provider_failure - ) - return - _raise_provider_failure(websocket.scope, outcome.provider_failure) + _raise_provider_failure(websocket.scope, provider_failure) async def _bridge( self, @@ -463,16 +547,34 @@ class BedrockRealtime(BaseAWSLLM): ) -> _BridgeOutcome: """Run both forwarding directions until the client leaves or either side fails.""" logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend + signals: Final = SessionEndSignals() async def collect_logged_events() -> None: - async for event in self._forward_bedrock_to_client( - bedrock_stream, websocket, transformation_config, model, logging_obj, session_state - ): - logged.append(event) + try: + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + signals=signals, + ): + logged.append(event) + except Exception: + signals.output_failed.set() + raise client_task: Final = asyncio.create_task( self._forward_client_to_bedrock( - websocket, bedrock_stream, transformation_config, model, session_state, logging_obj, initial_message + websocket, + bedrock_stream, + transformation_config, + model, + session_state, + logging_obj, + initial_message, + signals=signals, ) ) bedrock_task: Final = asyncio.create_task(collect_logged_events()) @@ -506,6 +608,8 @@ class BedrockRealtime(BaseAWSLLM): session_state: RealtimeResponseTransformInput, logging_obj: LiteLLMLogging | None = None, initial_message: str | None = None, + *, + signals: SessionEndSignals, ) -> None: """Forward messages from client WebSocket to Bedrock stream. @@ -521,7 +625,10 @@ class BedrockRealtime(BaseAWSLLM): async def send_to_bedrock(bedrock_message: str) -> None: event: Final = build_input_chunk(bedrock_message.encode("utf-8")) - await bedrock_stream.input_stream.send(event) + try: + await bedrock_stream.input_stream.send(event) + except Exception as e: + raise _stream_ended_before_session_closed() from e verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200]) try: @@ -553,20 +660,13 @@ class BedrockRealtime(BaseAWSLLM): client_ws, transformation_config, model, logging_obj, parsed_client_message ): break + signals.input_closing.set() finally: - - async def close_bedrock_input() -> None: - for close_message in transformation_config.session_close_messages(): - with contextlib.suppress(Exception): - await send_to_bedrock(close_message) - with contextlib.suppress(Exception): - await bedrock_stream.input_stream.close() - - try: - await asyncio.wait_for(close_bedrock_input(), self._close_timeout) - except (asyncio.TimeoutError, TimeoutError): - verbose_proxy_logger.debug( - "Bedrock Realtime: timed out closing Bedrock input stream after %.1fs", self._close_timeout + if signals.output_failed.is_set(): + self._close_failed_input_in_background(bedrock_stream, send_to_bedrock) + else: + await _close_bedrock_input( + bedrock_stream, transformation_config.session_close_messages(), send_to_bedrock, self._close_timeout ) async def _forward_bedrock_to_client( @@ -577,6 +677,8 @@ class BedrockRealtime(BaseAWSLLM): model: str, logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, + *, + signals: SessionEndSignals, ) -> AsyncIterator[OpenAIRealtimeEvents]: """Forward messages from Bedrock to the client, yielding the ones to record for spend logging. @@ -597,6 +699,8 @@ class BedrockRealtime(BaseAWSLLM): result = await output[1].receive() if result is None: + if not signals.input_closing.is_set(): + raise _stream_ended_before_session_closed() verbose_proxy_logger.debug("Bedrock Realtime: Bedrock stream ended") with contextlib.suppress(Exception): await client_ws.close() diff --git a/litellm/llms/openai/live/handler.py b/litellm/llms/openai/live/handler.py index 297a147a288..5b899ac4162 100644 --- a/litellm/llms/openai/live/handler.py +++ b/litellm/llms/openai/live/handler.py @@ -18,6 +18,7 @@ from litellm._logging import redact_secrets from litellm.constants import ( OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS, OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, ) @@ -316,12 +317,15 @@ async def _forward_live_backend_frames( backend_websocket: LiveBackendWebSocket, client_gone: asyncio.Event, event_retention: _LiveEventRetention, + upstream_ended_abnormally: asyncio.Event, ) -> None: from starlette.websockets import WebSocketDisconnect from websockets.exceptions import ConnectionClosed event_loop: Final = asyncio.get_running_loop() - async for message, close_deadline in _live_backend_messages(websocket, backend_websocket): + async for message, close_deadline in _live_backend_messages( + websocket, backend_websocket, upstream_ended_abnormally + ): if (event := _metering_event(message)) is not None: event_retention.retain(event) if client_gone.is_set(): @@ -341,6 +345,7 @@ async def _forward_live_backend_frames( async def _live_backend_messages( websocket: LiveClientWebSocket, backend_websocket: LiveBackendWebSocket, + upstream_ended_abnormally: asyncio.Event, ) -> AsyncIterator[tuple[str, float | None]]: from starlette.websockets import WebSocketDisconnect from websockets.exceptions import ConnectionClosed @@ -350,6 +355,8 @@ async def _live_backend_messages( yield message, None except ConnectionClosed as error: upstream_close: Final = backend_close_from(error) + if not upstream_close.ended_cleanly: + upstream_ended_abnormally.set() with suppress(WebSocketDisconnect, RuntimeError, ConnectionClosed, OSError): await websocket.close(code=upstream_close.code, reason=upstream_close.reason) return @@ -434,6 +441,7 @@ class OpenAILiveSessions(OpenAIRealtime): "complete_input_dict": {"session_start": rewritten_start}, } event_retention: Final = _LiveEventRetention() + session_ended_abnormally: Final = asyncio.Event() try: logging_obj.pre_call(input=None, api_key=api_key, additional_args=pre_call_args) async with self._websocket_connector( @@ -448,7 +456,11 @@ class OpenAILiveSessions(OpenAIRealtime): websocket=websocket, backend_websocket=backend_websocket, event_retention=event_retention, + upstream_ended_abnormally=session_ended_abnormally, ) + except Exception: + session_ended_abnormally.set() + raise finally: session_end_time: Final = self._clock() retained_events: Final = event_retention.results() @@ -463,6 +475,10 @@ class OpenAILiveSessions(OpenAIRealtime): logging_obj.model_call_details[ # rebind-ok: flags the shared logging object as already logged REALTIME_SESSION_SUCCESS_LOGGED_KEY ] = True + if session_ended_abnormally.is_set(): + logging_obj.model_call_details[ # rebind-ok: flags the shared logging object for the router + REALTIME_SESSION_ENDED_ABNORMALLY_KEY + ] = True except InvalidStatus as error: await close_after_upstream_handshake_refusal(websocket, error.response.status_code) except Exception as error: # noqa: BLE001 # any relay failure must still close the client socket with a redacted reason @@ -484,6 +500,7 @@ class OpenAILiveSessions(OpenAIRealtime): websocket: LiveClientWebSocket, backend_websocket: LiveBackendWebSocket, event_retention: _LiveEventRetention, + upstream_ended_abnormally: asyncio.Event, ) -> None: from websockets.exceptions import ConnectionClosed @@ -503,6 +520,7 @@ class OpenAILiveSessions(OpenAIRealtime): backend_websocket=backend_websocket, client_gone=client_gone, event_retention=event_retention, + upstream_ended_abnormally=upstream_ended_abnormally, ) ) client_gone_task: Final = asyncio.create_task(client_gone.wait()) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index c83551a815c..1cbe5eb39e8 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -709,7 +709,7 @@ class RequestRateLimiterStash: A single mutable instance is shared by every context forked from the request task (the SDK call, streaming generators, and the logging worker's captured context all see the same object), which is what makes the - ``reservation_released`` flag and ``parallel_slot`` clearing effective + released flags and ``parallel_slot`` clearing effective across sibling callbacks: the first release wins, later callbacks observe the cleared state. @@ -746,9 +746,32 @@ class RequestRateLimiterStash: batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = () reservation_released: bool = False + io_reservation_released: bool = False tpm_limited_tags: frozenset[str] = field(default_factory=frozenset) socket_agent_windows: tuple[RateLimitDescriptor, ...] = () + def clear_reserved_tokens(self) -> None: + self.reserved_tokens = 0 + self.ptu_reserved_tokens = 0 + self.itpm_reserved_tokens = 0 + self.otpm_reserved_tokens = 0 + + def mark_reservation_released(self) -> None: + self.reservation_released = True + self.io_reservation_released = True + + def discard_released_tpm_reservation(self) -> None: + if self.reservation_released: + self.reserved_tokens = 0 + self.ptu_reserved_tokens = 0 + self.reservation_released = False + + def discard_released_io_reservation(self) -> None: + if self.io_reservation_released: + self.itpm_reserved_tokens = 0 + self.otpm_reserved_tokens = 0 + self.io_reservation_released = False + @dataclass(frozen=True, slots=True) class CounterRefund: @@ -4238,6 +4261,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) + stash.discard_released_io_reservation() if itpm_reserved > 0: itpm_scopes: Final = tuple( (d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY @@ -4597,6 +4621,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # incremented so post-call reconciliation only applies # the (actual - reserved) delta to those — unreserved # scopes get charged the full actual usage instead. + stash.discard_released_tpm_reservation() stash.reserved_tokens = estimated_tokens stash.ptu_reserved_tokens = ptu_estimated_tokens stash.ptu_reserved_window_identities = frozenset( @@ -5140,7 +5165,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self._response_usage(response_obj) ) or self._aggregate_only_total_tokens(self._response_usage(kwargs.get("combined_usage_object"))) - if not response_usage[2] and not combined_usage[2] and aggregate_total <= 0 and not stash.reservation_released: + if ( + not response_usage[2] + and not combined_usage[2] + and aggregate_total <= 0 + and not stash.io_reservation_released + ): return () resolved_usage: Final = ( response_usage @@ -5153,25 +5183,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) billable_input, completion_tokens, _ = resolved_usage - if stash.reservation_released or ( + if stash.io_reservation_released or ( not stash.itpm_reserved_window_identities and not stash.otpm_reserved_window_identities ): return self._build_reservation_aware_tpm_ops( targets=tuple(stash.itpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes, actual_tokens=billable_input, - reserved_tokens=0 if stash.reservation_released else itpm_reserved, + reserved_tokens=0 if stash.io_reservation_released else itpm_reserved, ) + self._build_reservation_aware_tpm_ops( targets=tuple(stash.otpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes, actual_tokens=completion_tokens, - reserved_tokens=0 if stash.reservation_released else otpm_reserved, + reserved_tokens=0 if stash.io_reservation_released else otpm_reserved, ) itpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = ( self._build_project_reservation_ops( targets=tuple(stash.itpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes, actual_tokens=billable_input, reserved_tokens=itpm_reserved, reservation_window_identities=stash.itpm_reserved_window_identities, @@ -5182,7 +5212,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): otpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = ( self._build_project_reservation_ops( targets=tuple(stash.otpm_reserved_scopes), - reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes, + reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes, actual_tokens=completion_tokens, reserved_tokens=otpm_reserved, reservation_window_identities=stash.otpm_reserved_window_identities, @@ -5402,7 +5432,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): total_tokens = self._aggregate_only_total_tokens(usage=_usage) stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) - reserved_tokens: Final = stash.reserved_tokens if stash is not None else 0 + open_reservation: Final = stash if stash is not None and not stash.reservation_released else None + reserved_tokens: Final = open_reservation.reserved_tokens if open_reservation is not None else 0 reserved_model: Final = stash.reserved_model if stash is not None else None reserved_scopes: Final[frozenset[tuple[str, str]]] = stash.reserved_scopes if stash is not None else frozenset() # Reconciliation must target the same model-scoped counter that the @@ -5464,7 +5495,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): reconcile_model=reconcile_model, reserved_scopes=reserved_scopes, reserved_ceiling=stash.ptu_ceiling if stash is not None else None, - reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0, + reserved_tokens=open_reservation.ptu_reserved_tokens if open_reservation is not None else 0, reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(), total_tokens=0 if ptu_skips_socket_usage else total_tokens, ) @@ -5561,18 +5592,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else None, ) - await self._apply_usage_operations( - self._build_success_event_pipeline_operations( - kwargs=kwargs, - response_obj=response_obj, - rate_limit_type=rate_limit_type, - ), - litellm_parent_otel_span, + usage_operations: Final = self._build_success_event_pipeline_operations( + kwargs=kwargs, + response_obj=response_obj, + rate_limit_type=rate_limit_type, ) io_token_operations: Final = self._build_io_token_reservation_ops( kwargs=kwargs, response_obj=response_obj, ) + if stash is not None: + stash.clear_reserved_tokens() + await self._apply_usage_operations(usage_operations, litellm_parent_otel_span) if io_token_operations: if isinstance(io_token_operations, list): await self.async_increment_tokens_with_ttl_preservation( @@ -5733,10 +5764,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # already released it (proxy-level rejection that also bubbles up # here as an LLM-error callback). max_parallel_requests is its # own counter and is always decremented per call. - reserved_tokens, itpm_reserved, otpm_reserved = ( - (0, 0, 0) - if stash is None or stash.reservation_released - else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens) + reserved_tokens: Final = 0 if stash is None or stash.reservation_released else stash.reserved_tokens + itpm_reserved, otpm_reserved = ( + (0, 0) + if stash is None or stash.io_reservation_released + else (stash.itpm_reserved_tokens, stash.otpm_reserved_tokens) ) tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call) @@ -5806,6 +5838,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): else () ) + if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0): + stash.mark_reservation_released() if pipeline_operations: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline_post_call( pipeline_operations, parent_otel_span=litellm_parent_otel_span @@ -5820,8 +5854,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations=project_operations, parent_otel_span=litellm_parent_otel_span, ) - if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0): - stash.reservation_released = True if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call): await self._apply_usage_operations( self._build_success_event_pipeline_operations( @@ -5927,7 +5959,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): Idempotent: the slot release clears the stashed acquisition (and slot removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM - refund is guarded by the stash's ``reservation_released`` flag — if + refund is guarded by the stash's two released flags — if both this hook and async_log_failure_event end up running in the same flow, only the first release/refund applies. A mid-stream failure relayed here with recovered partial usage settles the reservation at @@ -5953,11 +5985,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) stash.batch_tpd_refund_ops = () - if stash.reservation_released: - return - reserved_tokens: Final = stash.reserved_tokens - itpm_reserved: Final = stash.itpm_reserved_tokens - otpm_reserved: Final = stash.otpm_reserved_tokens + reserved_tokens: Final = 0 if stash.reservation_released else stash.reserved_tokens + itpm_reserved: Final = 0 if stash.io_reservation_released else stash.itpm_reserved_tokens + otpm_reserved: Final = 0 if stash.io_reservation_released else stash.otpm_reserved_tokens if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0: return tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data) @@ -6024,6 +6054,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): itpm_reserved, otpm_reserved, ) + stash.mark_reservation_released() if combined_ops: await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline( increment_list=list(combined_ops), @@ -6040,7 +6071,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): pipeline_operations=project_ops, parent_otel_span=user_api_key_dict.parent_otel_span, ) - stash.reservation_released = True except Exception as e: verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e) return diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 980f0a5d410..8a015d396d0 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -64,7 +64,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export deployment_params_provider, get_request_model_access_groups, - sanitize_error_information_for_spend_logs, + sanitize_error_information_for_spend_logs, # pyright: ignore[reportUnusedImport] # backwards-compatible package export should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs @@ -79,7 +79,6 @@ from litellm.types.utils import ( CallTypes, LiteLLMBatch, StandardLoggingPayload, - StandardLoggingPayloadErrorInformation, ) from litellm.utils import get_end_user_id_for_cost_tracking @@ -492,13 +491,6 @@ class _ProxyDBLogger(CustomLogger): # ``.get("traceback")`` / truthy checks, and the TypedDict marks # the field as optional, so omitting is type-safe. _error_information.pop("traceback", None) - # Strip echoed request input + apply DB-size cap before storing in - # the spend-log metadata column (LIT-2992). Result is never None - # here because the input above is constructed non-None. - _error_information = cast( - StandardLoggingPayloadErrorInformation, - sanitize_error_information_for_spend_logs(_error_information, original_exception=original_exception), - ) _metadata["error_information"] = _error_information _metadata = ( # rebind-ok: enriched identity is shared with the earlier immutable capture diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 8b0ff718fec..a9478a1f889 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -598,6 +598,7 @@ def get_logging_payload( kwargs = {} rejected_as_unknown_model: Final = isinstance(response_obj, ProxyModelNotFoundError) + failure_exception: Final = response_obj if isinstance(response_obj, BaseException) else None response_served_from_cache: Final = served_from_cache(response_obj) if response_obj is None: response_obj = {} @@ -888,14 +889,11 @@ def get_logging_payload( additional_usage_values["cache_creation_input_tokens"] = cache_write_tokens clean_metadata["additional_usage_values"] = additional_usage_values drop_requester_ip_address: Final = requester_ip_address_logging_disabled() - error_information: Final = clean_metadata["error_information"] if drop_requester_ip_address: clean_metadata["requester_ip_address"] = None - if drop_requester_ip_address and isinstance(error_information, Mapping): - clean_metadata["error_information"] = cast( # cast-ok: same keys, only the str values are masked - StandardLoggingPayloadErrorInformation, - _error_information_without_ip_addresses(_OBJECT_MAPPING.validate_python(error_information)), - ) + clean_metadata["error_information"] = sanitize_error_information_for_spend_logs( + clean_metadata["error_information"], original_exception=failure_exception + ) if litellm.cache is None: cache_key = "Cache OFF" diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 17791019aab..7ccb23ebfa8 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -84,6 +84,8 @@ from litellm.constants import ( MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, + REALTIME_SESSION_ENDED_ABNORMALLY_KEY, + REALTIME_SESSION_SUCCESS_LOGGED_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, RUNTIME_UPDATABLE_ROUTER_SETTINGS, @@ -672,6 +674,7 @@ _NO_SESSION_KWARGS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType _NO_DEFAULT_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) _SESSION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_CALL_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _SILENT_MODEL_ADAPTER: Final = TypeAdapter(str | list[str]) _RESOLVED_RETRY_POLICY_ADAPTER: Final = TypeAdapter(RetryPolicy | None) @@ -6902,6 +6905,22 @@ class PythonCore(DeprecatedRouterMembers): raise safeguard_refusal_error(model=model, stop_details=refusal_details) self.success_calls[model_name] += 1 + realtime_session_logging: Final = ( + _CALL_KWARGS_ADAPTER.validate_python(kwargs).get("litellm_logging_obj") + if generic_function_name == "_arealtime" + else None + ) + realtime_session_deployment_id: Final = ( + _MODEL_INFO_ADAPTER.validate_python(kwargs["model_info"]).get("id") + if isinstance(realtime_session_logging, LiteLLMLogging) + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + and realtime_session_logging.model_call_details.get(REALTIME_SESSION_ENDED_ABNORMALLY_KEY) is not True + else None + ) + if realtime_session_deployment_id is not None: + increment_deployment_successes_for_current_minute( + ctx=self._core_ctx, deployment_id=str(realtime_session_deployment_id) + ) verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) return response diff --git a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 2ac31b6887d..cf2f25faaa9 100644 --- a/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/unit/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -1,12 +1,16 @@ import asyncio +import copy import json import sys import types +from collections.abc import Mapping +from datetime import datetime, timezone from types import SimpleNamespace -from typing import Final +from typing import Final, Literal from unittest.mock import MagicMock import pytest +from pydantic import TypeAdapter import litellm from litellm.constants import ( @@ -14,10 +18,15 @@ from litellm.constants import ( REALTIME_SESSION_SUCCESS_LOGGED_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.realtime_errors import websocket_close_reason from litellm.llms.bedrock.common_utils import BedrockError -from litellm.llms.bedrock.realtime.handler import BedrockRealtime +from litellm.llms.bedrock.realtime.handler import BedrockRealtime, SessionEndSignals +from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig from litellm.realtime_api import main as realtime_main +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import StandardLoggingPayload, StandardLoggingPayloadErrorInformation from tests.unit.llms.bedrock.event_loop_probe import host_in_loop_probing_vault @@ -36,6 +45,66 @@ def _isolate_host_aws_config(monkeypatch, tmp_path): monkeypatch.delenv(env_var, raising=False) +def _closing_input() -> SessionEndSignals: + signals: Final = SessionEndSignals() + signals.input_closing.set() + return signals + + +_SPEND_ROW_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_ERROR_INFORMATION_ADAPTER: Final = TypeAdapter(StandardLoggingPayloadErrorInformation | None) +_BEDROCK_INPUT_CHUNK_ADAPTER: Final = TypeAdapter(Mapping[str, Mapping[str, object]]) +_STANDARD_LOGGING_PAYLOAD_ADAPTER: Final = TypeAdapter(StandardLoggingPayload) + + +def _spend_row( + logging_obj: "FakeLogging | LiteLLMLogging", +) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: + return _spend_row_of(logging_obj.model_call_details) + + +def _spend_row_of( + model_call_details: dict[str, object], +) -> tuple[Literal["success", "failure"], StandardLoggingPayloadErrorInformation | None]: + now: Final = datetime.now(timezone.utc) + row: Final = get_logging_payload(kwargs=model_call_details, response_obj=None, start_time=now, end_time=now) + metadata: Final = _SPEND_ROW_METADATA_ADAPTER.validate_json(row["metadata"]) + return row["status"], _ERROR_INFORMATION_ADAPTER.validate_python(metadata.get("error_information")) + + +class LoggedEventRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.events: Final[list[tuple[Literal["success", "failure"], dict[str, object]]]] = [] + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.events.append(("success", dict(kwargs))) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.events.append(("failure", dict(kwargs))) + + def payload(self, event: Literal["success", "failure"]) -> StandardLoggingPayload: + (logged,) = (kwargs for logged_event, kwargs in self.events if logged_event == event) + return _STANDARD_LOGGING_PAYLOAD_ADAPTER.validate_python(logged["standard_logging_object"]) + + +class HeldSuccessLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.entered: Final = asyncio.Event() + self.release: Final = asyncio.Event() + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.entered.set() + await self.release.wait() + + class FakePayloadPart: def __init__(self, bytes_): self.bytes_ = bytes_ @@ -85,6 +154,19 @@ class HangingCloseInputStream(FakeInputStream): await asyncio.Future() +class BodyWriterInputStream(FakeInputStream): + """Models the SDK input stream, whose request-body writer task only finishes once the stream is closed""" + + def __init__(self): + super().__init__() + self._close_requested = asyncio.Event() + self.body_writer = asyncio.get_running_loop().create_task(self._close_requested.wait()) + + async def close(self): + await super().close() + self._close_requested.set() + + class FakeBedrockStream: def __init__(self, input_stream=None): self.input_stream = input_stream if input_stream is not None else FakeInputStream() @@ -124,6 +206,11 @@ class FakeLogging: self.litellm_trace_id = trace_id self.model_call_details = {} + def attempt_scoped_copy(self): + attempt = copy.copy(self) + attempt.model_call_details = dict(self.model_call_details) + return attempt + class DisconnectingClientWS: def __init__(self, messages): @@ -159,12 +246,15 @@ class EndedBedrockStream: class RealtimeClientWS: - def __init__(self): + def __init__(self, messages=()): self.closed = False self.sent_to_client = [] self.scope = {} + self._messages = list(messages) async def receive_text(self): + if self._messages: + return self._messages.pop(0) raise RuntimeError("client disconnected") async def send_text(self, message): @@ -178,8 +268,7 @@ class ConnectedClientWS(RealtimeClientWS): """Client that sends its scripted messages and then stays connected until the server closes it""" def __init__(self, messages): - super().__init__() - self._messages = list(messages) + super().__init__(messages) self._closed_event = asyncio.Event() async def receive_text(self): @@ -251,6 +340,77 @@ class ScriptedBedrockStream: return (None, self._receiver) +def _sent_event_names(input_stream: FakeInputStream) -> list[str]: + return [ + next(iter(_BEDROCK_INPUT_CHUNK_ADAPTER.validate_json(chunk.value.bytes_)["event"])) + for chunk in input_stream.sent + ] + + +class SessionEndingInputStream(FakeInputStream): + def __init__(self): + super().__init__() + self.session_ended = asyncio.Event() + + async def send(self, event): + await super().send(event) + if "sessionEnd" in json.loads(event.value.bytes_.decode("utf-8")).get("event", {}): + self.session_ended.set() + + async def close(self): + await super().close() + self.session_ended.set() + + +class SlowClosingInputStream(SessionEndingInputStream): + async def close(self): + await asyncio.Event().wait() + + +class OutputEndsAfterSessionEndReceiver(ScriptedBedrockReceiver): + def __init__(self, payloads, session_ended): + super().__init__(payloads) + self._session_ended = session_ended + + async def receive(self): + if not self._payloads: + await self._session_ended.wait() + return await super().receive() + + +class SessionBoundBedrockStream: + def __init__(self, payloads=(), input_stream_type=SessionEndingInputStream): + self.input_stream = input_stream_type() + self._receiver = OutputEndsAfterSessionEndReceiver(payloads, self.input_stream.session_ended) + + async def await_output(self): + return (None, self._receiver) + + +class ConnectionResetOnAudioInputStream(FakeInputStream): + def __init__(self, connection_reset): + super().__init__() + self._connection_reset = connection_reset + + async def send(self, event): + await asyncio.sleep(0) + if self._connection_reset.is_set() or "audioInput" in json.loads(event.value.bytes_.decode("utf-8"))["event"]: + self._connection_reset.set() + raise RuntimeError("bedrock connection reset") + self.sent.append(event) + + +class OutputEndsAfterConnectionResetReceiver(ScriptedBedrockReceiver): + def __init__(self, payloads, connection_reset): + super().__init__(payloads) + self._connection_reset = connection_reset + + async def receive(self): + if not self._payloads: + await self._connection_reset.wait() + return await super().receive() + + class FakeAWSCredentialsIdentity: def __init__(self, access_key_id, secret_access_key, session_token=None): self.access_key_id = access_key_id @@ -352,7 +512,7 @@ def stub_aws_sdk_client(monkeypatch): raise stream captured["open_stream"] = stream return stream - stream = ScriptedBedrockStream(captured.get("scripted_payloads", [])) + stream = SessionBoundBedrockStream(captured.get("scripted_payloads", [])) captured["open_stream"] = stream return stream @@ -407,7 +567,9 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] event_names = [next(iter(event["event"])) for event in sent_events] @@ -422,7 +584,7 @@ class TestBedrockRealtimeHandler: stream = FakeBedrockStream() await handler._forward_client_to_bedrock( - DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {} + DisconnectingClientWS([]), stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() ) assert stream.input_stream.sent == [] @@ -437,9 +599,13 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - with pytest.raises(RuntimeError, match="bedrock send failed"): - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + with pytest.raises(BedrockError) as failure: + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock send failed" assert stream.input_stream.closed @pytest.mark.asyncio @@ -452,7 +618,10 @@ class TestBedrockRealtimeHandler: ) await asyncio.wait_for( - handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}), 2 + handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ), + 2, ) assert not stream.input_stream.closed @@ -466,7 +635,9 @@ class TestBedrockRealtimeHandler: [json.dumps({"type": "session.update", "session": {"instructions": "You are helpful."}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) sent_events = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] event_names = [next(iter(event["event"])) for event in sent_events] @@ -495,6 +666,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, + signals=_closing_input(), ) ] @@ -526,6 +698,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", FakeLogging(), {}, + signals=_closing_input(), ) ] @@ -597,6 +770,7 @@ class TestBedrockRealtimeHandler: "amazon.nova-sonic-v1:0", MagicMock(), {}, + signals=_closing_input(), ): pass @@ -656,7 +830,9 @@ class TestBedrockRealtimeSessionLifecycle: [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})] ) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() + ) acked = [json.loads(message) for message in client_ws.sent_to_client] updated = [event for event in acked if event["type"] == "session.updated"] @@ -670,7 +846,9 @@ class TestBedrockRealtimeSessionLifecycle: stream = FakeBedrockStream() client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, signals=SessionEndSignals() + ) assert client_ws.sent_to_client == [] @@ -693,7 +871,9 @@ class TestBedrockRealtimeSessionLifecycle: stream = FakeBedrockStream() client_ws = DisconnectingClientWS([update([tool]), update([other_tool]), update([tool])]) - await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) + await handler._forward_client_to_bedrock( + client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging(), signals=SessionEndSignals() + ) acked: Final = [json.loads(message) for message in client_ws.sent_to_client] assert [event["type"] for event in acked] == ["session.updated", "error", "session.updated"] @@ -720,10 +900,10 @@ class TestBedrockRealtimeSessionLifecycle: async def test_malformed_setting_is_rejected_or_dropped_and_the_session_still_answers( self, stub_aws_sdk_client, session, drop_params, expected_errors ): - stream: Final = ScriptedBedrockStream([]) + stream: Final = SessionBoundBedrockStream() stub_aws_sdk_client["streams"] = [stream] user_text: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]} - websocket: Final = ConnectedClientWS( + websocket: Final = RealtimeClientWS( [ json.dumps({"type": "session.update", "session": session}), json.dumps({"type": "conversation.item.create", "item": user_text}), @@ -761,8 +941,8 @@ class TestBedrockRealtimeProviderFailurePropagation: @pytest.mark.asyncio async def test_readiness_failure_escapes_and_fallback_replays_session_update(self, stub_aws_sdk_client): handler = BedrockRealtime() - websocket = ConnectedClientWS([self.SESSION_UPDATE]) - healthy_stream = ScriptedBedrockStream([]) + websocket = RealtimeClientWS([self.SESSION_UPDATE]) + healthy_stream = SessionBoundBedrockStream() eager_failure = ServiceUnavailableException("fault injected before the stream was returned") stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), eager_failure, healthy_stream] @@ -848,9 +1028,6 @@ class TestBedrockRealtimeProviderFailurePropagation: await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] - flushed = [json.loads(chunk.value.bytes_.decode("utf-8")) for chunk in stream.input_stream.sent] - assert [next(iter(event["event"])) for event in flushed][-2:] == ["promptEnd", "sessionEnd"] - assert stream.input_stream.closed with pytest.raises(BedrockError) as replay: await handler.async_realtime( @@ -877,8 +1054,8 @@ class TestBedrockRealtimeProviderFailurePropagation: @pytest.mark.asyncio async def test_readiness_failure_on_an_input_first_start_replays_the_input(self, stub_aws_sdk_client): handler = BedrockRealtime() - websocket = ConnectedClientWS([self.USER_TEXT]) - healthy_stream = ScriptedBedrockStream([]) + websocket = RealtimeClientWS([self.USER_TEXT]) + healthy_stream = SessionBoundBedrockStream() stub_aws_sdk_client["streams"] = [UnavailableBedrockStream(), healthy_stream] with pytest.raises(BedrockError) as failure: @@ -907,7 +1084,7 @@ class TestBedrockRealtimeProviderFailurePropagation: [self.SESSION_UPDATE, json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})] ) - with pytest.raises(RuntimeError, match="bedrock input stream reset"): + with pytest.raises(BedrockError) as failure: await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", websocket=websocket, @@ -915,6 +1092,8 @@ class TestBedrockRealtimeProviderFailurePropagation: **self.AWS_PARAMS, ) + assert failure.value.status_code == 503 + assert str(failure.value.__cause__) == "bedrock input stream reset", "the proxy log keeps the reset's cause" assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] await spend_dispatch["coro"] assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] @@ -923,10 +1102,10 @@ class TestBedrockRealtimeProviderFailurePropagation: async def test_success_dispatch_stamps_the_ownership_marker_only_when_spend_was_logged( self, stub_aws_sdk_client, spend_dispatch ): - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream(self.TEXT_TURN)] await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", - websocket=ConnectedClientWS([self.SESSION_UPDATE]), + websocket=RealtimeClientWS([self.SESSION_UPDATE]), logging_obj=spend_dispatch["logging_obj"], **self.AWS_PARAMS, ) @@ -935,15 +1114,315 @@ class TestBedrockRealtimeProviderFailurePropagation: assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True idle_logging = FakeLogging() - stub_aws_sdk_client["streams"] = [ScriptedBedrockStream([])] + stub_aws_sdk_client["streams"] = [SessionBoundBedrockStream()] await BedrockRealtime().async_realtime( model="amazon.nova-sonic-v1:0", - websocket=ConnectedClientWS([self.SESSION_UPDATE]), + websocket=RealtimeClientWS([self.SESSION_UPDATE]), logging_obj=idle_logging, **self.AWS_PARAMS, ) assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in idle_logging.model_call_details + @pytest.mark.asyncio + async def test_bedrock_stream_ending_mid_session_fails_the_session_and_keeps_partial_spend( + self, stub_aws_sdk_client, spend_dispatch + ): + handler: Final = BedrockRealtime() + websocket: Final = ConnectedClientWS([self.SESSION_UPDATE]) + stream: Final = ScriptedBedrockStream(self.TEXT_TURN) + stub_aws_sdk_client["streams"] = [stream] + logging_obj: Final = spend_dispatch["logging_obj"] + request_metadata: Final = {"user_api_key": "hashed-key"} + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": request_metadata} + + with pytest.raises(BedrockError) as failure: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == 503 + assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" + assert "response.done" in [json.loads(m)["type"] for m in websocket.sent_to_client] + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + status, error_information = _spend_row(logging_obj) + assert status == "failure", "the Logs page must show the dropped session as failed while it bills its turn" + assert error_information is not None + assert error_information.get("error_code") == "503" + assert error_information.get("error_message") == failure.value.message + assert request_metadata == {"user_api_key": "hashed-key"}, "the request's own metadata stays as it was sent" + + with pytest.raises(BedrockError) as replay: + await handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert replay.value.status_code == 400, "a router retry must not restart the dropped session" + assert str(replay.value).startswith(failure.value.message), "the app's error event leads with the drop" + assert websocket_close_reason(str(replay.value), fallback="Internal server error").startswith( + failure.value.message + ), "the length-capped 1011 close reason must still carry the drop" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_reaches_the_app_without_waiting_on_the_input_close( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = HangingCloseInputStream() + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert failure.value.status_code == status_code + assert stub_aws_sdk_client["client_closed"], "closing the SDK client still tears the dead stream down" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_failed_bedrock_output_leaves_no_pending_input_writer( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stream: Final = ScriptedBedrockStream(self.TEXT_TURN, receiver_type=receiver_type) + stream.input_stream = BodyWriterInputStream() + stub_aws_sdk_client["streams"] = [stream] + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + await asyncio.wait_for(stream.input_stream.body_writer, timeout=1) + await spend_dispatch["coro"] + + @pytest.mark.asyncio + async def test_a_dropped_session_logs_one_success_usage_record_and_a_failed_spend_row( + self, stub_aws_sdk_client, spend_dispatch + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + recorder: Final = LoggedEventRecorder() + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-sonic-v1:0", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop", + function_id="nova-sonic-drop", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-sonic-v1:0", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + ((logged_event, usage_kwargs),) = recorder.events + assert logged_event == "success" + usage_record: Final = recorder.payload("success") + assert usage_record["status"] == "success", "the router's failure event is each logger's one error record" + assert [event["type"] for event in usage_record["response"]["results"]] == ["response.done"] + assert logging_obj.model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + status, error_information = _spend_row_of(usage_kwargs) + assert status == "failure" + assert error_information is not None + assert error_information.get("error_code") == "503" + + @pytest.mark.asyncio + async def test_a_dropped_session_bills_its_usage_to_loggers_while_the_failure_event_runs( + self, stub_aws_sdk_client, spend_dispatch + ): + usage_event: Final = json.dumps( + {"event": {"usageEvent": {"totalInputTokens": 1000, "totalOutputTokens": 50, "totalTokens": 1050}}} + ) + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((*self.TEXT_TURN, usage_event))] + held: Final = HeldSuccessLogger() + recorder: Final = LoggedEventRecorder() + logging_obj: Final = LiteLLMLogging( + model="amazon.nova-2-5-sonic", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="nova-sonic-drop-order", + function_id="nova-sonic-drop-order", + dynamic_async_success_callbacks=[held, recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_from_kwargs( + kwargs={"litellm_metadata": {"user_api_key": "hashed-key"}}, + model="amazon.nova-2-5-sonic", + optional_params={}, + litellm_params={"_arealtime": True}, + custom_llm_provider="bedrock", + ) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-2-5-sonic", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + success_dispatch: Final = asyncio.create_task(spend_dispatch["coro"]) + await asyncio.wait_for(held.entered.wait(), timeout=5) + logging_obj.failure_handler(failure.value, "", logging_obj.start_time, datetime.now(), for_async_request=True) + await logging_obj.async_failure_handler(failure.value, "", logging_obj.start_time, datetime.now()) + held.release.set() + await success_dispatch + + usage_record: Final = recorder.payload("success") + assert recorder.payload("failure")["status"] == "failure" + assert usage_record["status"] == "success" + assert usage_record["total_tokens"] == 1050 + assert usage_record["response_cost"] > 0 + ((_, usage_kwargs),) = (logged for logged in recorder.events if logged[0] == "success") + status, _ = _spend_row_of(usage_kwargs) + assert status == "failure" + + @pytest.mark.asyncio + async def test_a_dropped_session_without_request_metadata_still_fails_and_bills_its_turn( + self, stub_aws_sdk_client, spend_dispatch + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream(self.TEXT_TURN)] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": None, "metadata": {}} + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + await spend_dispatch["coro"] + + assert failure.value.status_code == 503 + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("receiver_type", "status_code"), + [(ScriptedBedrockReceiver, 503), (BreakingBedrockReceiver, 424)], + ids=["output-ends", "output-raises"], + ) + async def test_a_session_bedrock_fails_before_any_logged_event_dispatches_no_success( + self, stub_aws_sdk_client, spend_dispatch, receiver_type, status_code: int + ): + stub_aws_sdk_client["streams"] = [ScriptedBedrockStream((), receiver_type=receiver_type)] + logging_obj: Final = spend_dispatch["logging_obj"] + logging_obj.model_call_details["litellm_params"] = {"litellm_metadata": {"user_api_key": "hashed-key"}} + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=ConnectedClientWS([self.SESSION_UPDATE]), + logging_obj=logging_obj, + **self.AWS_PARAMS, + ) + + assert failure.value.status_code == status_code + assert "coro" not in spend_dispatch, "a failed session with no usage gets no success event, as on base" + assert REALTIME_SESSION_SUCCESS_LOGGED_KEY not in logging_obj.model_call_details + + @pytest.mark.asyncio + async def test_bedrock_output_ending_right_after_session_end_is_a_clean_finish( + self, stub_aws_sdk_client, spend_dispatch + ): + stream: Final = SessionBoundBedrockStream(self.TEXT_TURN, input_stream_type=SlowClosingInputStream) + stub_aws_sdk_client["streams"] = [stream] + websocket: Final = RealtimeClientWS([self.SESSION_UPDATE]) + spend_dispatch["logging_obj"].model_call_details["litellm_params"] = { + "litellm_metadata": {"user_api_key": "hashed-key"} + } + + await asyncio.wait_for( + BedrockRealtime(close_timeout=30).async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ), + timeout=5, + ) + + assert _sent_event_names(stream.input_stream)[-1] == "sessionEnd" + assert not stream.input_stream.closed, "the input close must still be pending when the output ends" + assert websocket.closed + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + assert spend_dispatch["logging_obj"].model_call_details.get(REALTIME_SESSION_SUCCESS_LOGGED_KEY) is True + assert _spend_row(spend_dispatch["logging_obj"]) == ("success", None) + + @pytest.mark.asyncio + async def test_bedrock_output_ending_after_the_input_side_failed_still_fails_the_session( + self, stub_aws_sdk_client, spend_dispatch + ): + connection_reset: Final = asyncio.Event() + stream: Final = ScriptedBedrockStream( + self.TEXT_TURN, + receiver_type=lambda payloads: OutputEndsAfterConnectionResetReceiver(payloads, connection_reset), + ) + stream.input_stream = ConnectionResetOnAudioInputStream(connection_reset) + stub_aws_sdk_client["streams"] = [stream] + websocket: Final = ConnectedClientWS([self.SESSION_UPDATE, self.USER_AUDIO]) + + with pytest.raises(BedrockError) as failure: + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=websocket, + logging_obj=spend_dispatch["logging_obj"], + **self.AWS_PARAMS, + ) + + assert (failure.value.status_code, failure.value.message) == ( + 503, + "Bedrock realtime stream ended before the session closed", + ), "the app sees one cause whichever side of the stream reports the reset first" + assert not websocket.closed, "the proxy route owns the client-facing error event and 1011 close" + await spend_dispatch["coro"] + assert [event["type"] for event in spend_dispatch["events"]] == ["response.done"] + @pytest.mark.asyncio async def test_stream_failure_after_client_disconnect_is_not_a_provider_failure(self, stub_aws_sdk_client): stream = ScriptedBedrockStream([], receiver_type=BreakingBedrockReceiver) @@ -982,7 +1461,13 @@ class TestBedrockRealtimeProviderFailurePropagation: with pytest.raises(ServiceUnavailableException): await handler._forward_client_to_bedrock( - client_ws, stream, BedrockRealtimeConfig(), "amazon.nova-sonic-v1:0", {}, FakeLogging() + client_ws, + stream, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + {}, + FakeLogging(), + signals=SessionEndSignals(), ) assert client_ws.sent_to_client == [] @@ -1128,7 +1613,6 @@ class TestBedrockRealtimeSdkLifecycle: ) assert stub_aws_sdk_client["client_closed"] - assert stub_aws_sdk_client["input_closed_before_client_close"] @pytest.mark.asyncio async def test_client_without_close_completes_session(self, monkeypatch): diff --git a/tests/unit/llms/openai/live/test_handler.py b/tests/unit/llms/openai/live/test_handler.py index ec1130376d0..bb7cda99e79 100644 --- a/tests/unit/llms/openai/live/test_handler.py +++ b/tests/unit/llms/openai/live/test_handler.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Coroutine +from collections.abc import Callable, Coroutine from datetime import datetime from typing import Final, NoReturn, cast @@ -13,11 +13,13 @@ from websockets.frames import Close from websockets.http11 import Response import litellm +from litellm import Router from litellm.constants import REALTIME_SESSION_SUCCESS_LOGGED_KEY, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.cost_calculator import handle_realtime_stream_cost_calculation from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.openai.live import handler as live_handler from litellm.llms.openai.live.handler import LiveSessionStart, OpenAILiveSessions, parse_live_session_start +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.utils import LiteLLMRealtimeStreamLoggingObject, Usage _LIVE_EVENT_FRAME_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -1206,3 +1208,91 @@ async def test_async_live_session_reports_redacted_server_error_and_bounds_close finally: if worker.coroutine is not None: await worker.coroutine + + +_LIVE_SESSION_STARTED: Final = '{"type":"session.started","session":{"model":"gpt-live-1"}}' +_LIVE_SESSION_CLOSED: Final = '{"type":"session.closed","usage":{"seconds":4}}' + + +def _upstream_closing_the_session() -> _FakeUpstream: + return _FakeUpstream(client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED, _LIVE_SESSION_CLOSED)) + + +def _upstream_failing_after_a_frame() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), + upstream_frames=(_LIVE_SESSION_STARTED,), + close_error=ConnectionClosedError(Close(code=1011, reason="upstream failed"), None), + ) + + +def _upstream_dropping_after_a_frame() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), upstream_frames=(_LIVE_SESSION_STARTED,), close_error=ConnectionClosedError(None, None) + ) + + +def _upstream_failing_a_client_send() -> _FakeUpstream: + return _FakeUpstream( + client_frames=(), + upstream_frames=(_LIVE_SESSION_STARTED,), + close_response=_LIVE_SESSION_CLOSED, + fail_on_send_type="conversation.item.create", + send_error=OSError("upstream socket write failed"), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("upstream_factory", "counted_successes"), + [ + (_upstream_closing_the_session, 1), + (_upstream_failing_after_a_frame, 0), + (_upstream_dropping_after_a_frame, 0), + (_upstream_failing_a_client_send, 0), + ], + ids=["session-closed", "upstream-1011-after-a-frame", "upstream-1006-after-a-frame", "upstream-send-failed"], +) +async def test_live_session_counts_a_router_success_only_when_it_ended_cleanly( + upstream_factory: Callable[[], _FakeUpstream], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + {"model_name": "live-voice", "litellm_params": {"model": "openai/gpt-live-1"}, "model_info": {"id": "live-a"}} + ], + num_retries=0, + ) + worker: Final = _FakeLoggingWorker() + + async def _arealtime(litellm_logging_obj: Logging, **kwargs: object) -> None: + await OpenAILiveSessions( + logging_worker=worker, websocket_connector=_FakeConnector(upstream_factory()) + ).async_live_session( + model="gpt-live-1", + websocket=_FakeClientWebSocket(messages=('{"type":"conversation.item.create"}',)), + logging_obj=litellm_logging_obj, + session_start={"type": "session.start", "session": {"model": "gpt-live-1"}}, + api_base="https://api.openai.com/", + api_key="live-test-key", + ) + + await asyncio.wait_for( + router._ageneric_api_call_with_fallbacks( + model="live-voice", + original_function=_arealtime, + litellm_logging_obj=Logging( + model="gpt-live-1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id="live-session", + function_id="live-session", + ), + ), + timeout=5, + ) + + if worker.coroutine is not None: + worker.coroutine.close() + assert get_deployment_successes_for_current_minute(router, "live-a") == counted_successes diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index bed185ba8f5..f9c234be55a 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3711,6 +3711,255 @@ async def test_failure_event_refunds_reservation_for_input_only_estimate_v3(): assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_first", [True, False]) +@pytest.mark.parametrize( + ("response_obj", "used_tokens"), + [ + ([], 0), + (ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)), 30), + ], +) +async def test_a_request_logging_a_failure_and_a_success_settles_its_reservation_once_v3( + failure_first: bool, response_obj: object, used_tokens: int +) -> None: + api_key: Final = hash_token("sk-failure-and-success") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens") + await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "dropped-call") + kwargs: Final = { + "litellm_call_id": "dropped-call", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = partial( + handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + log_success: Final = partial( + handler.async_log_success_event, kwargs=dict(kwargs), response_obj=response_obj, start_time=now, end_time=now + ) + + for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure): + await log_event() + + assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == used_tokens + + +class _YieldingIncrementInMemoryCache(InMemoryCache): + async def async_increment(self, key: str, value: float, **kwargs: object) -> float: + await asyncio.sleep(0) + return await super().async_increment(key, value, **kwargs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("success_starts_first", [True, False]) +@pytest.mark.parametrize("failure_path", ["failure_event", "post_call_failure_hook"]) +async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_once_v3( + success_starts_first: bool, failure_path: str +) -> None: + api_key: Final = hash_token("sk-concurrent-failure-and-success") + model: Final = "gpt-4o-mini" + project: Final = "proj-concurrent" + local_cache: Final = DualCache(in_memory_cache=_YieldingIncrementInMemoryCache()) + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + caller: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=10_000, + project_id=project, + project_metadata={"model_itpm_limit": {model: 10_000}, "model_otpm_limit": {model: 10_000}}, + ) + counter_keys: Final = ( + handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens"), + handler.create_rate_limit_keys( + key="model_per_project_itpm", value=f"{project}:{model}", rate_limit_type="tokens" + ), + handler.create_rate_limit_keys( + key="model_per_project_otpm", value=f"{project}:{model}", rate_limit_type="tokens" + ), + ) + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=local_cache, + data={ + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "concurrent-call", + }, + call_type="completion", + ) + kwargs: Final = { + "litellm_call_id": "concurrent-call", + "model": model, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = ( + partial(handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now) + if failure_path == "failure_event" + else partial( + handler.async_post_call_failure_hook, + request_data=dict(kwargs), + original_exception=Exception("stream ended"), + user_api_key_dict=caller, + ) + ) + log_success: Final = partial( + handler.async_log_success_event, + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)), + start_time=now, + end_time=now, + ) + + await asyncio.gather( + *((log_success(), log_failure()) if success_starts_first else (log_failure(), log_success())) + ) + + assert tuple([int(await local_cache.async_get_cache(key=key) or 0) for key in counter_keys]) == (30, 20, 10) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("second_call_type", "second_model"), + [("call_mcp_tool", "mcp-tool-call"), ("completion", "gpt-4o-mini")], +) +async def test_a_pre_call_after_a_failure_leaves_the_success_charging_project_io_once_v3( + second_call_type: str, second_model: str +) -> None: + api_key: Final = hash_token("sk-retried-request-with-mcp-tool") + model: Final = "gpt-4o-mini" + project: Final = "proj-retried-mcp" + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + caller: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=100_000, + project_id=project, + project_metadata={"model_itpm_limit": {model: 10_000}, "model_otpm_limit": {model: 10_000}}, + ) + request: Final = { + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "retried-call-with-mcp-tool", + } + kwargs: Final = { + "litellm_call_id": "retried-call-with-mcp-tool", + "model": model, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + itpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_itpm", value=f"{project}:{model}", rate_limit_type="tokens" + ) + otpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_otpm", value=f"{project}:{model}", rate_limit_type="tokens" + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, cache=local_cache, data=dict(request), call_type="completion" + ) + await handler.async_log_failure_event(kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now) + await handler.async_pre_call_hook( + user_api_key_dict=caller, + cache=local_cache, + data={**request, "model": second_model}, + call_type=second_call_type, + ) + await handler.async_log_success_event( + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + assert ( + int(await local_cache.async_get_cache(key=itpm_key) or 0), + int(await local_cache.async_get_cache(key=otpm_key) or 0), + ) == (20, 7) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("steps", "key_tpm_limit", "expected_counters"), + [ + (("failure", "mcp-tool-call", "failure"), 100_000, (0, 0, 0, 0)), + (("failure", "tool-model-with-io-limit", "post_call_failure"), None, (0, 0, 0, 0)), + (("failure", "mcp-tool-call", "post_call_failure", "success"), 100_000, (20, 7, 0, 0)), + (("tool-model-over-io-limit", "success"), 100_000, (20, 7, 0, 0)), + ], +) +async def test_a_nested_mcp_pre_call_settles_each_project_io_hold_once_v3( + steps: tuple[str, ...], key_tpm_limit: Optional[int], expected_counters: tuple[int, int, int, int] +) -> None: + api_key: Final = hash_token("sk-nested-mcp-hold-flags") + model: Final = "gpt-4o-mini" + tool_model: Final = "tool-model-with-io-limit" + project: Final = "proj-nested-mcp-hold-flags" + io_limits: Final = {model: 10_000, tool_model: 10_000, "tool-model-over-io-limit": 1} + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + caller: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=key_tpm_limit, + project_id=project, + project_metadata={"model_itpm_limit": io_limits, "model_otpm_limit": io_limits}, + ) + request: Final = { + "model": model, + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "nested-mcp-hold-flags", + } + kwargs: Final = { + "litellm_call_id": "nested-mcp-hold-flags", + "model": model, + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + counter_keys: Final = tuple( + handler.create_rate_limit_keys( + key=f"model_per_project_{kind}", value=f"{project}:{scope}", rate_limit_type="tokens" + ) + for scope, kind in ((model, "itpm"), (model, "otpm"), (tool_model, "itpm"), (tool_model, "otpm")) + ) + mcp_pre_call: Final = partial( + handler.async_pre_call_hook, user_api_key_dict=caller, cache=local_cache, call_type="call_mcp_tool" + ) + + await handler.async_pre_call_hook( + user_api_key_dict=caller, cache=local_cache, data=dict(request), call_type="completion" + ) + for step in steps: + match step: + case "failure": + await handler.async_log_failure_event( + kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + case "post_call_failure": + await handler.async_post_call_failure_hook( + request_data=dict(request), + original_exception=HTTPException(status_code=429, detail="upstream rate limited"), + user_api_key_dict=caller, + ) + case "success": + await handler.async_log_success_event( + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + case "tool-model-over-io-limit": + with pytest.raises(ProxyRateLimitError): + await mcp_pre_call(data={**request, "model": step}) + case _: + await mcp_pre_call(data={**request, "model": step}) + + assert tuple([int(await local_cache.async_get_cache(key=key) or 0) for key in counter_keys]) == expected_counters + + @pytest.mark.asyncio async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3(): """ @@ -4198,6 +4447,125 @@ async def test_failure_event_settles_project_itpm_otpm_at_recovered_partial_usag assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_first", [True, False]) +async def test_a_request_logging_a_failure_and_a_success_settles_its_project_itpm_otpm_once_v3( + failure_first: bool, +) -> None: + api_key: Final = hash_token("sk-failure-and-success-project-io") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + await handler.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + api_key=api_key, + project_id="proj-dropped", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ), + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": "dropped-project-call", + }, + call_type="completion", + ) + itpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_itpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens" + ) + otpm_key: Final = handler.create_rate_limit_keys( + key="model_per_project_otpm", value="proj-dropped:gpt-4o-mini", rate_limit_type="tokens" + ) + kwargs: Final = { + "litellm_call_id": "dropped-project-call", + "model": "gpt-4o-mini", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + now: Final = datetime.now() + log_failure: Final = partial( + handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now + ) + log_success: Final = partial( + handler.async_log_success_event, + kwargs=dict(kwargs), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + for log_event in (log_failure, log_success) if failure_first else (log_success, log_failure): + await log_event() + + assert int(await local_cache.async_get_cache(key=itpm_key) or 0) == 20 + assert int(await local_cache.async_get_cache(key=otpm_key) or 0) == 7 + + +@pytest.mark.asyncio +async def test_a_reservation_made_after_a_released_one_in_the_same_context_settles_at_its_usage_v3() -> None: + api_key: Final = hash_token("sk-second-reservation") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache)) + user_api_key_dict: Final = UserAPIKeyAuth( + api_key=api_key, + tpm_limit=10_000, + project_id="proj-second", + project_metadata={ + "model_itpm_limit": {"gpt-4o-mini": 10_000}, + "model_otpm_limit": {"gpt-4o-mini": 10_000}, + }, + ) + now: Final = datetime.now() + + async def reserve(call_id: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 50, + "litellm_call_id": call_id, + }, + call_type="completion", + ) + + def logged(call_id: str) -> dict[str, object]: + return { + "litellm_call_id": call_id, + "model": "gpt-4o-mini", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + } + + await reserve("first-call") + await handler.async_log_failure_event(kwargs=logged("first-call"), response_obj=None, start_time=now, end_time=now) + await reserve("second-call") + await handler.async_log_success_event( + kwargs=logged("second-call"), + response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)), + start_time=now, + end_time=now, + ) + + counters: Final = { + "tokens": ("api_key", api_key), + "itpm": ("model_per_project_itpm", "proj-second:gpt-4o-mini"), + "otpm": ("model_per_project_otpm", "proj-second:gpt-4o-mini"), + } + settled: Final = { + name: int( + await local_cache.async_get_cache( + key=handler.create_rate_limit_keys(key=key, value=value, rate_limit_type="tokens") + ) + or 0 + ) + for name, (key, value) in counters.items() + } + assert settled == {"tokens": 27, "itpm": 20, "otpm": 7} + # ----------------------- Per-MCP-server rate limiting (v3) ----------------------- diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index a4e78a48a1a..21b5bd300c1 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3491,6 +3491,40 @@ async def test_sidecar_ignores_an_undecodable_event(): # test-quality-ok: a dis mock_proxy_logging.db_spend_update_writer.update_database.assert_not_awaited() +def _spend_row_error_information(update_database_kwargs: Mapping[str, object]) -> dict[str, object]: + row: Final = get_logging_payload( + kwargs=cast(dict, update_database_kwargs["kwargs"]), + response_obj=update_database_kwargs["completion_response"], + start_time=cast(datetime, update_database_kwargs["start_time"]), + end_time=cast(datetime, update_database_kwargs["end_time"]), + ) + return json.loads(row["metadata"])["error_information"] + + +@pytest.mark.asyncio +async def test_failure_hook_spend_row_truncates_a_long_provider_error_exactly_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "100") + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + provider_error: Final = "upstream failure " + "x" * 1000 + + await logger.async_post_call_failure_hook( + request_data={"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}, + original_exception=litellm.InternalServerError(message=provider_error, llm_provider="openai", model="gpt-4o"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + stored_message: Final = _spend_row_error_information(writer.update_database.call_args.kwargs)["error_message"] + assert isinstance(stored_message, str) + assert stored_message.count("skipped") == 1 + kept_chars: Final = len(stored_message.split("... (")[0]) + len(stored_message.rsplit(") ...", 1)[1]) + skipped_chars: Final = int(stored_message.split("skipped ", 1)[1].split(" chars", 1)[0]) + assert kept_chars + skipped_chars > len(provider_error) + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_model_rejection(): raw_model: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" @@ -3504,9 +3538,7 @@ async def test_async_post_call_failure_hook_persists_no_raw_model_on_an_unknown_ user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), ) - error_information: Final = writer.update_database.call_args.kwargs["kwargs"]["litellm_params"]["metadata"][ - "error_information" - ] + error_information: Final = _spend_row_error_information(writer.update_database.call_args.kwargs) assert "medical records" not in json.dumps(error_information) assert ( error_information["error_message"] diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index e43a72541c7..bfb7a9d6199 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -3060,7 +3060,7 @@ def test_retry_success_charges_released_project_io_reservations( stash.itpm_reserved_scopes = frozenset({itpm_scope}) stash.otpm_reserved_tokens = 60 stash.otpm_reserved_scopes = frozenset({otpm_scope}) - stash.reservation_released = True + stash.mark_reservation_released() operations = handler._build_io_token_reservation_ops( {}, diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index f73d33cc93e..d5771dff626 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -7486,6 +7486,46 @@ def test_failure_error_information_masks_ip_addresses_in_long_text_and_at_its_tr assert sanitized["error_code"] == "500" +def test_spend_row_sanitizes_the_session_failure_a_realtime_success_row_carries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy.proxy_server import general_settings + + monkeypatch.setitem(general_settings, "store_prompts_in_spend_logs", False) + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", "200") + provider_error: Final = 'stream ended {"messages": [{"role": "user", "content": "private-prompt"}]} ' + "x" * 2000 + now: Final = datetime.datetime.now(timezone.utc) + + row: Final = get_logging_payload( + kwargs={ + "model": "bedrock/amazon.nova-sonic-v1:0", + "call_type": "_arealtime", + "litellm_params": { + "metadata": { + "user_api_key": "hashed-key", + "status": "failure", + "error_information": { + "error_code": "503", + "error_class": "BedrockError", + "error_message": provider_error, + "llm_provider": "bedrock", + }, + } + }, + }, + response_obj=None, + start_time=now, + end_time=now, + ) + + stored: Final = json.loads(row["metadata"])["error_information"] + assert row["status"] == "failure" + assert "private-prompt" not in stored["error_message"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["error_message"] + assert len(stored["error_message"]) < len(provider_error) + assert stored["error_class"] == "BedrockError" + + @pytest.mark.parametrize( "address", ["203.0.113.9", "2001:db8::9", "1:2:3:4:5:6:7:8", "1:2:3:4:5:6:7::", "::1:2:3:4:5:6:7", "fe80::1"] ) diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index b46a57ad5f1..5ad6eb819ef 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -3,8 +3,9 @@ import functools import gc import json import os +import uuid import weakref -from collections.abc import Awaitable, Callable, Iterator, Mapping +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping from datetime import datetime, timezone from types import MethodType from typing import Final @@ -13,6 +14,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx +from websockets.exceptions import ConnectionClosed +from websockets.frames import Close import litellm from litellm import Router @@ -21,8 +24,14 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.realtime_errors import close_after_upstream_handshake_refusal +from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import VertexChirpRealtimeConfig from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.types.litellm_params import RoutingStrategyName from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( @@ -540,6 +549,242 @@ async def test_deployment_callback_on_success_counts_a_responses_websocket_turn( assert await router.cache.async_get_cache(key=tpm_key) == 17 +class _DiscardingLoggingWorker: + def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, object, None]) -> None: + async_coroutine.close() + + +async def _client_never_sends() -> str: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +def _realtime_client_ws() -> MagicMock: + client_ws: Final = MagicMock() + client_ws.headers = {} + client_ws.receive_text = AsyncMock(side_effect=_client_never_sends) + client_ws.send_text = AsyncMock() + client_ws.close = AsyncMock() + return client_ws + + +def _realtime_session_logging() -> LiteLLMLogging: + return LiteLLMLogging( + model="gpt-realtime-2.1", + messages=[], + stream=False, + call_type="_arealtime", + start_time=datetime.now(), + litellm_call_id=str(uuid.uuid4()), + function_id="realtime-session", + ) + + +async def _relay_upstream( + litellm_logging_obj: LiteLLMLogging, + *upstream_frames: bytes | Exception, + client_ws: MagicMock | None = None, + provider_config: BaseRealtimeConfig | None = None, + upstream_stays_open: bool = False, +) -> None: + backend_ws: Final = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=_frames_then_silence(*upstream_frames) if upstream_stays_open else list(upstream_frames) + ) + backend_ws.send = AsyncMock() + relay: Final = RealTimeStreaming( + client_ws or _realtime_client_ws(), + backend_ws, + litellm_logging_obj, + provider_config=provider_config, + model="gpt-realtime-2.1", + logging_worker=_DiscardingLoggingWorker(), + ) + await asyncio.wait_for(relay.bidirectional_forward(), timeout=2) + + +_REALTIME_SESSION_CREATED: Final = json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode() + + +async def _clean_realtime_session(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1000, ""), None)) + + +async def _upstream_internal_error_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream( + litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(Close(1011, "upstream failed"), None) + ) + + +async def _upstream_drop_without_close_frame_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, ConnectionClosed(None, None)) + + +async def _upstream_close_before_any_frame(litellm_logging_obj: LiteLLMLogging) -> None: + await _relay_upstream(litellm_logging_obj, ConnectionClosed(Close(1008, "model not found"), None)) + + +def _frames_then_silence(*frames: bytes | str) -> Callable[[], Awaitable[bytes | str]]: + pending: Final = iter(frames) + + async def _next_frame() -> bytes | str: + frame: Final = next(pending, None) + if frame is None: + await asyncio.Event().wait() + raise AssertionError("unreachable") + return frame + + return _next_frame + + +def _client_ws_sending(*client_frames: str) -> MagicMock: + client_ws: Final = _realtime_client_ws() + client_ws.receive_text = AsyncMock(side_effect=_frames_then_silence(*client_frames)) + return client_ws + + +async def _vertex_access_token() -> str: + return "token" + + +async def _client_frame_the_provider_rejects(litellm_logging_obj: LiteLLMLogging) -> None: + deferred_setup_provider: Final = VertexChirpRealtimeConfig( + resolve_access_token=_vertex_access_token, project="proj-1", location="us" + ) + await _relay_upstream( + litellm_logging_obj, + client_ws=_client_ws_sending( + json.dumps( + { + "type": "session.update", + "session": {"type": "transcription", "audio": {"input": {"format": {"type": "audio/pcmu"}}}}, + } + ) + ), + provider_config=deferred_setup_provider, + upstream_stays_open=True, + ) + + +async def _client_hangs_up_after_a_frame(litellm_logging_obj: LiteLLMLogging) -> None: + client_ws: Final = _realtime_client_ws() + client_ws.receive_text = AsyncMock(side_effect=ConnectionClosed(Close(1000, ""), None)) + await _relay_upstream(litellm_logging_obj, _REALTIME_SESSION_CREATED, client_ws=client_ws, upstream_stays_open=True) + + +async def _refused_handshake(litellm_logging_obj: LiteLLMLogging) -> None: + await close_after_upstream_handshake_refusal(_realtime_client_ws(), 403) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("realtime_session", "counted_successes"), + [ + (_clean_realtime_session, 1), + (_upstream_close_before_any_frame, 0), + (_refused_handshake, 0), + (_upstream_internal_error_after_a_frame, 0), + (_upstream_drop_without_close_frame_after_a_frame, 0), + (_client_frame_the_provider_rejects, 0), + (_client_hangs_up_after_a_frame, 1), + ], + ids=[ + "clean-session", + "upstream-close-before-any-frame", + "refused-handshake", + "upstream-1011-after-a-frame", + "upstream-1006-after-a-frame", + "client-frame-the-provider-rejects", + "client-hangs-up-after-a-frame", + ], +) +async def test_realtime_session_counts_a_router_success_only_when_it_ended_without_a_failure( + realtime_session: Callable[[LiteLLMLogging], Awaitable[None]], counted_successes: int +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "realtime-voice", + "litellm_params": {"model": "openai/gpt-realtime-2.1"}, + "model_info": {"id": "realtime-a"}, + } + ], + num_retries=0, + ) + + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: + await realtime_session(litellm_logging_obj) + + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", original_function=_arealtime, litellm_logging_obj=_realtime_session_logging() + ) + + assert get_deployment_successes_for_current_minute(router, "realtime-a") == counted_successes + + +@pytest.mark.asyncio +@pytest.mark.parametrize("realtime_model", ["bedrock/amazon.nova-sonic-v1:0", "openai/gpt-realtime-2.1"]) +@pytest.mark.parametrize(("clean_sessions", "cooled_down"), [(10, False), (3, True)]) +async def test_realtime_cooldown_reads_the_failure_rate_of_the_sessions_a_deployment_served( + realtime_model: str, clean_sessions: int, cooled_down: bool +) -> None: + import time + + provider, _, provider_model = realtime_model.partition("/") + router: Final = Router( + model_list=[ + { + "model_name": "realtime-voice", + "litellm_params": {"model": realtime_model, "order": order}, + "model_info": {"id": deployment_id}, + } + for order, deployment_id in ((1, "realtime-a"), (2, "realtime-b")) + ], + num_retries=0, + ) + drop: Final = litellm.ServiceUnavailableError( + message="realtime provider stream ended before the session closed", + llm_provider=provider, + model=provider_model, + ) + + def realtime_session(failure: Exception | None): + async def _arealtime(litellm_logging_obj: LiteLLMLogging, **kwargs: object) -> None: + if failure is not None: + raise failure + await _clean_realtime_session(litellm_logging_obj) + + return _arealtime + + for _ in range(clean_sessions): + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(None), + litellm_logging_obj=_realtime_session_logging(), + ) + + async def drop_session() -> bool: + with pytest.raises(litellm.ServiceUnavailableError): + await router._ageneric_api_call_with_fallbacks( + model="realtime-voice", + original_function=realtime_session(drop), + litellm_logging_obj=_realtime_session_logging(), + ) + return router.deployment_callback_on_failure( + kwargs={ + "exception": drop, + "litellm_params": {"metadata": {"model_group": "realtime-voice"}, "model_info": {"id": "realtime-a"}}, + }, + completion_response=None, + start_time=time.time(), + end_time=time.time(), + ) + + drop_cooldowns: Final = [await drop_session() for _ in range(5)] + + assert any(drop_cooldowns) is cooled_down + + async def _kwargs_routed_to(router: Router, model: str, function_name: str, **request: object) -> dict[str, object]: routed: Final[dict[str, object]] = {} From 482f3c37ed6de423daec8e65f36142cc5408d808 Mon Sep 17 00:00:00 2001 From: tavren27 <348969968+tavren27@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:02:21 -0800 Subject: [PATCH 20/34] fix(azure_ai): keep azure_ad_token out of the request body (#49769) * fix(azure_ai): keep azure_ad_token out of the request body azure_ad_token is not a litellm-owned param, so get_optional_params routes it into extra_body and the azure_ai chat and Claude transformations merge it into the JSON body. Azure AI Foundry then rejects the call with 400 Unknown parameter / Extra inputs are not permitted. The Authorization header already comes from litellm_params, so drop the extra_body copy once at the top of the azure_ai dispatch, as the azure branches do. Type litellm.open_ai_chat_completion_models and open_ai_text_completion_models as Set[str] so the basedpyright gate does not grow from the untyped optional_params read. * fix(azure_ai): keep Entra credentials out of every azure_ai request body * fix(azure_ai): keep the Entra token out of the image edit form * fix(azure_ai): keep Entra credentials out of image edit extra_body too --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/images/main.py | 11 +- litellm/utils.py | 14 +- .../llms/azure_ai/test_azure_ai_entra_auth.py | 164 ++++++++++++++++++ 3 files changed, 184 insertions(+), 5 deletions(-) diff --git a/litellm/images/main.py b/litellm/images/main.py index f37b252fa45..27917cf9289 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -28,7 +28,7 @@ from litellm.litellm_core_utils.mock_functions import mock_image_generation from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params +from litellm.utils import exception_type, filter_out_litellm_params, get_litellm_params, header_auth_params #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() @@ -988,10 +988,15 @@ def image_edit( or custom_llm_provider == "azure" or custom_llm_provider in litellm.openai_compatible_providers ): + dropped_form_params: Final = header_auth_params(custom_llm_provider) image_edit_request_params.update( flatten_form_field_values( - non_default_params, - extra_body if isinstance(extra_body, dict) else None, + {key: value for key, value in non_default_params.items() if key not in dropped_form_params}, + ( + {key: value for key, value in extra_body.items() if key not in dropped_form_params} + if isinstance(extra_body, dict) + else None + ), ) ) diff --git a/litellm/utils.py b/litellm/utils.py index 108c6fae982..d881289deed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6161,6 +6161,14 @@ EXTRA_BODY_ROUTING_KEYS: Final = frozenset({"model"}) LITELLM_ONLY_PARAMS: Final = frozenset(("cache_control_injection_points",)) +def header_auth_params(custom_llm_provider: str) -> frozenset[str]: + if custom_llm_provider != "azure_ai": + return frozenset() + from litellm.llms.azure_ai.common_utils import AZURE_ENTRA_CREDENTIAL_PARAM_KEYS + + return AZURE_ENTRA_CREDENTIAL_PARAM_KEYS + + def add_provider_specific_params_to_optional_params( optional_params: dict, passed_params: dict, @@ -6195,8 +6203,10 @@ def add_provider_specific_params_to_optional_params( **extra_body, } - dropped_keys: Final = EXTRA_BODY_ROUTING_KEYS | frozenset( - param for param in (additional_drop_params or ()) if isinstance(param, str) + dropped_keys: Final = ( + EXTRA_BODY_ROUTING_KEYS + | header_auth_params(custom_llm_provider) + | frozenset(param for param in (additional_drop_params or ()) if isinstance(param, str)) ) ensure_extra_body_is_safe: Final = litellm_utils.ensure_extra_body_is_safe optional_params["extra_body"] = ensure_extra_body_is_safe( diff --git a/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py b/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py index 908e727bc4c..d0c1c4a6fb2 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_entra_auth.py @@ -5,9 +5,15 @@ Every azure_ai route must authenticate with an Entra ID token when no API key is instead of requiring an API key. """ +import json +import queue +from collections.abc import Mapping +from typing import Final from unittest.mock import patch +import httpx import pytest +import respx import litellm from litellm.llms.azure_ai.common_utils import ( @@ -16,6 +22,8 @@ from litellm.llms.azure_ai.common_utils import ( has_azure_entra_params, resolve_azure_ai_agent_auth_header, ) +from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.custom_httpx.http_handler import HTTPHandler ENTRA_PARAMS = {"azure_ad_token": "entra-token"} @@ -290,3 +298,159 @@ async def test_agent_auth_header_is_the_entra_bearer(): headers = await resolve_azure_ai_agent_auth_header({"azure_ad_token": "entra-token"}) assert headers == {"Authorization": "Bearer entra-token"} + + +OPENAI_CHAT_COMPLETION: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +OPENAI_EMBEDDING: Final = { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2]}], + "model": "cohere-embed-v3-english", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, +} +ANTHROPIC_MESSAGE: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "OK"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} + + +class _PreCallOptionalParamsCapture(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.optional_params: tuple[dict[str, object], ...] = () + + def log_pre_api_call(self, model: str, messages: list[object], kwargs: dict[str, dict[str, object]]) -> None: + self.optional_params = (*self.optional_params, kwargs["optional_params"]) + + +@pytest.mark.parametrize( + ("model", "provider_response"), + [ + pytest.param("azure_ai/gpt-6.1-sol", OPENAI_CHAT_COMPLETION, id="openai-shaped"), + pytest.param("azure_ai/claude-sonnet-5-5", ANTHROPIC_MESSAGE, id="claude"), + ], +) +def test_completion_sends_the_entra_token_only_as_the_bearer_header( + model: str, provider_response: Mapping[str, object], monkeypatch: pytest.MonkeyPatch +): + capture: Final = _PreCallOptionalParamsCapture() + monkeypatch.setattr(litellm, "callbacks", [capture]) + sent: Final[queue.Queue[httpx.Request]] = queue.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + sent.put_nowait(request) + return httpx.Response(200, json=provider_response) + + response: Final = litellm.completion( + model=model, + api_base="https://example-resource.services.ai.azure.com", + azure_ad_token="entra-token", + messages=[{"role": "user", "content": "hi"}], + client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), + ) + + request: Final = sent.get_nowait() + assert request.headers["Authorization"] == "Bearer entra-token" + assert "azure_ad_token" not in json.loads(request.content) + assert response.choices[0].message.content == "OK" + assert capture.optional_params + assert "entra-token" not in json.dumps(capture.optional_params, default=str) + + +@pytest.mark.respx(assert_all_called=True) +def test_embedding_sends_the_entra_token_only_as_the_bearer_header(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json=OPENAI_EMBEDDING) + ) + + litellm.embedding( + model="azure_ai/cohere-embed-v3-english", + api_base="https://example-resource.services.ai.azure.com", + azure_ad_token="entra-token", + input=["hello"], + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert "azure_ad_token" not in json.loads(request.content) + + +IMAGE_RESPONSE: Final = {"created": 1, "data": [{"b64_json": "aGk="}]} +PNG_BYTES: Final = b"\x89PNG\r\n\x1a\n" + b"\x00" * 32 + + +@pytest.mark.respx(assert_all_called=True) +@pytest.mark.parametrize( + "extra_kwargs", + [{}, {"extra_body": {"azure_ad_token": "entra-token", "client_secret": "entra-token"}}], + ids=["kwarg", "extra_body"], +) +def test_image_edit_sends_the_entra_token_only_as_the_bearer_header( + respx_mock: respx.MockRouter, extra_kwargs: dict[str, dict[str, str]] +): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json=IMAGE_RESPONSE) + ) + + litellm.image_edit( + model="azure_ai/FLUX.1-Kontext-pro", + api_base="https://example-resource.services.ai.azure.com", + api_version="2025-04-01-preview", + azure_ad_token="entra-token", + image=PNG_BYTES, + prompt="add a hat", + **extra_kwargs, + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert b"entra-token" not in request.content + + +@pytest.mark.respx(assert_all_called=True) +def test_image_generation_sends_the_entra_token_only_as_the_bearer_header(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json=IMAGE_RESPONSE) + ) + + litellm.image_generation( + model="azure_ai/FLUX-1.1-pro", + api_base="https://example-resource.services.ai.azure.com", + azure_ad_token="entra-token", + prompt="a red circle", + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert "azure_ad_token" not in json.loads(request.content) + + +@pytest.mark.respx(assert_all_called=True) +def test_transcription_sends_the_entra_token_only_as_the_bearer_header(respx_mock: respx.MockRouter): + route: Final = respx_mock.post(url__startswith="https://example-resource.services.ai.azure.com").mock( + return_value=httpx.Response(200, json={"text": "hi"}) + ) + + litellm.transcription( + model="azure_ai/whisper", + api_base="https://example-resource.services.ai.azure.com", + api_version="2024-06-01", + azure_ad_token="entra-token", + file=("hello.wav", b"RIFF" + b"\x00" * 32), + ) + + request: Final = route.calls.last.request + assert request.headers["Authorization"] == "Bearer entra-token" + assert b"entra-token" not in request.content From 28b4300be2f8b31beee82fa21b50608772a1c14c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:25:43 +0000 Subject: [PATCH 21/34] fix(typesafe): mark jev-latest as answering Box questions now that it points to Jev 1.14 (#49835) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 3 ++- model_prices_and_context_window.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d7c829f6e2d..be93d6fcd9b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -73102,7 +73102,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "typesafe/jev-preview": { "input_cost_per_token": 4.2e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d7c829f6e2d..be93d6fcd9b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -73102,7 +73102,8 @@ "max_input_tokens": 65536, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://docs.typesafe.ai/models" + "source": "https://docs.typesafe.ai/models", + "supports_box_questions": true }, "typesafe/jev-preview": { "input_cost_per_token": 4.2e-08, From 7bd342d7eadba4e9ebbc393c44d36719bb29c7f6 Mon Sep 17 00:00:00 2001 From: paveln83 <349044532+paveln83@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:56:08 -0800 Subject: [PATCH 22/34] fix(proxy): let a config license unlock enterprise settings and spell "environment" right in license errors (#49745) * fix(proxy): spell "environment" right in the allowed_ips and AWS KMS V2 license errors * fix(proxy): read general_settings.litellm_license before the allowed_ips and enforced_params gates * fix(proxy): read the config license before callbacks and gates, keeping a valid env license * fix(proxy): keep an invalid config license reported and sync license data with the license in use A config license that fails no longer overwrites the license string when the env license is not valid either, so /health/license still reports has_license. A failed boot-time env license is not re-checked. Both license writes go through one helper that also refreshes premium_user_data, so /health/license and the seat limits follow the license in use. * fix(proxy): keep the env license's boot result when a config license fails, without a second license check --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/proxy_server.py | 38 +++- litellm/secret_managers/aws_secret_manager.py | 2 +- .../proxy/proxy_server/test_proxy_config.py | 212 ++++++++++++++++++ ...test_proxy_server_endpoints_and_startup.py | 6 +- .../test_aws_secret_manager.py | 14 ++ 5 files changed, 261 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 424e68ae3bc..6fcf789dc1b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1185,6 +1185,27 @@ server_root_path: Final = get_server_root_path() _license_check = LicenseCheck() premium_user: bool = _license_check.is_premium() premium_user_data: Optional["EnterpriseLicenseData"] = _license_check.airgapped_license_data + + +def _set_premium_user(is_premium: bool) -> None: + global premium_user, premium_user_data + premium_user = is_premium # rebind-ok: config load can switch the license in use + premium_user_data = _license_check.airgapped_license_data # rebind-ok: tracks the license in use + + +def _use_config_license(config_license: str, env_license_is_premium: bool) -> bool: + env_license: Final = _license_check.license_str + env_license_data: Final = _license_check.airgapped_license_data + _license_check.license_str = config_license + if _license_check.is_premium(): + return True + if not env_license_is_premium: + return False + _license_check.license_str = env_license + _license_check.airgapped_license_data = env_license_data + return True + + global_max_parallel_request_retries_env: Final[str | None] = os.getenv("LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES") proxy_state: Final = ProxyState() SENSITIVE_DATA_MASKER: Final = SensitiveDataMasker() @@ -6686,7 +6707,6 @@ class ProxyConfig: def _load_environment_variables(self, config: dict): ## ENVIRONMENT VARIABLES - global premium_user environment_variables: Final = config.get("environment_variables", None) if environment_variables: for key, value in environment_variables.items(): @@ -6726,7 +6746,7 @@ class ProxyConfig: # check if litellm_license in general_settings if "LITELLM_LICENSE" in environment_variables: _license_check.license_str = os.getenv("LITELLM_LICENSE", None) - premium_user = _license_check.is_premium() + _set_premium_user(_license_check.is_premium()) def _set_config_environment_variable(self, key: str, value: str) -> None: if key in os.environ and os.environ[key] != value: @@ -6778,7 +6798,6 @@ class ProxyConfig: prompt_injection_detection_obj, \ redis_usage_cache, \ store_model_in_db, \ - premium_user, \ open_telemetry_logger, \ health_check_details, \ proxy_batch_polling_interval, \ @@ -6791,6 +6810,12 @@ class ProxyConfig: self._load_environment_variables(config=config) + config_license: Final = _GENERAL_SETTINGS_VIEW.validate_python( + _GENERAL_SETTINGS_VIEW.validate_python(config).get("general_settings") or {} + ).get("litellm_license") + if isinstance(config_license, str): + _set_premium_user(_use_config_license(config_license, env_license_is_premium=premium_user)) + ## Coordination Redis (before cache init, so the explicit block wins) coordination_redis_cache: Final = self._init_coordination_redis(config=config) if coordination_redis_cache is not None: @@ -7367,7 +7392,7 @@ class ProxyConfig: allowed_ips: Final = general_settings.get("allowed_ips", None) if allowed_ips is not None and premium_user is False: raise ValueError( - "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment." + "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." ) ## BUDGET RESCHEDULER ## proxy_budget_rescheduler_min_time = general_settings.get( @@ -7423,11 +7448,6 @@ class ProxyConfig: if general_settings.get("enforced_params") is not None and premium_user is not True: raise ValueError("Trying to use `enforced_params`" + CommonProxyErrors.not_premium_user.value) - # check if litellm_license in general_settings - if "litellm_license" in general_settings: - _license_check.license_str = general_settings["litellm_license"] - premium_user = _license_check.is_premium() - router_params: Final[dict] = { "cache_responses": litellm.cache is not None, # cache if user passed in cache values } diff --git a/litellm/secret_managers/aws_secret_manager.py b/litellm/secret_managers/aws_secret_manager.py index 88642ae141e..c87edc4b1dd 100644 --- a/litellm/secret_managers/aws_secret_manager.py +++ b/litellm/secret_managers/aws_secret_manager.py @@ -70,7 +70,7 @@ class AWSKeyManagementService_V2: is_litellm_license_in_env = True if is_litellm_license_in_env is False: raise ValueError( - "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment." + "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." ) def load_aws_kms(self, use_aws_kms: bool | None): diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 55226fc629b..712b4134c05 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -9,6 +9,7 @@ Pins covered: from __future__ import annotations import asyncio +import base64 import json import logging import os @@ -25,11 +26,15 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import padding, rsa +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey, RSAPublicKey from prisma.errors import DataError from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import turn_on_json +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.config_resolvers.settings_store import SettingsStore @@ -2671,6 +2676,213 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else []) +@pytest.mark.asyncio +async def test_load_config_refuses_allowed_ips_without_a_license(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + config_file: Final = tmp_path / "allowed_ips.yaml" + config_file.write_text('model_list: []\nlitellm_settings: {}\ngeneral_settings:\n allowed_ips: ["127.0.0.1"]\n') + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + expected_message: Final = ( + "allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." + ) + with pytest.raises(ValueError, match=f"^{re.escape(expected_message)}$"): + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + +def _signed_test_license(private_key: RSAPrivateKey, max_users: int) -> str: + message: Final = json.dumps({"expiration_date": "9999-12-31", "user_id": "test", "max_users": max_users}).encode() + signature: Final = private_key.sign( + message, + padding.PSS(mgf=padding.MGF1(hashes.SHA256()), salt_length=padding.PSS.MAX_LENGTH), + hashes.SHA256(), + ) + return base64.b64encode(message + b"." + signature).decode() + + +def _write_config(path: Path, general_settings: dict[str, object], litellm_settings: dict[str, object]) -> str: + path.write_text( + json.dumps({"model_list": [], "litellm_settings": litellm_settings, "general_settings": general_settings}) + ) + return str(path) + + +def _boot_proxy( + monkeypatch: pytest.MonkeyPatch, + public_key: RSAPublicKey, + env_license: str | None, + license_server_urls: list[str], + server_accepted_license: str | None = None, +) -> None: + from litellm.proxy import proxy_server + + def _license_server(request: httpx.Request) -> httpx.Response: + license_server_urls.append(str(request.url)) + return httpx.Response( + 200, json={"verify": server_accepted_license is not None and server_accepted_license in str(request.url)} + ) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.public_key", public_key) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.airgapped_license_data", None) + monkeypatch.setattr( + "litellm.proxy.proxy_server._license_check.http_handler", + HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_license_server))), + ) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + if env_license is None: + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + else: + monkeypatch.setenv("LITELLM_LICENSE", env_license) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.license_str", env_license) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", proxy_server._license_check.is_premium()) + monkeypatch.setattr( + "litellm.proxy.proxy_server.premium_user_data", proxy_server._license_check.airgapped_license_data + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "gated_config", + [ + {"general_settings": {"allowed_ips": ["127.0.0.1"]}}, + {"general_settings": {"enforced_params": ["user"]}}, + {"litellm_settings": {"callbacks": ["openai_moderations"]}}, + ], +) +async def test_load_config_lets_a_config_license_unlock_its_enterprise_gates( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, gated_config: dict[str, dict[str, list[str]]] +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + license_key: Final = _signed_test_license(private_key, max_users=10) + config_file: Final = _write_config( + tmp_path / "licensed.yaml", + {"litellm_license": license_key, **gated_config.get("general_settings", {})}, + dict(gated_config.get("litellm_settings", {})), + ) + _boot_proxy(monkeypatch, private_key.public_key(), env_license=None, license_server_urls=[]) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + + +@pytest.mark.asyncio +async def test_load_config_keeps_a_valid_env_license_when_the_config_license_is_invalid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license", "allowed_ips": ["127.0.0.1"]}, {} + ) + _boot_proxy( + monkeypatch, + private_key.public_key(), + env_license=_signed_test_license(private_key, max_users=5), + license_server_urls=[], + ) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + assert proxy_server._license_check.user_seat_limit() == 5 + + +@pytest.mark.asyncio +async def test_load_config_prefers_the_config_license_when_both_licenses_are_valid( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_license_endpoint + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "licensed.yaml", {"litellm_license": _signed_test_license(private_key, max_users=100)}, {} + ) + _boot_proxy( + monkeypatch, + private_key.public_key(), + env_license=_signed_test_license(private_key, max_users=5), + license_server_urls=[], + ) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server._license_check.user_seat_limit() == 100 + assert (await health_license_endpoint(user_api_key_dict=UserAPIKeyAuth()))["limits"]["max_users"] == 100 + + +@pytest.mark.asyncio +async def test_load_config_still_reports_an_invalid_config_license_as_configured( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_license_endpoint + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config(tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license"}, {}) + _boot_proxy(monkeypatch, private_key.public_key(), env_license=None, license_server_urls=[]) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + license_status: Final = await health_license_endpoint(user_api_key_dict=UserAPIKeyAuth()) + assert (license_status["has_license"], license_status["license_type"]) == (True, "community") + + +@pytest.mark.asyncio +async def test_load_config_does_not_recheck_an_env_license_that_failed_at_boot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config(tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license"}, {}) + license_server_urls: Final[list[str]] = [] + _boot_proxy(monkeypatch, private_key.public_key(), "also-not-a-valid-license", license_server_urls) + asked_at_boot: Final = len(license_server_urls) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is False + assert [url for url in license_server_urls[asked_at_boot:] if "also-not-a-valid-license" in url] == [] + assert any("not-a-valid-license" in url for url in license_server_urls[asked_at_boot:]) + + +@pytest.mark.asyncio +async def test_load_config_keeps_the_boot_verdict_of_a_server_checked_env_license( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy import proxy_server + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + config_file: Final = _write_config( + tmp_path / "stale_license.yaml", {"litellm_license": "not-a-valid-license", "allowed_ips": ["127.0.0.1"]}, {} + ) + license_server_urls: Final[list[str]] = [] + _boot_proxy( + monkeypatch, + private_key.public_key(), + "server-checked-license", + license_server_urls, + server_accepted_license="server-checked-license", + ) + asked_at_boot: Final = len(license_server_urls) + + await ProxyConfig().load_config(router=None, config_file_path=config_file) + + assert proxy_server.premium_user is True + assert proxy_server._license_check.license_str == "server-checked-license" + assert [url for url in license_server_urls[asked_at_boot:] if "server-checked-license" in url] == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( "setting", ["vector_store_deny_by_default", "search_tool_deny_by_default", "claude_code_marketplace_auth_required"] diff --git a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py index 82db8820447..9914247bc47 100644 --- a/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py +++ b/tests/unit/proxy/test_proxy_server_endpoints_and_startup.py @@ -4117,7 +4117,11 @@ async def test_load_environment_variables_litellm_license_and_edge_cases(): mock_license_check = MagicMock() mock_license_check.is_premium.return_value = True - with patch("litellm.proxy.proxy_server._license_check", mock_license_check): + with ( + patch("litellm.proxy.proxy_server._license_check", mock_license_check), + patch("litellm.proxy.proxy_server.premium_user", False), + patch("litellm.proxy.proxy_server.premium_user_data", None), + ): with patch.dict(os.environ, {}, clear=False): # Call the method under test proxy_config._load_environment_variables(test_config_with_license) diff --git a/tests/unit/secret_managers/test_aws_secret_manager.py b/tests/unit/secret_managers/test_aws_secret_manager.py index d9922b43ec8..e16da7a0844 100644 --- a/tests/unit/secret_managers/test_aws_secret_manager.py +++ b/tests/unit/secret_managers/test_aws_secret_manager.py @@ -1,4 +1,6 @@ import base64 +import re +from typing import Final import pytest from botocore.stub import Stubber @@ -34,3 +36,15 @@ def test_decrypt_value_turns_only_boolean_literals_into_bools(monkeypatch, plain assert decrypted == expected assert type(decrypted) is type(expected) + + +def test_kms_v2_without_a_license_tells_the_user_to_set_one(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("AWS_REGION_NAME", "us-west-2") + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + monkeypatch.delenv("LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE", raising=False) + + expected_message: Final = ( + "AWSKeyManagementService V2 is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your environment." + ) + with pytest.raises(ValueError, match=f"^{re.escape(expected_message)}$"): + AWSKeyManagementService_V2() From ff182c194e0b5f782e4a7d3405361fed11466416 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Tue, 3 Nov 2026 02:02:54 -0800 Subject: [PATCH 23/34] fix(logging): keep the caller's call_type when chat completions are bridged to the Responses API (#49710) * fix(logging): keep the caller's call_type when chat completions are bridged to the Responses API * fix(logging): log bridged chat completions with the chat response they returned * fix(logging): log bridged text completions and /v1/messages as served, with the request's options Bridged rows were translated without the request's options, so structured output and replay fields could differ from what the client got. Text completions and non-streamed /v1/messages through the chat bridge also kept a Responses API object or the encoded id. * fix(logging): redact reasoning summaries on bridged rows and log bridged text completions as text completions Redaction now drops reasoning_items on chat messages and deltas, so a bridged row keeps no reasoning summary when message logging is off. A non-streamed bridged text completion logs the text completion the client got, priced like the chat response it came from. The bridge handler's input TypedDict fields are ReadOnly. * fix(logging): redact bridged text completions for opted-out callbacks and pin the sync logged shape and usage ack * test(integration): look up bridged chat spend rows by the id the client got Bridged chat spend rows are keyed by the client-facing id and logged with the caller's call type, so the helpers that decoded the old resp_ key or pinned call_type responses now assert the new contract --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../handler.py | 22 +- .../transformation.py | 4 - litellm/litellm_core_utils/litellm_logging.py | 75 +++- litellm/litellm_core_utils/redact_messages.py | 52 ++- .../_mantle_gpt_prompt_cache_support.py | 11 +- .../test_bedrock_mantle_claude_chat_wire.py | 2 +- .../providers/test_openai_chat_wire.py | 17 +- .../spend/test_responses_bridge_spend_once.py | 4 +- ...responses_transformation_transformation.py | 425 +++++++++++++++++- .../test_redact_messages.py | 51 +++ 10 files changed, 587 insertions(+), 76 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 76d17ea9693..f6879c313de 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -11,7 +11,7 @@ from typing import ( cast, # noqa: TID251 # optional_params reaches the bridge as an untyped dict ) -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.llms.openai import ResponsesAPIResponse @@ -20,15 +20,15 @@ if TYPE_CHECKING: class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict): - model: str - messages: list - optional_params: dict - litellm_params: dict - headers: dict - model_response: "ModelResponse" - logging_obj: "LiteLLMLoggingObj" - custom_llm_provider: str - encoding: object + model: ReadOnly[str] + messages: ReadOnly[list] + optional_params: ReadOnly[dict[str, object]] + litellm_params: ReadOnly[dict[str, object]] + headers: ReadOnly[dict] + model_response: ReadOnly["ModelResponse"] + logging_obj: ReadOnly["LiteLLMLoggingObj"] + custom_llm_provider: ReadOnly[str] + encoding: ReadOnly[object] def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: @@ -184,6 +184,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} + logging_obj.record_bridged_chat_request(optional_params, litellm_params) request_data: Final = self.transformation_handler.transform_request( model=model, @@ -285,6 +286,7 @@ class ResponsesToCompletionBridgeHandler: custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} + logging_obj.record_bridged_chat_request(optional_params, litellm_params) try: request_data: Final = self.transformation_handler.transform_request( diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c03419436d6..8dd4d87ae13 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1235,10 +1235,6 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): api_model: Final = model - from litellm.types.utils import CallTypes - - setattr(litellm_logging_obj, "call_type", CallTypes.responses.value) - sanitized_litellm_params: Final = self._build_sanitized_litellm_params(litellm_params) request_data: Final = { diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 22c5f051d5c..97c019a2180 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -19,7 +19,7 @@ from datetime import datetime as dt_object from enum import Enum from functools import lru_cache from types import MappingProxyType, TracebackType -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast from httpx import Response, Timeout from openai import AsyncStream as OpenAIAsyncStream @@ -99,6 +99,7 @@ from litellm.litellm_core_utils.hidden_params import ( get_hidden_params, served_from_cache, set_hidden_param, + set_hidden_params, ) from litellm.litellm_core_utils.internal_call_metadata import ( MODEL_ACCESS_GROUP_METADATA_KEY, @@ -124,6 +125,7 @@ from litellm.litellm_core_utils.llm_cost_calc.zero_cost_diagnostic import ( diagnose_zero_cost, zero_cost_warning, ) +from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import LiteLLMResponseObjectHandler from litellm.litellm_core_utils.logging_utils import ( truncate_base64_in_messages, truncate_base64_in_messages_async, @@ -144,7 +146,7 @@ from litellm.litellm_core_utils.served_output_texts import ( from litellm.litellm_core_utils.thread_pool_executor import executor from litellm.llms.base_llm.ocr.transformation import OCRResponse from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse @@ -302,6 +304,11 @@ class _EnterpriseAlertingLoggers(NamedTuple): smtp_email: type[CustomLogger] +class BridgedChatRequest(NamedTuple): + optional_params: Mapping[str, object] + litellm_params: Mapping[str, object] + + @functools.cache def _enterprise_alerting_loggers() -> _EnterpriseAlertingLoggers: try: @@ -582,6 +589,7 @@ _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__) _REDACTED_RESULT: Final = TypeAdapter(object) +_LoggedResponse: Final = TypeVar("_LoggedResponse") _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -875,6 +883,7 @@ class Logging(LiteLLMLoggingBaseClass): # once that response is priced, the same way a non-streamed response is logged self.client_facing_stream_model: str | None = None self.client_facing_response_id: str | None = None + self.bridged_chat_request: BridgedChatRequest | None = None self.zero_cost_warned: bool = False self.llm_caching_handler: LLMCachingHandler | None = None @@ -3235,6 +3244,40 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"model": model}) + def _as_served_chat_response(self, response: _LoggedResponse) -> _LoggedResponse | ModelResponse: + request: Final = self.bridged_chat_request + if request is None or not isinstance(response, ResponsesAPIResponse): + return response + return self._bridged_chat_response(response, request) + + def _as_served_response( + self, response: _LoggedResponse + ) -> _LoggedResponse | ModelResponse | TextCompletionResponse: + request: Final = self.bridged_chat_request + if request is None or not isinstance(response, ResponsesAPIResponse): + return response + served: Final = self._bridged_chat_response(response, request) + if self.call_type not in (CallTypes.text_completion.value, CallTypes.atext_completion.value): # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped + return served + return self._as_text_completion_priced_as_chat(served) + + def _as_text_completion_priced_as_chat(self, served: ModelResponse) -> TextCompletionResponse: + text_completion: Final = LiteLLMResponseObjectHandler.convert_chat_to_text_completion( + served, TextCompletionResponse() + ) + set_hidden_params(text_completion, dict(get_hidden_params(served) or {})) + return text_completion + + def _bridged_chat_response(self, response: ResponsesAPIResponse, request: BridgedChatRequest) -> ModelResponse: + translated: Final = self._translate_responses_api_response_to_model_response(response, request) + served: Final = translated.model_copy( + update={ + "id": ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(response.id) + } + ) + set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})}) + return served + def _with_continued_response_id( self, response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, @@ -3316,6 +3359,8 @@ class Logging(LiteLLMLoggingBaseClass): ) result = self.normalize_endpoint_logging_result(result) + if self.stream is not True: + result = self._as_served_response(result) # rebind-ok: bridged chat calls log the served response logging_result: Final = self.normalize_logging_result(result=result) if cache_hit is None and _served_from_cache_at_no_charge(logging_result): @@ -3670,7 +3715,9 @@ class Logging(LiteLLMLoggingBaseClass): ) self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) + logged_streaming_response: Final = self._client_facing_streaming_response( + self._as_served_chat_response(complete_streaming_response) + ) self.model_call_details["complete_streaming_response"] = logged_streaming_response self.capture_usage_before_logging(logged_streaming_response, start_time, end_time) ## STANDARDIZED LOGGING PAYLOAD @@ -4268,7 +4315,9 @@ class Logging(LiteLLMLoggingBaseClass): await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._client_facing_streaming_response(complete_streaming_response) + logged_streaming_response: Final = self._client_facing_streaming_response( + self._as_served_chat_response(complete_streaming_response) + ) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD @@ -5227,6 +5276,13 @@ class Logging(LiteLLMLoggingBaseClass): row is built from this response rather than re-parsed from the served body.""" self.bridged_model_response = response + def record_bridged_chat_request( + self, optional_params: Mapping[str, object], litellm_params: Mapping[str, object] + ) -> None: + self.bridged_chat_request = BridgedChatRequest( + MappingProxyType(dict(optional_params)), MappingProxyType(dict(litellm_params)) + ) + def record_bridged_responses_api_response(self, response: ResponsesAPIResponse) -> None: self.record_bridged_model_response(self._translate_responses_api_response_to_model_response(response)) @@ -5298,6 +5354,9 @@ class Logging(LiteLLMLoggingBaseClass): ): result = result.response if isinstance(result, ResponsesAPIResponse): + bridged_request: Final = self.bridged_chat_request + if bridged_request is not None: + return self._bridged_chat_response(result, bridged_request) return self._translate_responses_api_response_to_model_response(result) raw_body: Final[object] = result @@ -5349,7 +5408,9 @@ class Logging(LiteLLMLoggingBaseClass): set_response_cost_in_hidden_params(result, reported_cost) return result - def _translate_responses_api_response_to_model_response(self, result: ResponsesAPIResponse) -> ModelResponse: + def _translate_responses_api_response_to_model_response( + self, result: ResponsesAPIResponse, request: BridgedChatRequest | None = None + ) -> ModelResponse: """ Convert a Responses API response into a ModelResponse for spend_logs. @@ -5372,8 +5433,8 @@ class Logging(LiteLLMLoggingBaseClass): logging_obj=self, request_data={}, messages=[], - optional_params={}, - litellm_params={}, + optional_params={} if request is None else dict(request.optional_params), + litellm_params={} if request is None else dict(request.litellm_params), encoding=litellm.encoding, ) translated.id = served_id or translated.id diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 078ec18e2db..9da0e35bdda 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -10,7 +10,7 @@ import asyncio import copy import inspect -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from typing import TYPE_CHECKING, Any, Final @@ -107,7 +107,13 @@ def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger: def _is_redactable_response(response_obj: object) -> bool: return isinstance( - response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse) + response_obj, + ( + litellm.ModelResponse, + litellm.TextCompletionResponse, + litellm.ResponsesAPIResponse, + litellm.EmbeddingResponse, + ), ) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj)) @@ -208,6 +214,7 @@ def _redact_choice_content(choice): choice.message.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.message, "thinking_blocks"): choice.message.thinking_blocks = None + _drop_field_when_present(choice.message, "reasoning_items") _redact_provider_specific_fields(choice.message) _redact_tool_calls(getattr(choice.message, "tool_calls", None)) _redact_function_call(getattr(choice.message, "function_call", None)) @@ -218,12 +225,14 @@ def _redact_choice_content(choice): choice.delta.reasoning_content = REDACTED_BY_LITELLM if hasattr(choice.delta, "thinking_blocks"): choice.delta.thinking_blocks = None + _drop_field_when_present(choice.delta, "reasoning_items") _redact_provider_specific_fields(choice.delta) _redact_tool_calls(getattr(choice.delta, "tool_calls", None)) _redact_function_call(getattr(choice.delta, "function_call", None)) elif isinstance(choice, litellm.utils.TextChoices): _redact_field_when_present(choice, "text") _redact_field_when_present(choice, "reasoning_content") + _drop_field_when_present(choice, "logprobs") def _as_mapping(value: object) -> Mapping[str, object] | None: @@ -284,6 +293,11 @@ def _redact_field_when_present(item: object, field: str) -> None: setattr(item, field, REDACTED_BY_LITELLM) +def _drop_field_when_present(item: object, field: str) -> None: + if getattr(item, field, None) is not None: + setattr(item, field, None) + + def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: @@ -440,32 +454,22 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None: function_call["arguments"] = REDACTED_BY_LITELLM -def _redact_model_response_dict_choices(choices, redacted_str: str): +def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: str) -> None: for choice in choices: if isinstance(choice, dict): _redact_provider_specific_fields_dict(choice) if choice.get("token_ids") is not None: choice["token_ids"] = None - if "message" in choice and isinstance(choice["message"], dict): - if choice["message"].get("content") is not None: - choice["message"]["content"] = redacted_str - if choice["message"].get("reasoning_content") is not None: - choice["message"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - _redact_tool_calls_dict(choice["message"]) - elif "delta" in choice and isinstance(choice["delta"], dict): - if choice["delta"].get("content") is not None: - choice["delta"]["content"] = redacted_str - if choice["delta"].get("reasoning_content") is not None: - choice["delta"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - _redact_tool_calls_dict(choice["delta"]) + if ( + body := next((choice[key] for key in ("message", "delta") if isinstance(choice.get(key), dict)), None) + ) is not None: + for field in ("content", "reasoning_content"): + if body.get(field) is not None: + body[field] = redacted_str + for field in ("thinking_blocks", "reasoning_items", "audio"): + if field in body: + body[field] = None + _redact_tool_calls_dict(body) elif choice.get("text") is not None: choice["text"] = redacted_str else: @@ -588,7 +592,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons return {"text": REDACTED_BY_LITELLM} _result: Final = copy.deepcopy(result) - if isinstance(_result, litellm.ModelResponse): + if isinstance(_result, (litellm.ModelResponse, litellm.TextCompletionResponse)): if hasattr(_result, "choices") and _result.choices is not None: for choice in _result.choices: _redact_choice_content(choice) diff --git a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py index 8f497fea74f..96261bab8b2 100644 --- a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py +++ b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py @@ -475,10 +475,6 @@ def row_key(request_id: str) -> str: return upstream_id_of(request_id.split("_cache_hit", 1)[0]) -def caller_sees_upstream_id(endpoint: Endpoint, *, stream: bool) -> bool: - return (endpoint, stream) != ("messages", True) - - def usage_of(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]: usage: Final = payload.get("usage") return JSON.validate_python(usage) if isinstance(usage, dict) else {} @@ -733,10 +729,9 @@ def marked_cell(gateway: Gateway, endpoint: Endpoint, *, stream: bool) -> None: assert_answered(outcome, marker) assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream) assert_usage(endpoint, outcome.usage) - row: Final = success_row(name, marker) + row: Final = success_row(name, marker, outcome.response_id) assert_priced_row(row, GPT) - if caller_sees_upstream_id(endpoint, stream=stream): - assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) + assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) if not stream: assert math.isclose(float(outcome.headers["x-litellm-response-cost"]), expected_spend(GPT), rel_tol=1e-9), ( outcome.headers @@ -901,7 +896,7 @@ def assert_marked_sdk_cell( only_received(wire), expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream ) assert_usage(endpoint, outcome.usage) - assert_priced_row(success_row(name, marker), GPT) + assert_priced_row(success_row(name, marker, outcome.response_id), GPT) HOSTILE_OPTIONS: Final[dict[str, JsonValue]] = { diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index 32b6c4f1690..3652e49fe8a 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -1211,6 +1211,6 @@ def test_non_claude_mantle_id_with_a_bare_bedrock_twin_keeps_its_route_and_spend cost: Final = _mantle_cost(gateway, _LUNA) assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(cost), response.text _assert_success_row( - _spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "responses", cost + _spend_row(_BY_CALL_ID, response.headers["x-litellm-call-id"]), model, _LUNA, "acompletion", cost ) assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)] diff --git a/tests/integration/providers/test_openai_chat_wire.py b/tests/integration/providers/test_openai_chat_wire.py index 69295485380..5ba631ac776 100644 --- a/tests/integration/providers/test_openai_chat_wire.py +++ b/tests/integration/providers/test_openai_chat_wire.py @@ -1,4 +1,3 @@ -import base64 import json import uuid from collections.abc import Callable @@ -547,14 +546,8 @@ def test_azure_gpt_6_bridged_no_cache_function_requests_each_reach_provider_and_ lambda found: len(found) == 2, seconds=70, ) - by_response_id: Final = { - (decoded := base64.b64decode(string_value(row["request_id"]).removeprefix("resp_")).decode()) - .rsplit("response_id:", 1)[1]: (decoded, row) - for row in rows - } - for response_id in response_ids: - decoded, row = by_response_id[response_id] - assert decoded.startswith("litellm:custom_llm_provider:azure;model_id:"), rows + assert sorted(string_value(row["request_id"]) for row in rows) == sorted(response_ids), rows + for row in rows: assert ( string_value(row["status"]), string_value(row["cache_hit"]), @@ -601,11 +594,7 @@ def test_azure_gpt_6_bridged_function_requests_without_cache_field_still_hit_cac seconds=70, ) priced, cached = rows - priced_request: Final = base64.b64decode( - string_value(priced["request_id"]).removeprefix("resp_") - ).decode() - assert priced_request.startswith("litellm:custom_llm_provider:azure;model_id:"), rows - assert priced_request.endswith(";response_id:resp_cached"), rows + assert string_value(priced["request_id"]) == "resp_cached", rows assert ( priced["status"], priced["cache_hit"], diff --git a/tests/integration/spend/test_responses_bridge_spend_once.py b/tests/integration/spend/test_responses_bridge_spend_once.py index 9eddd832ca0..cc2a0af58a7 100644 --- a/tests/integration/spend/test_responses_bridge_spend_once.py +++ b/tests/integration/spend/test_responses_bridge_spend_once.py @@ -1,5 +1,4 @@ import json -from base64 import b64decode from collections.abc import Callable from hashlib import sha256 from typing import Final @@ -100,8 +99,7 @@ def test_bridged_chat_completion_above_log_offload_threshold_logs_and_charges_on call_rows: Final = [row for row in rows if row["litellm_call_id"] == call_id] assert len(call_rows) == 1, rows assert float(str(call_rows[0]["spend"])) == pytest.approx(EXPECTED_SPEND), rows - logged_id: Final = b64decode(str(call_rows[0]["request_id"]).removeprefix("resp_")).decode() - assert logged_id.startswith("litellm:") and f";response_id:{upstream_id}" in logged_id, rows + assert call_rows[0]["request_id"] == upstream_id, rows key_spend: Final = eventually( lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)), diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 9313e3d7039..2d02468fd8c 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1,9 +1,13 @@ +import asyncio import copy import datetime import json import os +import queue import unittest -from collections.abc import Mapping, Sequence +from collections.abc import AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast, get_args from unittest.mock import ANY, MagicMock, Mock, patch @@ -24,15 +28,15 @@ import litellm from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) -from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.types.llms.openai import AllMessageValues, REASONING_EFFORT, ResponsesAPIResponse +from litellm.types.utils import ModelResponse, StandardLoggingPayload, TextCompletionResponse if TYPE_CHECKING: from openai.types.responses import ResponseOutputItem from openai.types.responses.response_reasoning_item import ResponseReasoningItem - from litellm.types.llms.openai import ResponsesAPIResponse - from litellm.types.utils import ModelResponse - def test_completion_over_responses_maps_anthropic_search_tool_to_openai_tool_search() -> None: response_body: Final = { @@ -191,6 +195,417 @@ def test_auto_summary_sends_the_level_the_target_accepts( assert request_body["reasoning"] == {"effort": "low", "summary": expected_summary} +@dataclass(frozen=True, slots=True) +class _LoggedSuccess: + call_type: str + payload: StandardLoggingPayload + response_obj: object + complete_streaming_response: object + + +class _LoggedCallTypes(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.seen_by_caller: Final = MappingProxyType( + { + "completion": queue.SimpleQueue[_LoggedSuccess](), + "acompletion": queue.SimpleQueue[_LoggedSuccess](), + } + ) + + @staticmethod + def _logged(kwargs: Mapping[str, object], response_obj: object, streaming_key: str) -> _LoggedSuccess: + return _LoggedSuccess( + call_type=str(kwargs["call_type"]), + payload=cast(StandardLoggingPayload, kwargs["standard_logging_object"]), + response_obj=response_obj, + complete_streaming_response=kwargs.get(streaming_key), + ) + + def log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.seen_by_caller["completion"].put(self._logged(kwargs, response_obj, "complete_streaming_response")) + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.seen_by_caller["acompletion"].put( + self._logged(kwargs, response_obj, "async_complete_streaming_response") + ) + + +_GET_WEATHER_TOOL: Final = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object", "properties": {}}}, +} + +_BRIDGED_PROVIDERS: Final = pytest.mark.parametrize( + ("model", "url", "provider_kwargs"), + [ + ("openai/gpt-5.3-codex", "https://api.openai.com/v1/responses", {}), + ( + "azure/gpt-6-sol", + "https://example.invalid/openai/responses", + {"api_base": "https://example.invalid", "api_version": "2025-04-01-preview"}, + ), + ("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, {"aws_region_name": "us-east-1"}), + ], + ids=["openai-responses-only", "azure-tools", "bedrock-mantle-tools"], +) + + +_PONG_MESSAGE: Final[Mapping[str, object]] = { + "type": "message", + "id": "msg_logged_call_type", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], +} + + +def _searched_responses_body(model: str) -> Mapping[str, object]: + return { + "id": "resp_logged_call_type", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": model.rsplit("/", 1)[-1], + "output": [ + { + "type": "web_search_call", + "id": "ws_logged_call_type", + "status": "completed", + "action": {"type": "search", "query": "weather"}, + }, + _PONG_MESSAGE, + ], + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def _responses_sse(response_body: Mapping[str, object]) -> str: + events: Final = ( + {"type": "response.created", "response": {**response_body, "status": "in_progress", "output": [], "usage": None}}, + { + "type": "response.output_text.delta", + "item_id": "msg_logged_call_type", + "output_index": 1, + "content_index": 0, + "delta": "pong", + }, + {"type": "response.completed", "response": response_body}, + ) + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + + +def _bridged_request(model: str, provider_kwargs: Mapping[str, object], stream: bool) -> Mapping[str, object]: + return { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "tools": [_GET_WEATHER_TOOL], + "api_key": "test-key", + "cache": {"no-cache": True}, + "stream": stream, + **provider_kwargs, + } + + +async def _served_chunk_ids(caller: str, request: Mapping[str, object]) -> frozenset[str]: + if caller == "acompletion": + async_stream: Final = await litellm.acompletion(**request) + assert isinstance(async_stream, CustomStreamWrapper) + return frozenset([chunk.id async for chunk in async_stream]) + sync_stream: Final = litellm.completion(**request) + assert isinstance(sync_stream, CustomStreamWrapper) + return frozenset(chunk.id for chunk in sync_stream) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@_BRIDGED_PROVIDERS +@pytest.mark.asyncio +async def test_bridged_chat_call_is_logged_with_the_callers_call_type( + monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body(model) + request: Final = _bridged_request(model, provider_kwargs, stream=False) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith=url).mock(return_value=httpx.Response(200, json=response_body)) + if caller == "acompletion": + await litellm.acompletion(**request) + else: + litellm.completion(**request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert (logged.call_type, logged.payload["call_type"]) == (caller, caller) + assert isinstance(logged.response_obj, ModelResponse) + assert logged.response_obj.id == response_body["id"] + assert logged.response_obj.choices[0].message.content == "pong" + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model=model + ) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@_BRIDGED_PROVIDERS +@pytest.mark.asyncio +async def test_bridged_chat_stream_is_logged_as_the_chat_completion_it_streamed( + monkeypatch: pytest.MonkeyPatch, caller: str, model: str, url: str, provider_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body(model) + request: Final = _bridged_request(model, provider_kwargs, stream=True) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith=url).mock( + return_value=httpx.Response( + 200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"} + ) + ) + served_ids: Final = await _served_chunk_ids(caller, request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(logged.complete_streaming_response, ModelResponse) + assert logged.response_obj is logged.complete_streaming_response + assert served_ids == {logged.complete_streaming_response.id} + assert logged.complete_streaming_response.choices[0].message.content == "pong" + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model=model + ) + + +def _json_message(item_id: str, document: Mapping[str, object]) -> Mapping[str, object]: + return { + "type": "message", + "id": item_id, + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": json.dumps(document), "annotations": []}], + } + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_structured_output_call_is_logged_with_the_json_document_the_client_got( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = { + **_searched_responses_body("openai/gpt-5.3-codex"), + "output": [_json_message("msg_first", {"answer": "pong"}), _json_message("msg_second", {"answer": "again"})], + } + request: Final = { + **_bridged_request("openai/gpt-5.3-codex", {}, stream=False), + "response_format": {"type": "json_object"}, + } + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + served: Final = await litellm.acompletion(**request) if caller == "acompletion" else litellm.completion(**request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(served, ModelResponse) + assert isinstance(logged.response_obj, ModelResponse) + served_message: Final = served.choices[0].message + logged_message: Final = logged.response_obj.choices[0].message + assert json.loads(served_message.content) == {"answer": "pong"} + assert logged_message.content == served_message.content + assert logged_message.provider_specific_fields == served_message.provider_specific_fields + assert logged.payload["response"]["choices"][0]["message"]["content"] == served_message.content + + +async def _logged_text_completion( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> tuple[TextCompletionResponse, _LoggedSuccess, Mapping[str, object]]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = _searched_responses_body("openai/gpt-5.3-codex") + request: Final = {"model": "openai/gpt-5.3-codex", "prompt": "ping", "api_key": "test-key"} + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=response_body) + ) + served: Final = ( + await litellm.atext_completion(**request) if caller == "acompletion" else litellm.text_completion(**request) + ) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + assert isinstance(served, TextCompletionResponse) + return served, logged, response_body + + +@pytest.mark.parametrize(("caller", "call_type"), [("completion", "text_completion"), ("acompletion", "atext_completion")]) +@pytest.mark.asyncio +async def test_bridged_text_completion_is_logged_and_priced_under_the_callers_call_type( + monkeypatch: pytest.MonkeyPatch, caller: str, call_type: str +) -> None: + served, logged, response_body = await _logged_text_completion(monkeypatch, caller) + + assert (logged.call_type, logged.payload["call_type"]) == (call_type, call_type) + assert getattr(logged.response_obj, "id") == served.id == response_body["id"] + assert logged.payload["response_cost"] == litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate(response_body), model="openai/gpt-5.3-codex" + ) + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_text_completion_is_logged_as_the_text_completion_it_served( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + served, logged, _ = await _logged_text_completion(monkeypatch, caller) + + assert isinstance(logged.response_obj, TextCompletionResponse) + assert logged.response_obj.choices[0].text == served.choices[0].text == "pong" + assert logged.payload["response"]["object"] == served.object == "text_completion" + assert logged.payload["response"]["choices"][0]["text"] == "pong" + + +@pytest.mark.asyncio +async def test_a_bridged_chat_response_waits_on_the_usage_ack_like_any_chat_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime, capture_proxy_response + from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings + from tests.unit.proxy.spend_tracking.test_usage_meter import CommitBarrier + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + database: Final = CommitBarrier() + runtime: Final = UsageCaptureRuntime() + runtime.start(database.write, UsageCaptureSettings(batch_size=2)) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + monkeypatch.setattr(proxy_server, "general_settings", {"accounting_sync_ack": True}) + logging_obj: Final = Logging( + model="openai/gpt-5.3-codex", + messages=[], + stream=False, + call_type="acompletion", + start_time=datetime.datetime.now(), + litellm_call_id="bridged-chat", + function_id="bridged-chat", + ) + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response(200, json=_searched_responses_body("openai/gpt-5.3-codex")) + ) + served: Final = await litellm.acompletion( + model="openai/gpt-5.3-codex", + messages=[{"role": "user", "content": "ping"}], + api_key="test-key", + litellm_logging_obj=logging_obj, + ) + response: Final = asyncio.create_task(capture_proxy_response(logging_obj, served)) + try: + await database.entered.wait() + assert not response.done() + assert database.committed == () + database.release.set() + await response + assert "bridged-chat:inbound" in database.committed + finally: + database.release.set() + await runtime.stop(1) + await response + + +def _served_message_start_id(events: Sequence[bytes]) -> object: + lines: Final = b"".join(events).decode().splitlines() + starts: Final = tuple( + json.loads(line.removeprefix("data:")) for line in lines if line.startswith("data:") and '"message_start"' in line + ) + assert len(starts) == 1 + return starts[0]["message"]["id"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["non-stream", "stream"]) +@pytest.mark.asyncio +async def test_bridged_messages_call_is_logged_as_anthropic_messages_under_the_id_the_client_got( + monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": [_PONG_MESSAGE]} + upstream: Final = ( + httpx.Response(200, text=_responses_sse(response_body), headers={"content-type": "text/event-stream"}) + if stream + else httpx.Response(200, json=response_body) + ) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock(return_value=upstream) + served: Final = await litellm.anthropic.messages.acreate( + model="openai/gpt-5.3-codex", + max_tokens=64, + messages=[{"role": "user", "content": "ping"}], + api_key="test-key", + stream=stream, + ) + served_id: Final = ( + _served_message_start_id([event async for event in cast(AsyncIterator[bytes], served)]) + if stream + else cast(Mapping[str, object], served)["id"] + ) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller["acompletion"].get, timeout=10) + + assert (logged.call_type, logged.payload["call_type"]) == ("anthropic_messages", "anthropic_messages") + assert logged.payload["id"] == served_id + assert logged.payload["response"]["choices"][0]["message"]["content"] == "pong" + + +@pytest.mark.parametrize("caller", ["completion", "acompletion"]) +@pytest.mark.asyncio +async def test_bridged_chat_stream_keeps_the_upstream_headers_when_its_output_cannot_be_translated( + monkeypatch: pytest.MonkeyPatch, caller: str +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _LoggedCallTypes() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + response_body: Final = {**_searched_responses_body("openai/gpt-5.3-codex"), "output": []} + request: Final = _bridged_request("openai/gpt-5.3-codex", {}, stream=True) + + with respx.mock(assert_all_called=True) as router: + router.post(url__startswith="https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + text=_responses_sse(response_body), + headers={"content-type": "text/event-stream", "x-request-id": "req_upstream"}, + ) + ) + await _served_chunk_ids(caller, request) + logged: Final = await asyncio.to_thread(recorder.seen_by_caller[caller].get, timeout=10) + + assert isinstance(logged.complete_streaming_response, ModelResponse) + assert logged.complete_streaming_response.id == response_body["id"] + assert logged.payload["hidden_params"]["additional_headers"]["llm_provider-x-request-id"] == "req_upstream" + + def test_convert_chat_completion_messages_to_responses_api_image_input(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 0aee61d8365..2fd3f16ab17 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -112,6 +112,57 @@ def test_redaction_reuses_message_rules_for_request_body_and_provider_fields(sha assert result == original_result and params == original_params +def _reasoning_items() -> list[dict[str, object]]: + return [{"type": "reasoning", "id": "rs_redaction", "summary": [{"type": "summary_text", "text": SECRET}]}] + + +def _message_response() -> ModelResponse: + return ModelResponse( + choices=[{"index": 0, "message": {"role": "assistant", "content": "answer", "reasoning_items": _reasoning_items()}}] + ) + + +def _delta_choices() -> dict[str, object]: + return {"choices": [{"index": 0, "delta": {"content": "answer", "reasoning_items": _reasoning_items()}}]} + + +@pytest.mark.parametrize( + ("details", "result"), + [ + pytest.param({}, _message_response(), id="message_object"), + pytest.param({}, _message_response().model_dump(), id="message_dict"), + pytest.param({}, _delta_choices(), id="delta_dict"), + pytest.param( + {"stream": True, "complete_streaming_response": ModelResponseStream(**_delta_choices())}, + None, + id="stream_chunk", + ), + pytest.param({"standard_logging_object": {"response": _message_response().model_dump()}}, None, id="logged"), + pytest.param({"standard_logging_object": {"response": _delta_choices()}}, None, id="logged_delta"), + ], +) +def test_redaction_hides_reasoning_item_summaries(details: dict[str, object], result: object) -> None: + assert SECRET in str(details) + str(result) + + redacted: Final = perform_redaction(details, result) + + assert SECRET not in str(redacted) and SECRET not in str(details), (redacted, details) + + +def test_an_opted_out_callback_gets_a_text_completion_with_its_answer_redacted() -> None: + logprobs: Final = {"tokens": [SECRET], "token_logprobs": [-0.1], "text_offset": [0], "top_logprobs": [{SECRET: -0.1}]} + response: Final = TextCompletionResponse( + choices=[{"index": 0, "text": SECRET, "logprobs": logprobs, "finish_reason": "stop"}] + ) + + redacted: Final = redacted_response_obj_for_custom_logger(response, CustomLogger(message_logging=False)) + + assert isinstance(redacted, TextCompletionResponse) + assert redacted.choices[0].text == "redacted-by-litellm" + assert SECRET not in redacted.model_dump_json() + assert response.choices[0].text == SECRET, "the caller's own response is not redacted" + + def test_redaction_redacts_reasoning_copied_into_hidden_params_in_place() -> None: hidden_params: Final = {"model_id": "deployment", "provider_specific_fields": _provider_fields()} original_hidden_params: Final = copy.deepcopy(hidden_params) From baddc67041e0888b8bcdc32d43e86c946c867c76 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Tue, 3 Nov 2026 04:56:22 -0800 Subject: [PATCH 24/34] fix(anthropic)!: carry prompt_cache_breakpoint to cache_control on chat completions to Claude (#49783) * fix(anthropic): carry prompt_cache_breakpoint to cache_control on chat completions to Claude A content part carrying OpenAI's prompt_cache_breakpoint on a /v1/chat/completions request to Claude (Anthropic, Bedrock Invoke and Converse, Vertex AI, Azure AI) now reaches Claude as cache_control {"type": "ephemeral"} on the same block, so prompt caching works for OpenAI-shaped clients instead of the marker being stripped. Only parts OpenAI accepts the marker on are translated, a client cache_control on the same block wins, and Converse translates only for models that take cache points. A 30m ttl or a malformed marker answers UnsupportedParamsError without drop_params and becomes a default 5m mark or no mark with it. * fix(anthropic): hold translated prompt_cache_breakpoint marks to the 4-breakpoint cap Marks past Claude's 4 cache breakpoints now answer 400 without drop_params and keep the earliest under it, counting the client's own message, tool, and top-level cache_control marks. The ttl and malformed-marker errors name drop_params for proxy users too and no longer name a single provider. * fix(bedrock): count Converse tool cache points in the breakpoint cap and gate unmapped models A client tool cache_control or a tool_config injection point becomes a Converse tool cachePoint, so it now takes one of the 4 slots before a translated marker does. Converse only translates prompt_cache_breakpoint for a model the cost map says takes prompt caching. An unmapped model such as an application inference profile ARN opts in through model_info supports_prompt_cache_breakpoint, so a marker sent to every model of a mixed fleet no longer turns a non-caching profile's 200 into Bedrock's 403. Router prompt-caching affinity now treats a marked block like a cache_control one, and requests without a marker skip the translation after one pass over their messages. * fix(bedrock): count only the tool cache points Converse sends against the breakpoint cap * fix(anthropic,bedrock): cap only the cache breakpoints each provider sends, and drop marks Nova cannot take on tool results * perf(anthropic,bedrock): find the breakpoint cut-off by binary search so trimming many marks stays fast * fix(anthropic): drop the duplicate drop_params left by the main merge * fix(bedrock): let an inference profile's cache breakpoint flag win over its base_model * fix(anthropic): keep the first and last prompt_cache_breakpoint marks past the cap Under drop_params the translated marks that fit the cap are now split between the start and the end of the request, so the stable prefix and the newest turns both stay cached. The cap count also includes a native tool search tool that keeps its cache_control on the wire. Bedrock Converse now translates the marker only for the Claude and Nova families, which Bedrock documents as taking cache checkpoints, unless the deployment sets supports_prompt_cache_breakpoint. A guarded user turn keeps its cache mark when the request fits the cap. * refactor(bedrock): decide guarded cache marks with a typed bool helper * fix(bedrock): validate guarded blocks before reading their cache mark * fix(anthropic): let injection points replace client marks and keep non-Claude routes untranslated When a deployment sets cache_control_injection_points, the client's prompt_cache_breakpoint marks are dropped on targets that do not read them, so the operator's injection points decide every Claude breakpoint. OpenCode Go models served over the Anthropic messages endpoint translate the marker only for Claude, and guarded Converse text keeps a cache mark only on a model known to take cache points. * fix(anthropic): keep client marks unless a message injection point replaces them on a translating route A tool_config-only injection point no longer strips client prompt_cache_breakpoint marks, a flagged Bedrock profile no longer counts as reading the marker natively, and guarded Converse text skips the tool count when no message carries a cache mark. * fix(anthropic): keep Bedrock and Claude deployments flagged for the marker on cache_control injection A supports_prompt_cache_breakpoint flag on a Bedrock profile or a Claude deployment made the injection hook write OpenAI markers and prompt_cache_options, which Bedrock answers 400 for. Routes that translate the marker now never take the OpenAI dialect. * fix(bedrock): reserve only the tool cache points Converse sends when trimming Claude cache marks under drop_params * fix(anthropic): name the marker's accepted ttl values in the malformed prompt_cache_breakpoint 400 --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: ryan-crabbe-berri --- .../anthropic_cache_control_hook.py | 228 ++++++++- .../prompt_templates/common_utils.py | 50 ++ litellm/llms/anthropic/chat/transformation.py | 20 +- .../bedrock/chat/converse_transformation.py | 158 +++++- litellm/llms/bedrock/common_utils.py | 51 +- litellm/llms/opencode/chat/transformation.py | 3 + litellm/router_utils/prompt_caching_cache.py | 25 +- .../test_anthropic_cache_control_hook.py | 130 +++++ .../test_anthropic_chat_transformation.py | 179 +++++++ .../chat/test_converse_transformation.py | 455 ++++++++++++++++++ .../llms/opencode/chat/test_transformation.py | 20 + .../router_utils/test_prompt_caching_cache.py | 27 ++ 12 files changed, 1308 insertions(+), 38 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 4302b24ca33..4e473c2f3f0 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -9,6 +9,7 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and """ +import bisect import copy import itertools import os @@ -21,12 +22,16 @@ from urllib.parse import urlparse from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.integrations.prompt_management_base import PromptManagementClient from litellm.litellm_core_utils.prompt_templates.common_utils import ( + PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT, cache_control_as_prompt_cache_breakpoint, + prompt_cache_breakpoint_as_cache_control, with_prompt_cache_breakpoint, + without_prompt_cache_breakpoints, ) from litellm.litellm_core_utils.prompt_templates.factory import ( tool_call_ids_answered_by_client, @@ -327,6 +332,12 @@ def _part_with_cache_control_renamed(part: object) -> object: return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part)) +def _part_with_prompt_cache_breakpoint_renamed(part: object, drop_params: bool) -> object: + if not _carries_prompt_cache_breakpoint(part): + return part + return prompt_cache_breakpoint_as_cache_control(_CONTENT_PART_ADAPTER.validate_python(part), drop_params) + + def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]: if not isinstance(value, (list, tuple)): return () @@ -551,6 +562,10 @@ def _carries_cache_breakpoint(block: object) -> bool: return any(_attribute_or_key(block, key) is not None for key in CACHE_BREAKPOINT_KEYS) +def _carries_cache_control(block: object) -> bool: + return _attribute_or_key(block, "cache_control") is not None + + def _attribute_or_key(value: object, key: str) -> object | None: if hasattr(value, key): return _ATTRIBUTE_VALUE.validate_python(getattr(value, key)) @@ -630,6 +645,12 @@ def _builds_anthropic_messages(model: str, custom_llm_provider: str | None) -> b return isinstance(chat_config, AnthropicConfig) and not isinstance(chat_config, OpenAIGPTConfig) +def _translates_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool: + return (custom_llm_provider or _served_provider_for_model(model)) == "bedrock" or _builds_anthropic_messages( + model, custom_llm_provider + ) + + def _custom_llm_provider(non_default_params: Mapping[str, object], point: Mapping[str, object]) -> str | None: provider: Final = non_default_params.get("custom_llm_provider") or point.get(CUSTOM_LLM_PROVIDER_STAMP) return provider if isinstance(provider, str) else None @@ -673,10 +694,79 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool: return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES +def _carries_prompt_cache_breakpoint(block: object) -> bool: + return _accepts_prompt_cache_breakpoint(block) and _attribute_or_key(block, "prompt_cache_breakpoint") is not None + + def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: return message.get("role") != "assistant" +def _tool_result_carries_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool: + return message.get("role") == "tool" and any( + _carries_prompt_cache_breakpoint(part) for part in _as_object_list(message.get("content")) or () + ) + + +def _part_without_prompt_cache_breakpoint(part: object) -> object: + if not _carries_prompt_cache_breakpoint(part): + return part + return { + key: value + for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() + if key != "prompt_cache_breakpoint" + } + + +def _tool_result_without_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues: + if not _tool_result_carries_prompt_cache_breakpoint(message): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) or [] + return cast( # cast-ok: the same tool message with its part marks removed + AllMessageValues, {**message, "content": [_part_without_prompt_cache_breakpoint(part) for part in parts]} + ) + + +def _created_cache_control(original: object, translated: object) -> bool: + return ( + _attribute_or_key(original, "cache_control") is None + and _attribute_or_key(translated, "cache_control") is not None + ) + + +def _created_cache_controls(original: object, translated: object) -> tuple[bool, ...]: + if translated is original: + return () + original_parts: Final = _as_object_list(_attribute_or_key(original, "content")) or [] + translated_parts: Final = _as_object_list(_attribute_or_key(translated, "content")) or [] + return tuple(_created_cache_control(before, after) for before, after in zip(original_parts, translated_parts)) + + +def _part_without_cache_control(part: object) -> object: + return {key: value for key, value in _CONTENT_PART_ADAPTER.validate_python(part).items() if key != "cache_control"} + + +def _message_dropping_created_cache_controls_between( + message: AllMessageValues, created: Sequence[bool], marks_before: int, first_dropped: int, first_kept_after: int +) -> AllMessageValues: + if not any(created): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) or [] + ordinals: Final = itertools.accumulate((int(flag) for flag in created), initial=marks_before) + return cast( # cast-ok: the same message with its middle translated marks removed + AllMessageValues, + { + **message, + "content": [ + _part_without_cache_control(part) + if is_created and first_dropped <= ordinal < first_kept_after + else part + for part, is_created, ordinal in zip(parts, created, ordinals) + ], + }, + ) + + # Set by a caller whose message list is not the one that goes upstream -- today the # Responses API layer, whose `instructions` only becomes a system message further down. # Tells this hook to hand role-targeted points to the pass holding the final messages @@ -762,9 +852,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): if not injection_points: return model, messages, non_default_params - # Create a deep copy of messages to avoid modifying the original list - processed_messages = copy.deepcopy(messages) - message_points: Final = tuple( cast(CacheControlMessageInjectionPoint, point) for point in injection_points @@ -775,6 +862,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): request_params: Final = cast(Mapping[str, object], non_default_params) # cast-ok: base hook's bare dict custom_llm_provider: Final = _custom_llm_provider(request_params, injection_points[0]) stamped_dialect: Final = injection_points[0].get("_litellm_openai_dialect") + model_info: Final = routed_model_info(request_params.get("model_info")) openai_dialect: Final = ( stamped_dialect if isinstance(stamped_dialect, bool) @@ -783,9 +871,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): custom_llm_provider, request_params.get("api_base") or request_params.get("base_url"), request_params.get("prompt_cache_options"), - routed_model_info(request_params.get("model_info")), + model_info, ) ) + processed_messages = copy.deepcopy( + [without_prompt_cache_breakpoints(message) for message in messages] + if not openai_dialect and message_points and _translates_prompt_cache_breakpoint(model, custom_llm_provider) + else messages + ) # A provisional message list defers every role-targeted point to the pass holding # the final one: a role with no message here may have one there, and settling all # of them in one pass is what lets config order decide the shared breakpoint @@ -876,7 +969,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) ) if hosted_flag is not None: - return hosted_flag + return hosted_flag and not _translates_prompt_cache_breakpoint(model, custom_llm_provider) if not supports_openai_prompt_cache_breakpoint(model, model_info=model_info): return False if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": @@ -929,6 +1022,19 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return int(wire_cache_control is not None) + tool_blocks + envelope_blocks + @staticmethod + def tools_keeping_cache_control_on_chat(tools: object) -> tuple[object, ...]: + return tuple( + tool for tool in _as_object_list(tools) or () if not _chat_transform_drops_tool_cache_control(tool) + ) + + @staticmethod + def count_chat_external_cache_breakpoints(optional_params: object) -> int: + params: Final = _validated_object_mapping(optional_params) or {} + return AnthropicCacheControlHook.count_external_cache_breakpoints( + _as_object_list(params.get("tools")), params.get("cache_control") + ) + @staticmethod def count_external_cache_breakpoints_on_messages_route( tools: Iterable[object] | None, cache_control: object, request_kwargs: object @@ -1084,6 +1190,16 @@ class AnthropicCacheControlHook(CustomPromptManagement): ) return message_count + content_count + tool_call_count + @staticmethod + def count_converse_cache_points(message: AllMessageValues) -> int: + content: Final = _as_object_list(_attribute_or_key(message, "content")) + tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or [] + marked_parts: Final = sum(1 for block in content or () if _carries_cache_control(block)) + marked_tool_calls: Final = sum(1 for tool_call in tool_calls if _carries_cache_control(tool_call)) + if message.get("role") == "tool": + return int(_carries_cache_control(message) or marked_parts > 0) + return (int(_carries_cache_control(message)) if content is None else marked_parts) + marked_tool_calls + @staticmethod def _message_has_cache_control(message: AllMessageValues, answered_tool_ids: Container[str]) -> bool: """Return True if the message already carries any cache_control.""" @@ -1142,6 +1258,92 @@ class AnthropicCacheControlHook(CustomPromptManagement): return renamed return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed) + @staticmethod + def prompt_cache_breakpoints_as_cache_control(message: AllMessageValues, drop_params: bool) -> AllMessageValues: + """Every OpenAI ``prompt_cache_breakpoint`` on a part OpenAI accepts one on moved to Anthropic's + ``cache_control``. Marks with no OpenAI slot are left for the caller to strip.""" + if not _may_carry_prompt_cache_breakpoint(message): + return message + parts: Final = _as_object_list(_attribute_or_key(message, "content")) + if parts is None or not any(_carries_prompt_cache_breakpoint(part) for part in parts): + return message + return cast( # cast-ok: the same message with its part marks renamed + AllMessageValues, + {**message, "content": [_part_with_prompt_cache_breakpoint_renamed(part, drop_params) for part in parts]}, + ) + + @staticmethod + def prompt_cache_breakpoints_dropped_from_tool_results( + messages: Sequence[AllMessageValues], drop_params: bool, model: str + ) -> tuple[AllMessageValues, ...]: + if not any(_tool_result_carries_prompt_cache_breakpoint(message) for message in messages): + return tuple(messages) + if not drop_params: + raise UnsupportedParamsError( + message=( + f"{model} takes no cache breakpoint on a tool result, so prompt_cache_breakpoint on a tool " + f"message has no equivalent. Remove it, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping prompt_cache_breakpoint from tool results for %s under drop_params", model) + return tuple(_tool_result_without_prompt_cache_breakpoints(message) for message in messages) + + @staticmethod + def prompt_cache_breakpoints_as_capped_cache_control( + messages: Sequence[AllMessageValues], + drop_params: bool, + sent_breakpoints: Callable[[AllMessageValues], int], + external_breakpoints: Callable[[], int], + ) -> tuple[AllMessageValues, ...]: + """``prompt_cache_breakpoints_as_cache_control`` over the whole request, held to the provider's + breakpoint cap, counting each message by the breakpoints ``sent_breakpoints`` says its provider sends. + Under ``drop_params`` the first and last translated marks fill the slots the client's own + ``cache_control`` marks leave, the first half taking the odd slot, so the stable prefix and the newest + turns stay cached, and without it a request over the cap is rejected.""" + translated: Final = tuple( + AnthropicCacheControlHook.prompt_cache_breakpoints_as_cache_control(message, drop_params) + for message in messages + ) + if all(after is before for before, after in zip(messages, translated)): + return translated + created: Final = tuple(_created_cache_controls(before, after) for before, after in zip(messages, translated)) + created_marks: Final = sum(sum(flags) for flags in created) + if created_marks == 0: + return translated + external: Final = external_breakpoints() + breakpoints: Final = sum(map(sent_breakpoints, translated)) + external + excess: Final = breakpoints - MAX_CACHE_CONTROL_BLOCKS + if excess <= 0: + return translated + if not drop_params: + raise UnsupportedParamsError( + message=( + f"This model takes at most {MAX_CACHE_CONTROL_BLOCKS} cache breakpoints per request, and this " + f"request has {breakpoints} once its prompt_cache_breakpoint marks are translated. Remove {excess} " + f"of them, or {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to keep the first and last of them." + ), + status_code=400, + ) + verbose_logger.debug("Dropping the middle prompt_cache_breakpoint marks past the cap under drop_params") + marks_before: Final = tuple(itertools.accumulate((sum(flags) for flags in created), initial=0)) + + def capped_request(kept_marks: int) -> tuple[AllMessageValues, ...]: + first_dropped: Final = (kept_marks + 1) // 2 + first_kept_after: Final = created_marks - kept_marks // 2 + return tuple( + _message_dropping_created_cache_controls_between( + message, flags, before, first_dropped, first_kept_after + ) + for message, flags, before in zip(translated, created, marks_before) + ) + + def over_cap(kept_marks: int) -> bool: + return sum(map(sent_breakpoints, capped_request(kept_marks))) + external > MAX_CACHE_CONTROL_BLOCKS + + first_over_cap: Final = bisect.bisect_left(range(max(0, created_marks - excess) + 1), True, key=over_cap) + return capped_request(max(0, first_over_cap - 1)) + @staticmethod def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues: if not _may_carry_prompt_cache_breakpoint(message): @@ -1487,9 +1689,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points")) if configured: - tools_keeping_marks: Final = tuple( - tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool) - ) + tools_keeping_marks: Final = AnthropicCacheControlHook.tools_keeping_cache_control_on_chat(tools) stamped: Final = AnthropicCacheControlHook._stamped_for_prompt_hook( configured, AnthropicCacheControlHook.count_external_cache_breakpoints( @@ -1782,9 +1982,17 @@ def messages_within_cache_control_cap( Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. """ - reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( - _validated_object_list(tools), cache_control, request_kwargs + return messages_within_reserved_cache_control_cap( + messages, + AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ), ) + + +def messages_within_reserved_cache_control_cap( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) if dropped > 0: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 6287dc17e8f..6ecab166dcd 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -21,12 +21,14 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.batch_utils import InMemoryFile from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAudioObject, + ChatCompletionCachedContent, ChatCompletionFileObject, ChatCompletionImageObject, ChatCompletionPromptCacheBreakpoint, @@ -2183,6 +2185,9 @@ def check_is_function_call(logging_obj: "LoggingClass") -> bool: _MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object]) +PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT: Final = ( + "set `drop_params: true` on the request or in litellm_settings (`litellm.drop_params = True` in the SDK)" +) def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT: @@ -2262,6 +2267,51 @@ def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT: ) +def prompt_cache_breakpoint_as_cache_control(block: _MarkedT, drop_params: bool) -> _MarkedT: + """OpenAI's ``prompt_cache_breakpoint`` mark rewritten as Anthropic's ``cache_control`` on the same block. + A ``cache_control`` the client already wrote on the block wins, TTL included.""" + marker: Final = block.get("prompt_cache_breakpoint") + if marker is None: + return block + unmarked: Final = {key: value for key, value in block.items() if key != "prompt_cache_breakpoint"} + control: Final = ( + None + if block.get("cache_control") is not None + else _cache_control_for_prompt_cache_breakpoint(marker, drop_params) + ) + marked: Final = unmarked if control is None else {**unmarked, "cache_control": control} + return cast(_MarkedT, marked) # cast-ok: same block shape as the input with the marker key renamed + + +def _cache_control_for_prompt_cache_breakpoint(marker: object, drop_params: bool) -> ChatCompletionCachedContent | None: + try: + breakpoint_mark: Final = _PROMPT_CACHE_BREAKPOINT.validate_python(marker) + except ValidationError: + if not drop_params: + raise UnsupportedParamsError( + message=( + f"prompt_cache_breakpoint must be {{'mode': 'explicit'}} with an optional ttl of '30m' or '12h', got " + f"{marker!r}. Fix the marker, or " + f"{PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to drop it." + ), + status_code=400, + ) + verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker) + return None + ttl: Final = breakpoint_mark.get("ttl") + if ttl is not None and not drop_params: + raise UnsupportedParamsError( + message=( + f"This model's prompt cache lasts 5m, or 1h where it offers one, so prompt_cache_breakpoint " + f"ttl={ttl!r} has no equivalent. Remove the ttl, {PROMPT_CACHE_BREAKPOINT_DROP_PARAMS_HINT} to send " + "a 5m breakpoint instead, or send cache_control {'type': 'ephemeral', 'ttl': '1h'} on the block " + "where the model offers a 1h cache." + ), + status_code=400, + ) + return ChatCompletionCachedContent(type="ephemeral") + + _BREAKPOINT_BLOCK_LIST_KEYS: Final = frozenset({"content", "tool_calls"}) _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object]) _BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object]) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1977e3e2a7d..6b96df8f879 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2370,6 +2370,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params ) + def translates_prompt_cache_breakpoint(self, model: str) -> bool: + return True + def transform_request( self, model: str, @@ -2383,19 +2386,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): """ ## VALIDATE REQUEST """Anthropic requires ``tools`` when messages include tool blocks; LiteLLM injects a dummy tool if omitted (no ``modify_params`` needed).""" + from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook, cache_control_count from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, ) drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + translated_messages: Final = ( + list( + AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + messages, + drop_params, + cache_control_count, + lambda: AnthropicCacheControlHook.count_chat_external_cache_breakpoints(optional_params), + ) + ) + if self.translates_prompt_cache_breakpoint(model) + else messages + ) capped_messages: Final = ( list( self._messages_within_cache_control_cap( - messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) + translated_messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) ) ) if drop_params - else messages + else translated_messages ) if "tools" not in optional_params and has_tool_call_blocks(capped_messages): diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f2985237554..056f34f1bf4 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -23,6 +23,11 @@ from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, ) +from litellm.integrations.anthropic_cache_control_hook import ( + MAX_CACHE_CONTROL_BLOCKS, + AnthropicCacheControlHook, + configured_injection_points, +) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, drops_unsupported_params, @@ -131,6 +136,7 @@ from ..common_utils import ( bedrock_capability_model, bedrock_converse_supports_parallel_tool_use_config, bedrock_model_accepts_cache_points, + bedrock_model_known_to_accept_cache_points, bedrock_reasoning_effort_disabled, bedrock_rejects_stop_sequences, get_anthropic_beta_from_headers, @@ -204,6 +210,11 @@ def _has_native_text_format(output_config: object) -> bool: return False +def _cache_mark_kept(block: Mapping[str, object], keeps_cache_marks: bool) -> Mapping[str, object]: + cache_control: Final = block.get("cache_control") + return {"cache_control": cache_control} if keeps_cache_marks and cache_control is not None else {} + + class AmazonConverseConfig(BaseConfig): """ Reference - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html @@ -243,7 +254,7 @@ class AmazonConverseConfig(BaseConfig): @staticmethod def _convert_consecutive_user_messages_to_guarded_text( - messages: list[AllMessageValues], optional_params: dict + messages: list[AllMessageValues], optional_params: dict, keeps_cache_marks: bool = False ) -> list[AllMessageValues]: """ Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present @@ -281,7 +292,11 @@ class AmazonConverseConfig(BaseConfig): new_content = [] for item in content: if isinstance(item, dict) and item.get("type") == "text": - new_item = {"type": "guarded_text", "text": item["text"]} + new_item = { + "type": "guarded_text", + "text": item["text"], + **_cache_mark_kept(_JSON_OBJECT.validate_python(item), keeps_cache_marks), + } new_content.append(new_item) else: new_content.append(item) @@ -289,7 +304,13 @@ class AmazonConverseConfig(BaseConfig): messages_copy[user_message_index]["content"] = new_content elif isinstance(content, str): # If content is a string, convert it to guarded_text - messages_copy[user_message_index]["content"] = [{"type": "guarded_text", "text": content}] + messages_copy[user_message_index]["content"] = [ + { + "type": "guarded_text", + "text": content, + **_cache_mark_kept(_JSON_OBJECT.validate_python(user_message), keeps_cache_marks), + } + ] return messages_copy @@ -1595,6 +1616,74 @@ class AmazonConverseConfig(BaseConfig): return ChatCompletionTextObject(type="text", text=text) return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control) + def _messages_with_cache_marks( + self, + messages: Sequence[AllMessageValues], + model: str, + deployment_model: str, + drop_params: bool, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> Sequence[AllMessageValues]: + if not bedrock_model_known_to_accept_cache_points(model, deployment_model, model_info): + return messages + markable: Final = ( + AnthropicCacheControlHook.prompt_cache_breakpoints_dropped_from_tool_results(messages, drop_params, model) + if "nova" in model.lower() + else messages + ) + return AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + markable, + drop_params, + AnthropicCacheControlHook.count_converse_cache_points, + lambda: self._tool_cache_points_on_the_wire( + model, deployment_model, optional_params, litellm_params, model_info + ), + ) + + def _tool_cache_points_on_the_wire( + self, + model: str, + deployment_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> int: + tools: Final = optional_params.get("tools") + bedrock_tools, _ = self._process_tools_and_beta( + tools if isinstance(tools, list) else [], model, None, {}, litellm_params, deployment_model, model_info + ) + injects_tool_config_point: Final = bool(bedrock_tools) and any( + point.get("location") == "tool_config" + for point in configured_injection_points(optional_params.get("cache_control_injection_points")) + ) + return sum(1 for tool in bedrock_tools if "cachePoint" in tool) + int(injects_tool_config_point) + + def _guarded_text_keeps_cache_marks( + self, + messages: Sequence[AllMessageValues], + system_content_blocks: Sequence[SystemContentBlock], + model: str, + deployment_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, + ) -> bool: + if "guardrailConfig" not in optional_params or not bedrock_model_known_to_accept_cache_points( + model, deployment_model, model_info + ): + return False + message_cache_points: Final = sum(map(AnthropicCacheControlHook.count_converse_cache_points, messages)) + if message_cache_points == 0: + return False + cache_points: Final = ( + sum(1 for block in system_content_blocks if "cachePoint" in block) + + message_cache_points + + self._tool_cache_points_on_the_wire(model, deployment_model, optional_params, litellm_params, model_info) + ) + return cache_points <= MAX_CACHE_CONTROL_BLOCKS + def _transform_system_message( self, messages: list[AllMessageValues], @@ -2152,22 +2241,27 @@ class AmazonConverseConfig(BaseConfig): return data - @staticmethod def _claude_messages_within_cache_control_cap( + self, messages: Sequence[AllMessageValues], capability_model: str, + deployment_model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object], + model_info: Mapping[str, object] | None, ) -> Sequence[AllMessageValues]: if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): return messages from litellm.integrations.anthropic_cache_control_hook import ( - messages_within_cache_control_cap, + messages_within_reserved_cache_control_cap, tool_message_with_single_cache_mark, ) - return messages_within_cache_control_cap( - tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + return messages_within_reserved_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), + self._tool_cache_points_on_the_wire( + capability_model, deployment_model, optional_params, litellm_params, model_info + ), ) async def _async_transform_request( @@ -2183,10 +2277,20 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) model_info: Final = routed_model_info(litellm_params.get("model_info")) + json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params) + marked_messages: Final = self._messages_with_cache_marks( + messages, + capability_model, + model, + drops_unsupported_params(litellm_params.get("drop_params")), + json_optional_params, + litellm_params, + model_info, + ) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, @@ -2195,8 +2299,19 @@ class AmazonConverseConfig(BaseConfig): ) # Convert last user message to guarded_text if guardrailConfig is present + guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params) messages = self._convert_consecutive_user_messages_to_guarded_text( - self._handle_orphaned_tool_blocks(messages, optional_params), optional_params + guard_input, + optional_params, + keeps_cache_marks=self._guarded_text_keeps_cache_marks( + guard_input, + system_content_blocks, + capability_model, + model, + json_optional_params, + litellm_params, + model_info, + ), ) ## TRANSFORMATION ## @@ -2276,10 +2391,20 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) model_info: Final = routed_model_info(litellm_params.get("model_info")) + json_optional_params: Final = _JSON_OBJECT.validate_python(optional_params) + marked_messages: Final = self._messages_with_cache_marks( + messages, + capability_model, + model, + drops_unsupported_params(litellm_params.get("drop_params")), + json_optional_params, + litellm_params, + model_info, + ) messages, system_content_blocks = self._transform_system_message( list( self._claude_messages_within_cache_control_cap( - messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + marked_messages, capability_model, model, json_optional_params, litellm_params, model_info ) ), model=capability_model, @@ -2288,8 +2413,19 @@ class AmazonConverseConfig(BaseConfig): ) # Convert last user message to guarded_text if guardrailConfig is present + guard_input: Final = self._handle_orphaned_tool_blocks(messages, optional_params) messages = self._convert_consecutive_user_messages_to_guarded_text( - self._handle_orphaned_tool_blocks(messages, optional_params), optional_params + guard_input, + optional_params, + keeps_cache_marks=self._guarded_text_keeps_cache_marks( + guard_input, + system_content_blocks, + capability_model, + model, + json_optional_params, + litellm_params, + model_info, + ), ) _data: Final[CommonRequestObject] = self._transform_request_helper( diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 1f10b73505b..c1b3b5616b8 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1559,22 +1559,63 @@ def bedrock_model_accepts_cache_points( def bedrock_declared_cache_point_support(model: str, model_info: Mapping[str, object] | None = None) -> bool | None: if _OPENAI_FAMILY_MODEL_RE.search(model): return False + explicit_marker_support: Final = _explicit_cache_point_support(model, model_info) + if explicit_marker_support is not None: + return explicit_marker_support + return _mapped_prompt_caching_support(model, model_info) + + +def bedrock_model_known_to_accept_cache_points( + model: str, deployment_model: str | None = None, model_info: Mapping[str, object] | None = None +) -> bool: + """Like ``bedrock_model_accepts_cache_points``, but only for the Claude and Nova families Bedrock documents + as taking cache checkpoints, unless the entry sets ``supports_prompt_cache_breakpoint``: an OpenAI + ``prompt_cache_breakpoint`` reaches every model of a mixed fleet, not only the ones a client marked for + Bedrock caching, and other families price implicit cache reads yet may reject the cachePoint field.""" + deployment_declared: Final = ( + None if deployment_model is None else _declared_marker_support(deployment_model, model_info) + ) + declared: Final = ( + deployment_declared if deployment_declared is not None else _declared_marker_support(model, model_info) + ) + return declared is True + + +def _declared_marker_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + if _OPENAI_FAMILY_MODEL_RE.search(model): + return False + explicit_marker_support: Final = _explicit_cache_point_support(model, model_info) + if explicit_marker_support is not None: + return explicit_marker_support + mapped: Final = _mapped_prompt_caching_support(model, model_info) + if not _documents_cache_checkpoints(model): + return None if mapped is None else False + return mapped is not False + + +def _documents_cache_checkpoints(model: str) -> bool: + return _is_claude(model) or get_bedrock_base_model(model).startswith("amazon.nova") + + +def _explicit_cache_point_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: deployment_flag: Final = None if model_info is None else model_info.get("supports_prompt_cache_breakpoint") if deployment_flag is not None: return deployment_flag is True catalog_entry: Final = _catalog_entry_reader(model_info) - map_keys: Final = (model, get_bedrock_base_model(model)) registered_keys: Final = tuple(f"bedrock/{route}{model}" for route in ("", "converse/", "converse_like/")) - explicit_marker_support: Final = next( + return next( ( entry.get("supports_prompt_cache_breakpoint") is True - for key in (*registered_keys, *map_keys) + for key in (*registered_keys, model, get_bedrock_base_model(model)) if (entry := catalog_entry(key)) is not None and entry.get("supports_prompt_cache_breakpoint") is not None ), None, ) - if explicit_marker_support is not None: - return explicit_marker_support + + +def _mapped_prompt_caching_support(model: str, model_info: Mapping[str, object] | None) -> bool | None: + catalog_entry: Final = _catalog_entry_reader(model_info) + map_keys: Final = (model, get_bedrock_base_model(model)) if not any(catalog_entry(key) is not None for key in map_keys): return None deployment_caching: Final = None if model_info is None else lax_bool(model_info.get("supports_prompt_caching")) diff --git a/litellm/llms/opencode/chat/transformation.py b/litellm/llms/opencode/chat/transformation.py index 7e16f93d7cb..c6d288b6d84 100644 --- a/litellm/llms/opencode/chat/transformation.py +++ b/litellm/llms/opencode/chat/transformation.py @@ -112,6 +112,9 @@ class OpenCodeGoMessagesChatConfig(_GoReportedCostMixin, AnthropicConfig): def custom_llm_provider(self) -> str: return "opencode_go" + def translates_prompt_cache_breakpoint(self, model: str) -> bool: + return "claude" in model.lower() + def get_complete_url( self, api_base: str | None, diff --git a/litellm/router_utils/prompt_caching_cache.py b/litellm/router_utils/prompt_caching_cache.py index f78b5982186..1d91255e28c 100644 --- a/litellm/router_utils/prompt_caching_cache.py +++ b/litellm/router_utils/prompt_caching_cache.py @@ -55,6 +55,17 @@ def _sorted_pairs(pairs: Iterable[tuple[str, JsonValue]]) -> tuple[tuple[str, Js return tuple(sorted(pairs, key=lambda pair: pair[0])) +_CACHE_MARKER_KEYS: Final = frozenset(("cache_control", "prompt_cache_breakpoint")) + + +def _is_cache_marked(block: Mapping[str, object]) -> bool: + match block: + case {"cache_control": {"type": "ephemeral"}}: + return True + case _: + return block.get("prompt_cache_breakpoint") is not None + + def _canonical_bytes(value: object) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() @@ -66,7 +77,7 @@ def _block_unit( return _canonical_bytes((envelope, block)), message_run_type block_type: Final = block.get("type") block_run_type: Final = block_type if isinstance(block_type, str) and block_type in _TOOL_RUN_BLOCK_TYPES else None - stripped: Final = _sorted_pairs(item for item in block.items() if item[0] != "cache_control") + stripped: Final = _sorted_pairs(item for item in block.items() if item[0] not in _CACHE_MARKER_KEYS) return _canonical_bytes((envelope, stripped)), message_run_type or block_run_type @@ -264,15 +275,9 @@ class PromptCachingCache: continue for content_idx, content_block in enumerate(content): - if isinstance(content_block, dict): - cache_control = content_block.get("cache_control") - if ( - cache_control is not None - and isinstance(cache_control, dict) - and cache_control.get("type") == "ephemeral" - ): - last_cacheable_message_idx = msg_idx - last_cacheable_content_idx = content_idx + if isinstance(content_block, dict) and _is_cache_marked(content_block): + last_cacheable_message_idx = msg_idx + last_cacheable_content_idx = content_idx # If no cacheable block found, return empty list (no cacheable prefix) if last_cacheable_message_idx is None: diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index c040d8e5bac..766038d5c4f 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -1401,6 +1401,109 @@ def _marks_the_route_sends(model: str, custom_llm_provider: str | None, messages _SECOND_SEARCH_CALL: Final = {**_SEARCH_CALL, "id": "srvtoolu_02Search"} +_CLIENT_MARKED_CONVERSATION: Final = ( + {"role": "system", "content": [{"type": "text", "text": "Handbook", "prompt_cache_breakpoint": {"mode": "explicit"}}]}, + {"role": "user", "content": [{"type": "text", "text": "First", "prompt_cache_breakpoint": {"mode": "explicit"}}]}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second"}, +) + + +def _hook_result( + model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object +) -> tuple[list[AllMessageValues], Mapping[str, object]]: + _, processed, returned_params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=model, + messages=cast(list[AllMessageValues], copy.deepcopy(list(_CLIENT_MARKED_CONVERSATION))), + non_default_params={"cache_control_injection_points": list(points), **params}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + return processed, returned_params + + +def _hook_output( + model: str, points: tuple[Mapping[str, object], ...] = ({"location": "message", "index": -1},), **params: object +) -> list[AllMessageValues]: + return _hook_result(model, points, **params)[0] + + +def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_claude(): + request: Final = AnthropicConfig().transform_request( + model="claude-sonnet-5-5", + messages=_hook_output("anthropic/claude-sonnet-5-5"), + optional_params={"max_tokens": 64}, + litellm_params={}, + headers={}, + ) + + assert _marks_in(request["system"]) == 0 + assert [_marks_in(message) for message in request["messages"]] == [0, 0, 1] + assert "prompt_cache_breakpoint" not in json.dumps(request) + + +@pytest.mark.parametrize( + "sibling_model_infos", + [ + pytest.param((), id="one-deployment"), + pytest.param( + ({"supports_prompt_cache_breakpoint": True, "base_model": "anthropic.claude-unmapped-v1:0"},), + id="beside-a-deployment-of-the-same-profile-with-a-base-model", + ), + ], +) +@pytest.mark.parametrize("route", ["bedrock", "bedrock/converse"]) +def test_cache_control_injection_points_replace_client_prompt_cache_breakpoints_on_a_flagged_bedrock_profile( + local_model_cost_map, sibling_model_infos: tuple[Mapping[str, object], ...], route: str +): + from litellm import Router + + model: Final = f"{route}/arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged" + Router( + model_list=[ + { + "model_name": f"claude-profile-{position}", + "litellm_params": {"model": model, "aws_region_name": "us-east-1"}, + "model_info": model_info, + } + for position, model_info in enumerate(({"supports_prompt_cache_breakpoint": True}, *sibling_model_infos)) + ] + ) + processed, params = _hook_result(model) + + assert "prompt_cache_breakpoint" not in json.dumps(processed) + assert "prompt_cache_options" not in params + assert [_marks_in(message) for message in processed] == [0, 0, 0, 1] + + +def test_tool_config_injection_point_leaves_client_prompt_cache_breakpoints_on_claude(): + processed: Final = _hook_output("anthropic/claude-sonnet-5-5", points=({"location": "tool_config"},)) + + assert [message["content"] for message in processed[:2]] == [ + message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2] + ] + + +@pytest.mark.parametrize( + ("model", "params"), + [ + pytest.param("openai/gpt-5.6", {"api_base": "https://gateway.example.com/v1"}, id="reads-them"), + pytest.param("openai/gpt-4o", {}, id="openai-model-that-predates-them"), + pytest.param("hosted_vllm/qwen3", {"api_base": "http://vllm.example.com/v1"}, id="self-hosted"), + pytest.param("azure_ai/qa-unflagged-gpt", {"api_base": "https://qa.services.ai.azure.com/models"}, id="foundry"), + ], +) +def test_cache_control_injection_points_keep_client_prompt_cache_breakpoints_on_a_route_that_does_not_translate_them( + model: str, params: Mapping[str, object] +): + processed: Final = _hook_output(model, **params) + + assert [message["content"] for message in processed[:2]] == [ + message["content"] for message in _CLIENT_MARKED_CONVERSATION[:2] + ] + + @pytest.mark.parametrize( ("model", "custom_llm_provider"), [ @@ -4965,6 +5068,33 @@ class TestMalformedInjectionPointsAreIgnored: assert kwargs == {"cache_control_injection_points": value} +def test_capping_thousands_of_marks_on_one_tool_result_counts_the_request_a_logarithmic_number_of_times(): + def marked_parts(prefix: str, count: int) -> list[dict]: + return [ + {"type": "text", "text": f"{prefix} {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(count) + ] + + messages: Final = [ + {"role": "user", "content": marked_parts("part", 4)}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": marked_parts("result", 2000)}, + ] + sent_breakpoints: Final = MagicMock(side_effect=AnthropicCacheControlHook.count_converse_cache_points) + + capped: Final = AnthropicCacheControlHook.prompt_cache_breakpoints_as_capped_cache_control( + cast(List[AllMessageValues], messages), True, sent_breakpoints, lambda: 0 + ) + + assert [part.get("cache_control") for part in capped[0]["content"]] == [{"type": "ephemeral"}] * 3 + [None] + assert ["cache_control" in part for part in capped[2]["content"]] == [False] * 1997 + [True] * 3 + assert sent_breakpoints.call_count < 3 * 20 + + def test_cache_control_hook_leaves_the_duplicate_tool_result_warning_to_the_request_builder( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 13ff11e8a3f..fc873832956 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,4 +1,5 @@ import copy +import itertools import json import logging from collections.abc import Iterator @@ -4422,6 +4423,184 @@ def test_transform_request_includes_cache_control(): assert result["cache_control"] == {"type": "ephemeral"} +def _anthropic_body(messages: list, litellm_params: dict | None = None, tools: list | None = None) -> dict: + return AnthropicConfig().transform_request( + model="claude-sonnet-5-5", + messages=messages, + optional_params={"max_tokens": 100, **({"tools": tools} if tools else {})}, + litellm_params=litellm_params or {}, + headers={}, + ) + + +def test_transform_request_carries_prompt_cache_breakpoint_to_cache_control(): + body: Final = _anthropic_body( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}, + { + "type": "text", + "text": "Bye", + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + ], + }, + ] + ) + + assert body["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}] + assert body["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Bye", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + } + ] + + +@pytest.mark.parametrize( + "marker, expected_block", + [ + ({"mode": "explicit", "ttl": "30m"}, {"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}), + ({"mode": "implicit"}, {"type": "text", "text": "Hello"}), + ], +) +def test_transform_request_prompt_cache_breakpoint_without_claude_equivalent(marker, expected_block): + messages: Final = [{"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": marker}]}] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + _anthropic_body(copy.deepcopy(messages)) + + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}) + assert body["messages"] == [{"role": "user", "content": [expected_block]}] + + +def test_transform_request_names_the_marker_ttl_rule_for_a_ttl_only_claude_takes(): + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "1h"}}]} + ] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="optional ttl of '30m' or '12h'"): + _anthropic_body(messages) + + +def test_transform_request_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): + client_marked_part: Final = {"type": "text", "text": "part 0", "cache_control": {"type": "ephemeral"}} + marked_parts: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(1, 5) + ] + messages: Final = [{"role": "user", "content": [client_marked_part, *marked_parts]}] + tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "cache_control": {"type": "ephemeral"}}] + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): + _anthropic_body(copy.deepcopy(messages), tools=copy.deepcopy(tools)) + + body: Final = _anthropic_body(messages, litellm_params={"drop_params": True}, tools=tools) + assert [part.get("cache_control") for part in body["messages"][0]["content"]] == [ + {"type": "ephemeral"}, + {"type": "ephemeral"}, + None, + None, + {"type": "ephemeral"}, + ] + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + + +def test_transform_request_counts_a_cache_marked_tool_search_tool_toward_the_cap(): + config: Final = AnthropicConfig() + optional_params: Final = config.map_openai_params( + non_default_params={ + "max_tokens": 100, + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + optional_params={}, + model="claude-sonnet-5-5", + drop_params=False, + ) + marked_turns: Final = [ + { + "role": "user", + "content": [{"type": "text", "text": f"turn {index}", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + for index in range(4) + ] + + def body(litellm_params: dict) -> dict: + return config.transform_request( + model="claude-sonnet-5-5", + messages=copy.deepcopy(marked_turns), + optional_params=copy.deepcopy(optional_params), + litellm_params=litellm_params, + headers={}, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): + body({}) + + capped: Final = body({"drop_params": True}) + assert capped["tools"][0]["cache_control"] == {"type": "ephemeral"} + sent_parts: Final = itertools.chain.from_iterable(message["content"] for message in capped["messages"]) + assert [part.get("cache_control") for part in sent_parts] == [ + {"type": "ephemeral"}, + {"type": "ephemeral"}, + None, + {"type": "ephemeral"}, + ] + + +@pytest.mark.parametrize( + "unsent_marks", + [ + pytest.param( + [ + { + "role": "assistant", + "content": [{"type": "text", "text": "Noted.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + ], + id="assistant_part", + ), + pytest.param( + [{"role": "user", "content": "Also this.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + id="message_level", + ), + ], +) +def test_transform_request_caps_only_the_marks_claude_receives(unsent_marks: list[dict]): + marked_parts: Final = [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} for index in range(4) + ] + + body: Final = _anthropic_body( + [{"role": "user", "content": marked_parts}, *unsent_marks, {"role": "user", "content": "Go on."}] + ) + + sent_marks: Final = [ + part.get("cache_control") + for part in itertools.chain.from_iterable(message["content"] for message in body["messages"]) + if isinstance(part, dict) + ] + assert sent_marks.count({"type": "ephemeral"}) == 4 + + def test_compaction_block_empty_list_not_added(): """ Test that empty compaction_blocks list is not added to provider_specific_fields. diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 8cab1018d25..7c19ef5b751 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -10073,6 +10073,461 @@ def test_bedrock_prompt_caching_message(messages, expected_cache_control): assert "cachePoint" not in json.dumps(transformed_messages) +def test_converse_carries_prompt_cache_breakpoint_to_cache_point(): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}} + ], + }, + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + ], + optional_params={}, + litellm_params={}, + ) + + assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] + assert request["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]} + ] + + +def test_converse_keeps_the_first_and_last_prompt_cache_breakpoints_within_the_cap(): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(5) + ], + } + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params={}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="at most 4 cache breakpoints"): + transform({}) + + cache_point: Final = {"cachePoint": {"type": "default"}} + assert transform({"drop_params": True})["messages"] == [ + { + "role": "user", + "content": [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + {"text": "part 3"}, + cache_point, + {"text": "part 4"}, + cache_point, + ], + } + ] + + +_CONVERSE_CAP_TOOL: Final[dict[str, object]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, +} + + +@pytest.mark.parametrize( + "tool_cache_params", + [ + {"tools": [{**_CONVERSE_CAP_TOOL, "cache_control": {"type": "ephemeral"}}]}, + { + "tools": [_CONVERSE_CAP_TOOL], + "cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}], + }, + ], + ids=["client_tool_cache_control", "tool_config_injection_point"], +) +def test_converse_reserves_the_tool_cache_point_within_the_cap(tool_cache_params: dict[str, object]): + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(4) + ], + } + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params=copy.deepcopy(tool_cache_params), + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="this request has 5"): + transform({}) + + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + {"text": "part 3"}, + cache_point, + ] + assert request["toolConfig"]["tools"][-1] == cache_point + + + +@pytest.mark.parametrize( + "unsent_tool_params", + [ + { + "tools": [ + _CONVERSE_CAP_TOOL, + {"type": "tool_search_tool_regex_20251119", "name": "find", "cache_control": {"type": "ephemeral"}}, + ] + }, + {"tools": [_CONVERSE_CAP_TOOL, {"type": "web_search", "cache_control": {"type": "ephemeral"}}]}, + { + "tools": [{"type": "web_search"}], + "cache_control_injection_points": [{"location": "tool_config", "control": {"type": "ephemeral"}}], + }, + ], + ids=["tool_search_tool", "built_in_tool", "tool_config_point_with_no_tool_sent"], +) +@pytest.mark.parametrize("litellm_params", [{}, {"drop_params": True}], ids=["strict", "drop_params"]) +def test_converse_leaves_the_cap_to_markers_when_a_marked_tool_is_not_sent( + unsent_tool_params: dict[str, object], litellm_params: dict[str, object] +): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(4) + ], + } + ], + optional_params=copy.deepcopy(unsent_tool_params), + litellm_params=dict(litellm_params), + ) + + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + cache_point, + {"text": "part 3"}, + cache_point, + ] + assert cache_point not in request.get("toolConfig", {}).get("tools", []) + +def _converse_tool_turn(*result_parts: dict[str, object]) -> list[dict[str, object]]: + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": list(result_parts)}, + ] + + +def _marked_text_parts(count: int) -> list[dict[str, object]]: + return [ + {"type": "text", "text": f"part {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(count) + ] + + +def _marked_tool_result_parts() -> list[dict[str, object]]: + return [ + {"type": "text", "text": f"result {index}", "prompt_cache_breakpoint": {"mode": "explicit"}} + for index in range(2) + ] + + +def test_converse_counts_one_cache_point_for_a_tool_result_with_several_marks(): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + {"role": "user", "content": _marked_text_parts(3)}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ], + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params={}, + ) + + assert len(_collect_cache_points(request["messages"])) == 4 + + +def test_converse_drops_marks_until_a_merged_tool_result_fits_the_cap(): + messages: Final = [ + {"role": "user", "content": _marked_text_parts(4)}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=copy.deepcopy(messages), + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="this request has 5"): + transform({}) + + request: Final = transform({"drop_params": True}) + cache_point: Final = {"cachePoint": {"type": "default"}} + assert request["messages"][0]["content"] == [ + {"text": "part 0"}, + cache_point, + {"text": "part 1"}, + cache_point, + {"text": "part 2"}, + cache_point, + {"text": "part 3"}, + ] + assert len(_collect_cache_points(request["messages"][1:])) == 1 + + +def test_converse_drops_prompt_cache_breakpoint_from_a_nova_tool_result(): + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "You are terse.", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + {"role": "user", "content": "Look it up."}, + *_converse_tool_turn(*_marked_tool_result_parts()), + ] + + def transform(litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="us.amazon.nova-lite-v1:0", + messages=copy.deepcopy(messages), + optional_params={"tools": [_CONVERSE_CAP_TOOL]}, + litellm_params=litellm_params, + ) + + with pytest.raises(litellm.UnsupportedParamsError, match="tool result"): + transform({}) + + request: Final = transform({"drop_params": True}) + assert request["system"] == [{"text": "You are terse."}, {"cachePoint": {"type": "default"}}] + assert _collect_cache_points(request["messages"]) == [] + + +def test_converse_translates_prompt_cache_breakpoint_on_an_unmapped_arn_only_when_flagged(local_model_cost_map): + from litellm import Router + + flagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-flagged" + unflagged_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/mistral" + Router( + model_list=[ + { + "model_name": "claude-profile", + "litellm_params": {"model": f"bedrock/{flagged_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": True}, + }, + { + "model_name": "mistral-profile", + "litellm_params": {"model": f"bedrock/{unflagged_arn}", "aws_region_name": "us-east-1"}, + }, + ] + ) + + def user_content(model: str, ttl: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + } + ], + optional_params={}, + litellm_params={}, + )["messages"][0]["content"] + + assert user_content(flagged_arn, {}) == [{"text": "Hello"}, {"cachePoint": {"type": "default"}}] + assert user_content(unflagged_arn, {}) == [{"text": "Hello"}] + assert user_content(unflagged_arn, {"ttl": "30m"}) == [{"text": "Hello"}] + + + +def test_converse_prompt_cache_breakpoint_follows_the_arn_flag_over_its_base_model(local_model_cost_map): + from litellm import Router + + opted_in_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-next" + opted_out_arn: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude-opted-out" + Router( + model_list=[ + { + "model_name": "opted-in", + "litellm_params": {"model": f"bedrock/{opted_in_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": True}, + }, + { + "model_name": "opted-out", + "litellm_params": {"model": f"bedrock/{opted_out_arn}", "aws_region_name": "us-east-1"}, + "model_info": {"supports_prompt_cache_breakpoint": False}, + }, + ] + ) + + def user_content(model: str, base_model: str, ttl: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", **ttl}}], + } + ], + optional_params={}, + litellm_params={"base_model": base_model}, + )["messages"][0]["content"] + + assert user_content(opted_in_arn, "anthropic.claude-unmapped-v1:0", {}) == [ + {"text": "Hello"}, + {"cachePoint": {"type": "default"}}, + ] + assert user_content(opted_out_arn, "anthropic.claude-sonnet-5-5", {"ttl": "30m"}) == [{"text": "Hello"}] + + +def test_converse_translates_prompt_cache_breakpoint_only_for_families_that_take_cache_points(local_model_cost_map): + litellm.register_model( + {"zai.glm-next-v1:0": {"litellm_provider": "bedrock_converse", "mode": "chat", "supports_prompt_caching": True}} + ) + + def user_content(model: str, mark: dict) -> list: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", **mark}]}], + optional_params={}, + litellm_params={}, + )["messages"][0]["content"] + + marker: Final = {"prompt_cache_breakpoint": {"mode": "explicit"}} + client_mark: Final = {"cache_control": {"type": "ephemeral"}} + cache_point: Final = {"cachePoint": {"type": "default"}} + assert user_content("zai.glm-next-v1:0", marker) == [{"text": "Hello"}] + assert user_content("zai.glm-next-v1:0", client_mark) == [{"text": "Hello"}, cache_point] + assert user_content("us.anthropic.claude-next-v1:0", marker) == [{"text": "Hello"}, cache_point] + assert user_content("us.amazon.nova-next-v1:0", marker) == [{"text": "Hello"}, cache_point] + + +def test_converse_prompt_cache_breakpoint_on_guarded_user_text_keeps_its_cache_point(local_model_cost_map): + guardrail: Final = {"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}} + + def request(system_marks: int) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + *( + { + "role": "system", + "content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}], + } + for i in range(system_marks) + ), + { + "role": "user", + "content": [{"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit"}}], + }, + ], + optional_params=dict(guardrail), + litellm_params={}, + ) + + assert request(0)["messages"][0]["content"] == [ + {"guardContent": {"text": {"text": "Hello"}}}, + {"cachePoint": {"type": "default"}}, + ] + with pytest.raises(litellm.UnsupportedParamsError): + request(4) + + client_marked: Final = litellm.AmazonConverseConfig()._transform_request( + model="anthropic.claude-sonnet-5-5", + messages=[ + *( + { + "role": "system", + "content": [{"type": "text", "text": f"Rule {i}", "cache_control": {"type": "ephemeral"}}], + } + for i in range(4) + ), + {"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}, + ], + optional_params=dict(guardrail), + litellm_params={}, + ) + assert sum("cachePoint" in block for block in client_marked["system"]) == 4 + assert client_marked["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}] + + +def test_converse_guarded_user_text_keeps_no_cache_point_on_a_model_not_known_to_take_one(local_model_cost_map): + request: Final = litellm.AmazonConverseConfig()._transform_request( + model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/unmapped1", + messages=[{"role": "user", "content": [{"type": "text", "text": "Hello", "cache_control": {"type": "ephemeral"}}]}], + optional_params={"guardrailConfig": {"guardrailIdentifier": "gr-1", "guardrailVersion": "DRAFT"}}, + litellm_params={}, + ) + + assert request["messages"][0]["content"] == [{"guardContent": {"text": {"text": "Hello"}}}] + + +def _converse_request_with_30m_breakpoint(model: str, litellm_params: dict) -> dict: + return litellm.AmazonConverseConfig()._transform_request( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}} + ], + } + ], + optional_params={}, + litellm_params=litellm_params, + ) + + +def test_converse_rejects_30m_breakpoint_ttl_only_where_a_cache_point_is_written(): + claude: Final = "anthropic.claude-sonnet-5-5" + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"): + _converse_request_with_30m_breakpoint(claude, {}) + + assert _converse_request_with_30m_breakpoint(claude, {"drop_params": True})["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}, {"cachePoint": {"type": "default"}}]} + ] + assert _converse_request_with_30m_breakpoint("meta.llama3-1-8b-instruct-v1:0", {})["messages"] == [ + {"role": "user", "content": [{"text": "Hello"}]} + ] + + @pytest.mark.parametrize( "model, expected_supports_tool_call", [ diff --git a/tests/unit/llms/opencode/chat/test_transformation.py b/tests/unit/llms/opencode/chat/test_transformation.py index 91361a3d81d..08c80db294c 100644 --- a/tests/unit/llms/opencode/chat/test_transformation.py +++ b/tests/unit/llms/opencode/chat/test_transformation.py @@ -14,6 +14,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.opencode.chat.transformation import OpenCodeGoMessagesChatConfig from litellm.llms.opencode.common_utils import go_headers @@ -591,3 +592,22 @@ def test_stream_chunk_builder_reads_list_cost_from_dict_chunks(): assert assembled is not None assert assembled._hidden_params["response_cost"] == GO_REPORTED_USAGE["opencode"]["list_cost"] + + +@pytest.mark.parametrize(("model", "cache_marks"), [("qwen3.8-flash", 0), ("minimax-m3", 0), ("claude-haiku-5-5", 4)]) +def test_go_messages_route_carries_prompt_cache_breakpoint_to_cache_control_only_for_claude(model: str, cache_marks: int): + marked_notes: Final = [ + {"type": "text", "text": f"Note {n}", "prompt_cache_breakpoint": {"mode": "explicit"}} for n in range(4) + ] + + request: Final = OpenCodeGoMessagesChatConfig().transform_request( + model=model, + messages=[{"role": "user", "content": [*marked_notes, {"type": "text", "text": "Answer in one line."}]}], + optional_params={"max_tokens": 64}, + litellm_params={}, + headers={}, + ) + + wire: Final = json.dumps(request) + assert wire.count('"cache_control"') == cache_marks + assert "prompt_cache_breakpoint" not in wire diff --git a/tests/unit/router_utils/test_prompt_caching_cache.py b/tests/unit/router_utils/test_prompt_caching_cache.py index ce8f04b8c54..1500d7433d2 100644 --- a/tests/unit/router_utils/test_prompt_caching_cache.py +++ b/tests/unit/router_utils/test_prompt_caching_cache.py @@ -1,4 +1,5 @@ import asyncio +from collections.abc import Mapping from typing import Final, cast import pytest @@ -377,6 +378,32 @@ def test_extract_cacheable_prefix_mixed_string_and_list_content(): assert isinstance(result[2]["content"], list) +def test_prompt_cache_breakpoint_marks_the_same_cacheable_prefix_as_cache_control(): + def _messages(marker: Mapping[str, Mapping[str, str]]) -> list[AllMessageValues]: + return cast( # cast-ok: the text part type does not declare the prompt_cache_breakpoint key + list[AllMessageValues], + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Here is the full text of a complex legal agreement" * 400, **marker} + ], + }, + {"role": "user", "content": "what are the key terms?"}, + ], + ) + + marked_key: Final = PromptCachingCache.get_prompt_caching_cache_key( + _messages({"prompt_cache_breakpoint": {"mode": "explicit"}}), None + ) + + assert marked_key is not None + assert marked_key == PromptCachingCache.get_prompt_caching_cache_key( + _messages({"cache_control": {"type": "ephemeral"}}), None + ) + assert PromptCachingCache.get_prompt_caching_cache_key(_messages({}), None) is None + + @pytest.mark.asyncio async def test_same_cacheable_prefix_routes_to_same_deployment() -> None: from litellm import Router From 1cc7da6274b861e5841c4ddbabf68c2ac8fcf002 Mon Sep 17 00:00:00 2001 From: tessellark <349031000+tessellark@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:14:04 -0800 Subject: [PATCH 25/34] fix(proxy): list slash-less wildcard groups like claude-* on /v1/models and /model/info (#49832) * fix(proxy): list slash-less wildcard groups like claude-* on /v1/models and /model/info * fix(proxy): expand team-owned wildcard rows on /model/info through their public name * fix(proxy): keep listing wildcard names wider than their deployment and skip the Azure placeholder A group like *gemini-3* over gemini/gemini-3* lists the catalog ids the name matches and routes back, as main did on /model/info, now on /v1/models too. Wildcard listings no longer list the Azure-LLM placeholder. Team-owned wildcard rows on /model/info go back to main's behavior. * fix(proxy): keep Azure listings on slashed wildcards and list bare deployments through custom_llm_provider get_provider_models returns main's list again, and only slash-less names skip the Azure-LLM catalog placeholder. A slash-less name over a bare wildcard deployment now lists the catalog of its custom_llm_provider. * test(proxy): pin custom_llm_provider guards and slashed wider-name listing * fix(proxy): list bare ids for wider names over a custom_llm_provider deployment --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 113 +++++++++++----- litellm/utils.py | 5 +- tests/unit/proxy/auth/test_model_checks.py | 145 ++++++++++++++++++++- 3 files changed, 227 insertions(+), 36 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index e41c9354a8a..73edcd66bd7 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -23,7 +23,12 @@ from litellm.router import Router from litellm.router_api import fallback_chain from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order +from litellm.utils import ( + AZURE_MODEL_PLACEHOLDER, + ProviderConfigManager, + get_valid_models, + potential_model_names_in_lookup_order, +) LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) _PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value}) @@ -375,6 +380,10 @@ def _ids_routing_to_catalog_models( catalog_ids: Sequence[str], routes_to_model: Callable[[str, str], bool], ) -> tuple[str, ...]: + if name_pattern.count("*") > deployment_pattern.count("*"): + return _catalog_ids_a_wider_name_matches( + name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model + ) if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments( name_pattern, deployment_pattern ): @@ -419,6 +428,23 @@ def _ids_routing_to_catalog_models( ) +def _catalog_ids_a_wider_name_matches( + name_pattern: str, + deployment_pattern: str, + provider: str, + catalog_ids: Sequence[str], + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: + name_regex: Final = _wildcard_name_regex(name_pattern) + return tuple( + catalog_id + for catalog_id in _ids_routing_to_catalog_models( + deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model + ) + if name_regex.fullmatch(catalog_id) + ) + + def _model_info_field(deployment: Mapping[str, object], field: str) -> object: model_info: Final = deployment.get("model_info") return model_info.get(field) if is_str_keyed_mapping(model_info) else None @@ -456,11 +482,12 @@ def _routes_to_model( return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id) -def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None: +def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None: if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model): return None + name_suffix: Final = wildcard_model.split("/", 1)[-1] deployment_suffix: Final = litellm_params.model.partition("/")[2] - return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None + return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None def get_known_models_from_wildcard( @@ -468,6 +495,43 @@ def get_known_models_from_wildcard( litellm_params: LiteLLM_Params | None = None, routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True, ) -> list[str]: + provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params) + if litellm_params is None or provider_id_prefix is None: + return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model)) + return [ + listed_id.removeprefix(provider_id_prefix) + for listed_id in _known_models_from_wildcard( + wildcard_model, + litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}), + lambda listed_id, model: routes_to_model( + listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix) + ), + ) + ] + + +def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None: + if ( + litellm_params is None + or not litellm_params.custom_llm_provider + or "/" in wildcard_model + or wildcard_model == "*" + or "/" in litellm_params.model + or not _check_wildcard_routing(litellm_params.model) + ): + return None + return f"{litellm_params.custom_llm_provider}/" + + +def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]: + return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER) + + +def _known_models_from_wildcard( + wildcard_model: str, + litellm_params: LiteLLM_Params | None, + routes_to_model: Callable[[str, str], bool], +) -> tuple[str, ...]: wildcard_model_to_expand: Final = ( litellm_params.model if wildcard_model == "*" @@ -476,15 +540,14 @@ def get_known_models_from_wildcard( and "/" in litellm_params.model else wildcard_model ) - try: - wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1) - except ValueError: # safely fail - return [] deployment_pattern_naming_listed_ids: Final = ( wildcard_model_to_expand if wildcard_model_to_expand != wildcard_model - else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params) + else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params) ) + wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/") + if not has_provider_prefix and deployment_pattern_naming_listed_ids is None: + return () # Use provider from litellm_params when available, otherwise from wildcard prefix # (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router) @@ -501,16 +564,14 @@ def get_known_models_from_wildcard( wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params) if wildcard_models is None: - return [] + return () if deployment_pattern_naming_listed_ids is not None: - return list( - _ids_routing_to_catalog_models( - name_pattern=wildcard_model, - deployment_pattern=deployment_pattern_naming_listed_ids, - provider=provider, - catalog_ids=wildcard_models, - routes_to_model=routes_to_model, - ) + return _ids_routing_to_catalog_models( + name_pattern=wildcard_model, + deployment_pattern=deployment_pattern_naming_listed_ids, + provider=provider, + catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models), + routes_to_model=routes_to_model, ) if wildcard_suffix != "*": model_prefix: Final = wildcard_suffix.replace("*", "") @@ -526,7 +587,7 @@ def get_known_models_from_wildcard( # add model prefix to wildcard models wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models] - return list( + return tuple( dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models) ) @@ -564,16 +625,6 @@ def _names_no_deployment_serves_by_name( return tuple(name for name in names if not deployed_by_name(name, team_id)) -def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None: - if not _check_wildcard_routing(model_name): - return None - if "/" in model_name or model_name == "*": - return model_name - if _check_wildcard_routing(litellm_model): - return litellm_model - return model_name - - def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool: if allowed_model_names is None: return True @@ -601,10 +652,10 @@ def expand_wildcard_deployments_for_model_info( raw_params = deployment.get("litellm_params") litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {} litellm_model = str(litellm_params_dict.get("model") or "") - if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None: + if not _check_wildcard_routing(model_name): expanded.append(deployment) continue - if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names): + if not _can_expand_to_an_allowed_name(model_name, allowed_model_names): continue try: @@ -613,7 +664,7 @@ def expand_wildcard_deployments_for_model_info( expanded.append(deployment) continue expanded_names = get_known_models_from_wildcard( - wildcard_model=wildcard_pattern, + wildcard_model=model_name, litellm_params=litellm_params, routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), ) diff --git a/litellm/utils.py b/litellm/utils.py index d881289deed..e003f0a3987 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9069,6 +9069,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]: return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {}))) +AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM" + + def get_valid_models( check_provider_endpoint: bool | None = None, custom_llm_provider: str | None = None, @@ -9124,7 +9127,7 @@ def get_valid_models( continue if provider == "azure": - valid_models.append("Azure-LLM") + valid_models.append(AZURE_MODEL_PLACEHOLDER) elif provider_config is not None and check_provider_endpoint and provider is not None: valid_models.extend( _get_valid_models_from_provider_api( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index eef7e51982a..bc9366aa607 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -19,6 +19,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID from litellm import Router +from litellm.utils import AZURE_MODEL_PLACEHOLDER def test_get_team_models_for_all_models_and_team_only_models(): @@ -1105,13 +1106,13 @@ def test_expand_wildcard_invalid_litellm_params_passthrough(): @pytest.mark.parametrize( ("allowed_model_names", "expected_names", "expected_listings"), [ - pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"), - pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"), - pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"), + pytest.param(None, ["qwen-qwen2.5"], 1, id="unscoped"), + pytest.param(["qwen-qwen2.5"], ["qwen-qwen2.5"], 1, id="granted-a-listed-name"), + pytest.param(["hosted_vllm/qwen2.5"], [], 0, id="granted-only-the-provider-id"), ], ) @respx.mock -def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern( +def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_own_pattern( monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest, allowed_model_names: list[str] | None, @@ -2099,6 +2100,34 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} pytest.param( "*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all" ), + pytest.param( + "gpt-partial-route-*", + "openai/gpt-partial-route-*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="slashless-name-repeating-the-deployment-partial-word", + ), + pytest.param("team-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="slashless-alias-over-partial-deployment"), + pytest.param( + "team-*", "openai/*", None, {f"openai/{_ROUTE_OPENAI_ID}"}, id="slashless-alias-over-provider-catch-all" + ), + pytest.param( + "team-*", "cloudflare/@cf/*/*", None, set(), id="slashless-alias-with-fewer-wildcards-than-its-deployment" + ), + pytest.param( + "*partial-route-9*", + "openai/gpt-partial-route-9*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="slashless-name-with-more-wildcards-than-its-deployment", + ), + pytest.param( + "gemini/*partial-route-9*", + "gemini/gemini-*", + None, + {_ROUTE_GEMINI_ID}, + id="slashed-name-with-more-wildcards-than-its-deployment", + ), pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"), pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"), pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"), @@ -2176,6 +2205,114 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names( assert listed_model_info_names == sorted(reachable) +@pytest.mark.parametrize( + ("model_name", "expected_v1_models"), + [ + pytest.param("azure/*", [f"azure/{AZURE_MODEL_PLACEHOLDER}"], id="provider-wildcard-lists-as-before"), + pytest.param("*", [AZURE_MODEL_PLACEHOLDER], id="catch-all-lists-as-before"), + pytest.param("my-*", [], id="slashless-alias-skips-the-placeholder"), + ], +) +def test_only_slashless_azure_wildcards_skip_the_catalog_placeholder( + model_name: str, expected_v1_models: list[str] +) -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "azure/*"}}]) + + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + ] + + assert v1_models == expected_v1_models + assert model_info_names == (expected_v1_models or [model_name]) + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + pytest.param("gpt-*", "gpt-*", id="same-wildcards-as-its-deployment"), + pytest.param("*gpt-5*", "gpt-5*", id="more-wildcards-than-its-deployment"), + ], +) +def test_slashless_wildcard_over_a_bare_deployment_lists_its_custom_llm_provider_catalog( + model_name: str, deployment_model: str +) -> None: + from fnmatch import fnmatchcase + + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + from litellm.utils import get_valid_models + + router: Final = Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": {"model": deployment_model, "custom_llm_provider": "openai", "api_key": "sk-test"}, + } + ] + ) + + v1_models: Final = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=[model_name], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_rows: Final = expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + openai_ids: Final = frozenset(model.removeprefix("openai/") for model in get_valid_models(custom_llm_provider="openai")) + routed_models: Final = { + listed_id: [deployment["litellm_params"]["model"] for deployment in router.wildcard_route_deployments(listed_id, None)] + for listed_id in v1_models + } + + assert v1_models + assert [ + listed_id for listed_id in v1_models if listed_id not in openai_ids or not fnmatchcase(listed_id, model_name) + ] == [] + assert {listed_id: models for listed_id, models in routed_models.items() if models != [listed_id]} == {} + assert sorted(str(row["model_name"]) for row in model_info_rows) == sorted(v1_models) + assert [row for row in model_info_rows if row["litellm_params"]["model"] != row["model_name"]] == [] + + +@pytest.mark.parametrize( + ("model_name", "deployment_model"), + [ + pytest.param("*", "*", id="catch-all-over-a-bare-catch-all"), + pytest.param("foo/gpt-*", "gpt-*", id="slashed-name-over-a-bare-deployment"), + pytest.param("gpt-*", "openai/gpt-*", id="slashless-name-over-a-provider-prefixed-deployment"), + ], +) +def test_custom_llm_provider_changes_no_listing_outside_slashless_names_over_bare_deployments( + model_name: str, deployment_model: str +) -> None: + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + with_provider: Final = get_known_models_from_wildcard( + model_name, LiteLLM_Params(model=deployment_model, custom_llm_provider="openai") + ) + without_provider: Final = get_known_models_from_wildcard(model_name, LiteLLM_Params(model=deployment_model)) + + assert with_provider == without_provider + + def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None: from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params From 36cd03f10d804c4689edd7b848724df387ab417c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:10:58 -0800 Subject: [PATCH 26/34] fix(gemini): lower the gemini-3.5-flash-lite batch cache price and date the gemini-2.5-pro and 2.5-flash shutdown (#49871) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 +++- model_prices_and_context_window.json | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index be93d6fcd9b..0f895c63ca8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -31421,6 +31421,7 @@ "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-12-01", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, @@ -32004,6 +32005,7 @@ "output_cost_per_token_flex": 5e-06, "supports_url_context": true, "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "deprecation_date": "2026-12-01", "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, @@ -32107,7 +32109,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 2e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index be93d6fcd9b..0f895c63ca8 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -31421,6 +31421,7 @@ "cache_read_input_token_cost_batches": 3e-08, "cache_read_input_token_cost_flex": 3e-08, "cache_read_input_token_cost_priority": 5.4e-08, + "deprecation_date": "2026-12-01", "input_cost_per_audio_token": 1e-06, "input_cost_per_audio_token_flex": 5e-07, "input_cost_per_token": 3e-07, @@ -32004,6 +32005,7 @@ "output_cost_per_token_flex": 5e-06, "supports_url_context": true, "cache_read_input_token_cost_above_200k_tokens_batches": 2.5e-07, + "deprecation_date": "2026-12-01", "input_cost_per_token_above_200k_tokens_batches": 1.25e-06, "output_cost_per_token_above_200k_tokens_batches": 7.5e-06 }, @@ -32107,7 +32109,7 @@ }, "gemini/gemini-3.5-flash-lite": { "cache_read_input_token_cost": 3e-08, - "cache_read_input_token_cost_batches": 2e-08, + "cache_read_input_token_cost_batches": 1.5e-08, "cache_read_input_token_cost_flex": 2e-08, "cache_read_input_token_cost_priority": 5e-08, "input_cost_per_token": 3e-07, From add7dea9aa87eb5e845b396ac892c5e7f649e29c Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:53:49 +0000 Subject: [PATCH 27/34] fix(vertex-ai): drop batch prices from gemini-robotics-er-2-preview (#49895) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 12 ++---------- model_prices_and_context_window.json | 12 ++---------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0f895c63ca8..974f1bb0c8b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76956,23 +76956,19 @@ }, "vertex_ai/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -77002,23 +76998,19 @@ }, "gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0f895c63ca8..974f1bb0c8b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76956,23 +76956,19 @@ }, "vertex_ai/gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", @@ -77002,23 +76998,19 @@ }, "gemini-robotics-er-2-preview": { "cache_read_input_token_cost": 1e-07, - "cache_read_input_token_cost_batches": 5e-08, "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, "max_output_tokens": 65536, "mode": "chat", "output_cost_per_reasoning_token": 5e-06, "output_cost_per_token": 5e-06, - "output_cost_per_token_batches": 2.5e-06, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_endpoints": [ "/v1/chat/completions", - "/v1/completions", - "/v1/batch" + "/v1/completions" ], "supported_modalities": [ "text", From 0aa7079f042f9611272b2a08f50b405eec7975af Mon Sep 17 00:00:00 2001 From: shrey Date: Tue, 3 Nov 2026 10:51:09 -0800 Subject: [PATCH 28/34] fix(spend): read only the newest rows when paging sessions (LIT-8513) (#48886) --- litellm/proxy/spend_tracking/session_page.py | 221 +++++++++ .../spend_management_endpoints.py | 210 +++----- .../spend/test_session_grouped_logs_pages.py | 455 ++++++++++++++++++ .../proxy/spend_tracking/test_session_page.py | 327 +++++++++++++ .../test_spend_management_endpoints.py | 103 ++-- .../test_spend_query_optimization.py | 40 +- 6 files changed, 1117 insertions(+), 239 deletions(-) create mode 100644 litellm/proxy/spend_tracking/session_page.py create mode 100644 tests/integration/spend/test_session_grouped_logs_pages.py create mode 100644 tests/unit/proxy/spend_tracking/test_session_page.py diff --git a/litellm/proxy/spend_tracking/session_page.py b/litellm/proxy/spend_tracking/session_page.py new file mode 100644 index 00000000000..810afbfa7b7 --- /dev/null +++ b/litellm/proxy/spend_tracking/session_page.py @@ -0,0 +1,221 @@ +"""Which sessions a logs page shows and how many match, read from the newest rows instead of every row. Reads grow +until one settles both, so the result is what grouping every row would give.""" + +import math +from collections.abc import Awaitable, Callable, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, TypeAlias + +from typing_extensions import ReadOnly, TypedDict + +MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" +AGENT_CALL_TYPE_SQL: Final = "'asend_message'" +SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" +SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( + f"(call_type = {AGENT_CALL_TYPE_SQL}) DESC, " + f'CASE WHEN call_type = {AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' + f'call_type IN {MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' +) + + +class _ReadRow(TypedDict): + session_key: ReadOnly[str | None] + api_key: ReadOnly[str | None] + last_activity: ReadOnly[str | None] + settled: ReadOnly[bool | None] + rows_read: ReadOnly[int] + session_count: ReadOnly[int] + + +class _SupportsQueryRaw(Protocol): + async def query_raw(self, query: str, *args: object) -> Sequence[_ReadRow]: ... + + +@dataclass(frozen=True, slots=True) +class Session: + key: str + api_key: str + last_activity: str + + +@dataclass(frozen=True, slots=True) +class SessionPage: + sessions: tuple[Session, ...] + has_more: bool + next_cursor: str | None + total: int + total_is_capped: bool + + +@dataclass(frozen=True, slots=True) +class _Request: + cursor: tuple[str, str, str] | None + offset: int + page_limit: int + sort_desc: bool + status: str | None + cap: int + + def first_read_size(self) -> int | None: + """Every row when oldest first, or when filtered by outcome, which any of a session's rows can decide. Otherwise + the fewest that could settle a page: a session past the cap and an older row.""" + return self.cap + 2 if self.sort_desc and self.status is None else None + + +@dataclass(frozen=True, slots=True) +class _Read: + candidates: tuple[Session, ...] + last_settled: bool + rows_read: int + session_count: int + saw_every_row: bool + + def settles(self, request: _Request) -> bool: + page_settled: Final = ( + request.page_limit <= 0 + or self.saw_every_row + or (len(self.candidates) > request.page_limit and self.last_settled) + ) + return page_settled and (self.saw_every_row or self.session_count > request.cap) + + def next_size(self, request: _Request) -> int: + rows_per_session: Final = self.rows_read / self.session_count + return max(2 * self.rows_read, math.ceil(1.25 * (request.cap + 1) * rows_per_session)) + + +_Reader: TypeAlias = Callable[[int | None], Awaitable[_Read]] + + +def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: + if session_cursor is None or session_cursor.count("|") < 2: + return None + last_activity, _, rest = session_cursor.partition("|") + api_key, _, session_key = rest.partition("|") + if not last_activity or not session_key: + return None + return (last_activity, session_key, api_key) + + +async def read_session_page( + db: _SupportsQueryRaw, + *, + where_clause: str, + params: Sequence[object], + status: str | None, + page: int, + page_size: int, + sort_desc: bool, + session_cursor: str | None, + cap: int, +) -> SessionPage: + """One page of sessions after ``session_cursor``, or after ``(page - 1) * page_size`` sessions, keeping only those + whose outcome is ``status`` when given. ``where_clause`` reads ``params`` as ``$1`` onward.""" + cursor: Final = _parse_session_cursor(session_cursor) + offset: Final = (page - 1) * page_size if cursor is None else 0 + request: Final = _Request(cursor, offset, min(page_size, cap - offset), sort_desc, status, cap) + return await _session_page(_sql_reader(db, where_clause, params, request), request) + + +async def _session_page(read: _Reader, request: _Request) -> SessionPage: + last = await read(request.first_read_size()) # rebind-ok: each read replaces the last until one settles the page + while not last.settles(request): + last = await read(last.next_size(request)) + has_more: Final = len(last.candidates) > request.page_limit + sessions: Final = last.candidates[: request.page_limit] + final: Final = sessions[-1] if has_more and sessions else None + return SessionPage( + sessions=sessions, + has_more=has_more, + next_cursor=f"{final.last_activity}|{final.api_key}|{final.key}" if final else None, + total=min(last.session_count, request.cap), + total_is_capped=last.session_count > request.cap, + ) + + +def _sessions_sql(request: _Request, status_index: int) -> str: + """Each session's last activity, from its rows in ``scanned``, keeping only sessions whose outcome is the status.""" + if request.status is None: + return ( + 'SELECT session_key, api_key, MAX("startTime") AS last_activity FROM scanned GROUP BY session_key, api_key' + ) + return f""" + SELECT session_key, api_key, last_activity + FROM ( + SELECT DISTINCT ON (session_key, api_key) + session_key, + api_key, + status, + MAX("startTime") OVER (PARTITION BY session_key, api_key) AS last_activity + FROM scanned + ORDER BY session_key, api_key, {SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS outcomes + WHERE COALESCE(status, 'success') = ${status_index} + """ + + +def _sql_reader(db: _SupportsQueryRaw, where_clause: str, params: Sequence[object], request: _Request) -> _Reader: + status_index: Final = len(params) + 1 + status_args: Final = () if request.status is None else (request.status,) + cursor_index: Final = status_index + len(status_args) + after_cursor: Final = ( + f"WHERE (last_activity, session_key, api_key) {'<' if request.sort_desc else '>'} " + f"(${cursor_index}::timestamp, ${cursor_index + 1}, ${cursor_index + 2})" + if request.cursor + else "" + ) + outcome_columns: Final = ', status, call_type, "endTime", request_id' if request.status is not None else "" + direction: Final = "DESC" if request.sort_desc else "ASC" + page_rows: Final = request.page_limit + 1 if request.page_limit > 0 else 0 + skip: Final = f"OFFSET {request.offset}" if page_rows and request.offset else "" + + async def read(size: int | None) -> _Read: + # LEFT JOIN keeps the summary row when no session is on the page. + rows: Final = await db.query_raw( + f""" + WITH scanned AS ( + SELECT {SESSION_KEY_EXPR} AS session_key, api_key, "startTime"{outcome_columns} + FROM "LiteLLM_SpendLogs" + WHERE {where_clause} + {f'ORDER BY "startTime" DESC LIMIT {size}' if size is not None else ""} + ), + sessions AS ({_sessions_sql(request, status_index)}), + summary AS ( + SELECT COUNT(*) AS rows_read, + MIN("startTime") AS oldest, + (SELECT COUNT(*) FROM sessions) AS session_count + FROM scanned + ) + SELECT page.session_key, + page.api_key, + page.last_activity::text AS last_activity, + page.last_activity > summary.oldest AS settled, + summary.rows_read, + summary.session_count + FROM summary + LEFT JOIN LATERAL ( + SELECT session_key, api_key, last_activity + FROM sessions + {after_cursor} + ORDER BY last_activity {direction}, session_key {direction}, api_key {direction} + LIMIT {page_rows} {skip} + ) AS page ON TRUE + ORDER BY page.last_activity {direction}, page.session_key {direction}, page.api_key {direction} + """, + *params, + *status_args, + *(request.cursor or ()), + ) + rows_read: Final = int(rows[0]["rows_read"]) + return _Read( + candidates=tuple( + Session(str(row["session_key"]), str(row["api_key"]), str(row["last_activity"])) + for row in rows + if row["session_key"] is not None + ), + last_settled=bool(rows[-1]["settled"]), + rows_read=rows_read, + session_count=int(rows[0]["session_count"]), + saw_every_row=size is None or rows_read < size, + ) + + return read diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py index 89200792c0b..fcd719b24b9 100644 --- a/litellm/proxy/spend_tracking/spend_management_endpoints.py +++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py @@ -52,6 +52,13 @@ from litellm.proxy.auth.authorization_dependencies import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.openai_error_payload import error_status_code, openai_error_param, openai_error_type from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup +from litellm.proxy.spend_tracking.session_page import ( + AGENT_CALL_TYPE_SQL, + MCP_CALL_TYPES_SQL, + SESSION_KEY_EXPR, + SESSION_REPRESENTATIVE_ORDER_SQL, + read_session_page, +) from litellm.proxy.spend_tracking.spend_capture_rate import ( ProviderBillingCredentialMissing, ProviderBillingRequestFailed, @@ -87,23 +94,15 @@ router: Final = APIRouter() SPEND_LOGS_PAGINATION_COUNT_CAP: Final = 10000 _DECODED_JSON: Final = TypeAdapter(object) -_SESSION_KEY_EXPR: Final = "COALESCE(NULLIF(session_id, ''), request_id)" -_SESSION_GROUP_KEY_SQL: Final = f"{_SESSION_KEY_EXPR}, api_key" -_MCP_CALL_TYPES_SQL: Final = "('call_mcp_tool', 'list_mcp_tools')" -_AGENT_CALL_TYPE_SQL: Final = "'asend_message'" -_SESSION_REPRESENTATIVE_ORDER_SQL: Final = ( - f"(call_type = {_AGENT_CALL_TYPE_SQL}) DESC, " - f'CASE WHEN call_type = {_AGENT_CALL_TYPE_SQL} THEN "endTime" END DESC NULLS LAST, ' - f'call_type IN {_MCP_CALL_TYPES_SQL}, "startTime" DESC, request_id' -) +_SESSION_GROUP_KEY_SQL: Final = f"{SESSION_KEY_EXPR}, api_key" _BATCH_CALL_TYPES_SQL: Final = "('acreate_batch', 'create_batch', 'aretrieve_batch', 'retrieve_batch')" _SPAN_TYPE_SQL_CONDITIONS: Final[Mapping[str, str]] = MappingProxyType( { - "mcp": f"call_type IN {_MCP_CALL_TYPES_SQL}", - "agent": f"call_type = {_AGENT_CALL_TYPE_SQL}", + "mcp": f"call_type IN {MCP_CALL_TYPES_SQL}", + "agent": f"call_type = {AGENT_CALL_TYPE_SQL}", "batch": f"call_type IN {_BATCH_CALL_TYPES_SQL}", "llm": ( - f"(call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} " + f"(call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} " f"AND call_type NOT IN {_BATCH_CALL_TYPES_SQL})" ), } @@ -2948,23 +2947,6 @@ async def ui_view_spend_logs( sql_params.append(json.dumps(used_client_oauth_token)) p += 1 - if status_filter is not None and group_by_session is True and not is_search_lookup: - session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" - sql_conditions.append( - f"""({_SESSION_GROUP_KEY_SQL}) IN ( - SELECT session_key, api_key FROM ( - SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) - {_SESSION_KEY_EXPR} AS session_key, api_key, status - FROM "LiteLLM_SpendLogs" - WHERE {session_filter_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} - ) AS session_outcomes - WHERE COALESCE(status, 'success') = ${p} - )""" - ) - sql_params.append(status_filter) - p += 1 - if ( group_by_session is True and not is_v2 @@ -2977,12 +2959,30 @@ async def ui_view_spend_logs( sql_conditions=sql_conditions, sql_params=sql_params, next_param_index=p, + status_filter=status_filter, page=page, page_size=page_size, sort_desc=order_direction != "asc", session_cursor=session_cursor, ) + if status_filter is not None and group_by_session is True and not is_search_lookup: + session_filter_conditions: Final = " AND ".join(sql_conditions) or "TRUE" + sql_conditions.append( + f"""({_SESSION_GROUP_KEY_SQL}) IN ( + SELECT session_key, api_key FROM ( + SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) + {SESSION_KEY_EXPR} AS session_key, api_key, status + FROM "LiteLLM_SpendLogs" + WHERE {session_filter_conditions} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} + ) AS session_outcomes + WHERE COALESCE(status, 'success') = ${p} + )""" + ) + sql_params.append(status_filter) + p += 1 + # Build the ORDER BY expression. ttft_ms is computed from # completionStartTime - startTime; non-streaming rows (where # completionStartTime is null or equals endTime) yield NULL, so we @@ -3031,7 +3031,7 @@ async def ui_view_spend_logs( {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {joined_conditions} - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives ORDER BY {exact_request_id_first}{_order_expr} {_sql_dir}{_nulls_clause}, request_id LIMIT ${p} OFFSET ${p + 1} @@ -3080,22 +3080,6 @@ async def ui_view_spend_logs( raise handle_exception_on_proxy(e) -class _SessionPageRow(TypedDict): - session_key: ReadOnly[str] - api_key: ReadOnly[str] - last_activity: ReadOnly[str] - - -def _parse_session_cursor(session_cursor: str | None) -> tuple[str, str, str] | None: - if session_cursor is None or session_cursor.count("|") < 2: - return None - last_activity, _, rest = session_cursor.partition("|") - api_key, _, session_key = rest.partition("|") - if not last_activity or not session_key: - return None - return (last_activity, session_key, api_key) - - async def _fetch_session_representatives( prisma_client: "PrismaClient", where_clause: str, @@ -3104,16 +3088,18 @@ async def _fetch_session_representatives( session_keys: Sequence[tuple[str, str]], ) -> list[dict[str, object]]: # mutable-ok: _build_ui_spend_logs_response writes session counts onto each row """Fetch the final agent outcome, or newest non-MCP row, of each ``(session_key, api_key)`` session, in ``session_keys`` order.""" + # Lets Postgres use the session_id and request_id indexes. rep_query: Final = f""" SELECT * FROM ( SELECT DISTINCT ON ({_SESSION_GROUP_KEY_SQL}) {_SPEND_LOG_LIST_COLUMNS} FROM "LiteLLM_SpendLogs" WHERE {where_clause} + AND (session_id = ANY(${next_param_index}::text[]) OR request_id = ANY(${next_param_index}::text[])) AND ({_SESSION_GROUP_KEY_SQL}) IN ( SELECT * FROM unnest(${next_param_index}::text[], ${next_param_index + 1}::text[]) ) - ORDER BY {_SESSION_GROUP_KEY_SQL}, {_SESSION_REPRESENTATIVE_ORDER_SQL} + ORDER BY {_SESSION_GROUP_KEY_SQL}, {SESSION_REPRESENTATIVE_ORDER_SQL} ) AS session_representatives """ rep_rows: Final[Sequence[dict[str, object]]] = await _query_raw( # mutable-ok: rows are enriched in place @@ -3129,117 +3115,32 @@ async def _fetch_session_representatives( return [rep_by_key[key] for key in session_keys if key in rep_by_key] -async def _count_grouped_sessions( - prisma_client: "PrismaClient", - where_clause: str, - sql_params: Sequence[object], - next_param_index: int, -) -> tuple[int, bool]: - """Count the sessions matching the filter, returning ``(total, total_is_capped)`` bounded by the count cap.""" - count_query: Final = f""" - SELECT COUNT(*) AS total_count - FROM ( - SELECT 1 - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - LIMIT ${next_param_index} - ) AS bounded_sessions - """ - count_rows: Final[Sequence[_SpendLogsCountRow]] = await _query_raw( - prisma_client, count_query, *sql_params, SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - ) - raw_total: Final = int(count_rows[0]["total_count"]) if count_rows else 0 - return ( - (SPEND_LOGS_PAGINATION_COUNT_CAP, True) if raw_total > SPEND_LOGS_PAGINATION_COUNT_CAP else (raw_total, False) - ) - - async def _ui_session_grouped_spend_logs( prisma_client: "PrismaClient", sql_conditions: Sequence[str], sql_params: Sequence[object], next_param_index: int, + status_filter: str | None, page: int, page_size: int, sort_desc: bool, session_cursor: str | None, ) -> Mapping[str, object]: - """ - One row per session, keyset-paginated by session last activity. - - Sessions are derived on the fly from ``LiteLLM_SpendLogs`` (no extra - table): rows sharing a ``session_id`` and ``api_key`` form a session, rows - without a session id are singletons keyed by ``request_id``. A page is the - next ``page_size`` sessions ordered by ``(MAX(startTime), session_key, - api_key)``, resumed from the ``session_cursor`` keyset - ``'||'`` instead of an OFFSET, so - page depth does not degrade the query plan. A request for ``page > 1`` - without a cursor (the UI jumping straight to the last page, or back to a - page it never walked through) falls back to ``OFFSET (page - 1) * - page_size``, trimmed to the end of the ``SPEND_LOGS_PAGINATION_COUNT_CAP`` - window the capped ``total`` promises, so a page never runs past that total - and one starting at or past it returns no rows without a query. Each session is represented - by its final agent outcome (or newest non-MCP row), enriched by ``_build_ui_spend_logs_response`` - exactly like the flat listing, and the response carries - ``next_session_cursor`` / ``has_more`` while ``total`` counts sessions - (capped like the flat total). A page that runs out of sessions while still - holding some is itself the end of the list, so its ``total`` is - ``offset + len(page)`` and the grouped count query is skipped; a page that - starts past the end says nothing about the total, so that one is counted. - """ + """One row per session, keyset-paginated by last activity. ``read_session_page`` picks and counts the sessions, + and each is shown by its final agent outcome or newest non-MCP row.""" where_clause: Final = " AND ".join(sql_conditions) if sql_conditions else "TRUE" - cmp_op: Final = "<" if sort_desc else ">" - direction: Final = "DESC" if sort_desc else "ASC" - - cursor: Final = _parse_session_cursor(session_cursor) - having_clause: Final = ( - f'HAVING (MAX("startTime"), {_SESSION_GROUP_KEY_SQL}) {cmp_op} ' - f"(${next_param_index}::timestamp, ${next_param_index + 1}, ${next_param_index + 2})" - if cursor - else "" + session_page: Final = await read_session_page( + prisma_client.db, # pyright: ignore[reportArgumentType] # PrismaWrapper forwards raw queries via __getattr__ + where_clause=where_clause, + params=sql_params, + status=status_filter, + page=page, + page_size=page_size, + sort_desc=sort_desc, + session_cursor=session_cursor, + cap=SPEND_LOGS_PAGINATION_COUNT_CAP, ) - cursor_params: Final[tuple[object, ...]] = cursor if cursor else () - limit_index: Final = next_param_index + len(cursor_params) - offset: Final = (page - 1) * page_size if cursor is None else 0 - page_limit: Final = min(page_size, SPEND_LOGS_PAGINATION_COUNT_CAP - offset) - offset_params: Final[tuple[int, ...]] = (offset,) if offset and page_limit > 0 else () - offset_clause: Final = f"OFFSET ${limit_index + 1}" if offset_params else "" - - page_query: Final = f""" - SELECT {_SESSION_KEY_EXPR} AS session_key, - api_key, - MAX("startTime")::text AS last_activity - FROM "LiteLLM_SpendLogs" - WHERE {where_clause} - GROUP BY {_SESSION_GROUP_KEY_SQL} - {having_clause} - ORDER BY MAX("startTime") {direction}, {_SESSION_KEY_EXPR} {direction}, api_key {direction} - LIMIT ${limit_index} {offset_clause} - """ - page_rows: Final[Sequence[_SessionPageRow]] = ( - () - if page_limit <= 0 - else await _query_raw(prisma_client, page_query, *sql_params, *cursor_params, page_limit + 1, *offset_params) - ) - - has_more: Final = len(page_rows) > page_limit - visible_rows: Final = page_rows[:page_limit] - next_cursor: Final = ( - f"{visible_rows[-1]['last_activity']}|{visible_rows[-1]['api_key']}|{visible_rows[-1]['session_key']}" - if has_more and visible_rows - else None - ) - - page_starts_inside_the_list: Final = offset == 0 or len(page_rows) > 0 - page_ends_the_list: Final = cursor is None and page_limit > 0 and not has_more and page_starts_inside_the_list - total_records, total_is_capped = ( - (offset + len(page_rows), False) - if page_ends_the_list - else await _count_grouped_sessions(prisma_client, where_clause, sql_params, next_param_index) - ) - - session_keys: Final = tuple((row["session_key"], row["api_key"]) for row in visible_rows) + session_keys: Final = tuple((session.key, session.api_key) for session in session_page.sessions) data: Final[list[dict[str, object]]] = ( # mutable-ok: _build_ui_spend_logs_response writes onto each row await _fetch_session_representatives( prisma_client=prisma_client, @@ -3253,18 +3154,17 @@ async def _ui_session_grouped_spend_logs( ) _hydrate_spend_log_metadata(data) - total_pages: Final = (total_records + page_size - 1) // page_size response: Final[Mapping[str, object]] = await _build_ui_spend_logs_response( prisma_client, data, - total_records, + session_page.total, page, page_size, - total_pages, + (session_page.total + page_size - 1) // page_size, enrich_session_counts=True, - total_is_capped=total_is_capped, + total_is_capped=session_page.total_is_capped, ) - return {**response, "next_session_cursor": next_cursor, "has_more": has_more} + return {**response, "next_session_cursor": session_page.next_cursor, "has_more": session_page.has_more} class RequestResponsePayload(NamedTuple): @@ -4720,16 +4620,16 @@ async def _build_ui_spend_logs_response( ) ), 0)::bigint AS session_total_duration_ms, COUNT(*) FILTER ( - WHERE call_type IN {_MCP_CALL_TYPES_SQL} + WHERE call_type IN {MCP_CALL_TYPES_SQL} )::int AS mcp_tool_call_count, COALESCE(SUM(spend) FILTER ( - WHERE call_type IN {_MCP_CALL_TYPES_SQL} + WHERE call_type IN {MCP_CALL_TYPES_SQL} ), 0)::double precision AS mcp_tool_call_spend, COUNT(*) FILTER (WHERE LOWER(cache_hit) = 'true')::int AS session_cache_hit_count, COUNT(*) FILTER ( - WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL} + WHERE call_type NOT IN {MCP_CALL_TYPES_SQL} AND call_type != {AGENT_CALL_TYPE_SQL} )::int AS session_llm_count, - COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count, + COUNT(*) FILTER (WHERE call_type = {AGENT_CALL_TYPE_SQL})::int AS session_agent_count, COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens, COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens, COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens diff --git a/tests/integration/spend/test_session_grouped_logs_pages.py b/tests/integration/spend/test_session_grouped_logs_pages.py new file mode 100644 index 00000000000..19854d60bd3 --- /dev/null +++ b/tests/integration/spend/test_session_grouped_logs_pages.py @@ -0,0 +1,455 @@ +import itertools +import json +import os +import uuid +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import psycopg +import pytest +from prisma import Prisma +from psycopg import sql +from pydantic import BaseModel, ConfigDict, Field + +from litellm.proxy.spend_tracking.session_page import read_session_page +from litellm.proxy.spend_tracking.spend_management_endpoints import _fetch_session_representatives +from tests.integration._support.client import Gateway +from tests.integration._support.database import write_rows + +_URL: Final = "/spend/logs/ui" +_WINDOW_SECONDS: Final = 3600 + + +@dataclass(frozen=True, slots=True) +class _Row: + request: str + session: str | None + api_key: str + second: int + status: str = "success" + model: str = "m-a" + + +_ROWS: Final = ( + _Row("z1", "z", "key-1", -1), + _Row("a1", "a", "key-1", 10), + _Row("a2", "a", "key-1", 50), + _Row("b1", "b", "key-1", 20, model="m-b"), + _Row("b2", "b", "key-1", 60), + _Row("b3", "b", "key-1", 30), + _Row("c1", "c", "key-2", 60), + _Row("d1", "b", "key-2", 40, model="m-b"), + _Row("e1", None, "key-1", 55, model="m-b"), + _Row("f1", "", "key-1", 45), + _Row("g1", "g", "key-3", 5, status="failure"), + _Row("g2", "g", "key-3", 15, status="failure"), + *(_Row(f"h{second - 69}", "h", "key-1", second) for second in range(70, 76)), + _Row("i1", "i", "key-3", 35, status="failure"), + _Row("i2", "i", "key-3", _WINDOW_SECONDS + 1), +) +_NEWEST_FIRST: Final = ( + ("h", "key-1"), + ("c", "key-2"), + ("b", "key-1"), + ("e1", "key-1"), + ("a", "key-1"), + ("f1", "key-1"), + ("b", "key-2"), + ("i", "key-3"), + ("g", "key-3"), +) + + +class _LogRow(BaseModel): + model_config = ConfigDict(frozen=True) + + request_id: str + session_id: str | None + api_key: str + session_total_count: int + + +class _Page(BaseModel): + model_config = ConfigDict(frozen=True) + + data: tuple[_LogRow, ...] + total: int + total_is_capped: bool + has_more: bool + next_session_cursor: str | None + + +@dataclass(frozen=True, slots=True) +class _Seeded: + prefix: str + window: Mapping[str, str] + + def session(self, row: _LogRow) -> tuple[str, str]: + key: Final = row.session_id or row.request_id + assert key.startswith(self.prefix), key + return key.removeprefix(self.prefix), row.api_key + + +@pytest.fixture +def seeded(gateway: Gateway) -> Iterator[_Seeded]: + prefix: Final = f"sess{uuid.uuid4().hex[:12]}-" + start: Final = datetime(1900, 1, 1) + timedelta(days=int(prefix[4:16], 16) % 200000) + rows: Final = [ + { + "request_id": f"{prefix}{row.request}", + "session_id": None if row.session is None else f"{prefix}{row.session}" if row.session else "", + "api_key": row.api_key, + "start_time": (start + timedelta(seconds=row.second)).isoformat(), + "status": row.status, + "model": row.model, + } + for row in _ROWS + ] + with gateway.scenario() as scenario: + scenario.cleanups.callback( + write_rows, 'DELETE FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (f"{prefix}%",) + ) + write_rows( + 'INSERT INTO "LiteLLM_SpendLogs" (request_id, call_type, api_key, "startTime", "endTime", model, ' + "session_id, status) SELECT request_id, 'acompletion', api_key, start_time, start_time, model, session_id, " + "status FROM jsonb_to_recordset(%s::jsonb) AS seeded(request_id text, session_id text, api_key text, " + "start_time timestamp, status text, model text)", + (json.dumps(rows),), + ) + yield _Seeded( + prefix, + { + "start_date": f"{start:%Y-%m-%d %H:%M:%S}", + "end_date": f"{start + timedelta(seconds=_WINDOW_SECONDS):%Y-%m-%d %H:%M:%S}", + "group_by_session": "true", + }, + ) + + +def _page(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str]) -> _Page: + return _Page.model_validate(gateway.get(_URL, {**seeded.window, **params})) + + +def _cursor_walk(gateway: Gateway, seeded: _Seeded, params: Mapping[str, str], number: int = 1) -> tuple[_Page, ...]: + page: Final = _page(gateway, seeded, {**params, "page_size": "2", "page": str(number)}) + if page.next_session_cursor is None or number > len(_NEWEST_FIRST): + return (page,) + following: Final = {**params, "session_cursor": page.next_session_cursor} + return (page, *_cursor_walk(gateway, seeded, following, number + 1)) + + +def _rows(pages: tuple[_Page, ...]) -> Iterator[_LogRow]: + return itertools.chain.from_iterable(page.data for page in pages) + + +def _sessions(seeded: _Seeded, pages: tuple[_Page, ...]) -> tuple[tuple[str, str], ...]: + return tuple(seeded.session(row) for row in _rows(pages)) + + +def test_cursor_pages_show_every_session_once_newest_first_with_its_newest_row( + gateway: Gateway, seeded: _Seeded +) -> None: + pages: Final = _cursor_walk(gateway, seeded, {}) + assert _sessions(seeded, pages) == _NEWEST_FIRST + assert [(page.total, page.total_is_capped) for page in pages] == [(9, False)] * 5 + assert [page.has_more for page in pages] == [True, True, True, True, False] + representatives: Final = tuple( + (row.request_id.removeprefix(seeded.prefix), row.session_total_count) for row in _rows(pages) + ) + assert representatives == ( + ("h6", 6), + ("c1", 1), + ("b2", 3), + ("e1", 1), + ("a2", 2), + ("f1", 1), + ("d1", 1), + ("i1", 2), + ("g2", 2), + ) + + +@pytest.mark.parametrize( + "sort_order, expected", + [("desc", _NEWEST_FIRST), ("asc", tuple(reversed(_NEWEST_FIRST)))], + ids=["newest-first", "oldest-first"], +) +def test_numbered_pages_show_the_same_sessions_as_the_cursor_walk( + gateway: Gateway, seeded: _Seeded, sort_order: str, expected: tuple[tuple[str, str], ...] +) -> None: + pages: Final = tuple( + _page(gateway, seeded, {"page_size": "2", "page": str(number), "sort_order": sort_order}) + for number in range(1, 7) + ) + assert _sessions(seeded, pages) == expected + assert [len(page.data) for page in pages] == [2, 2, 2, 2, 1, 0] + assert [page.has_more for page in pages] == [True, True, True, True, False, False] + assert {page.total for page in pages} == {len(expected)} + + +def test_failure_filter_keeps_only_sessions_whose_outcome_failed(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, {"status_filter": "failure"}) + assert _sessions(seeded, pages) == (("i", "key-3"), ("g", "key-3")) + assert pages[0].total == 2 + + +def test_model_filter_ranks_each_session_by_its_matching_rows_only(gateway: Gateway, seeded: _Seeded) -> None: + pages: Final = _cursor_walk(gateway, seeded, {"model": "m-b"}) + assert _sessions(seeded, pages) == (("e1", "key-1"), ("b", "key-2"), ("b", "key-1")) + assert tuple(row.request_id.removeprefix(seeded.prefix) for row in _rows(pages)) == ("e1", "d1", "b1") + assert pages[0].total == 3 + + +_COPIED_ROWS: Final = 20_000 +_COPY_START: Final = datetime(2000, 1, 1, tzinfo=UTC) +_COPY_WINDOW: Final = ( + "\"startTime\" >= ($1::timestamptz AT TIME ZONE 'UTC') AND \"startTime\" <= ($2::timestamptz AT TIME ZONE 'UTC')" +) +_COPY_WINDOW_PARAMS: Final = (_COPY_START, _COPY_START + timedelta(days=1)) + + +def _generated_sessions(rows_per_session: int) -> tuple[sql.SQL, tuple[object, ...]]: + """`_COPIED_ROWS` rows a second apart in sessions of `rows_per_session` rows, a one-row session keyed by request_id.""" + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) ' + "SELECT 'r' || n, 'acompletion', 'key-1', moment, moment, " + "CASE WHEN %s > 1 THEN 's' || (n - 1) / %s END FROM generate_series(1, %s) n, " + "LATERAL (SELECT %s::timestamp + n * INTERVAL '1 second' AS moment) seconds" + ), (rows_per_session, rows_per_session, _COPIED_ROWS, _COPY_START.replace(tzinfo=None)) + + +def _sessions_tied_at_the_newest_moment() -> tuple[sql.SQL, tuple[object, ...]]: + """`_generated_sessions(1)` plus thirty one-row sessions sharing a newer moment, inserted and keyed against session + order so a read cut inside the tie misses the sessions that rank first.""" + start: Final = _COPY_START.replace(tzinfo=None) + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id) ' + "SELECT request_id, 'acompletion', 'key-1', moment, moment, session_id FROM (" + "SELECT n AS ordinal, 'r' || n AS request_id, %s::timestamp + n * INTERVAL '1 second' AS moment, " + "NULL AS session_id FROM generate_series(1, %s) n UNION ALL " + "SELECT %s + 30 - n, 't' || lpad((29 - n)::text, 2, '0'), %s::timestamp + %s * INTERVAL '1 second', " + "'s' || lpad(n::text, 2, '0') FROM generate_series(0, 29) n) seeded ORDER BY ordinal" + ), (start, _COPIED_ROWS, _COPIED_ROWS, start, _COPIED_ROWS + 1) + + +@dataclass(frozen=True, slots=True) +class _Call: + session: str + call_type: str + status: str | None + second: int + + +def _calls(calls: tuple[_Call, ...]) -> tuple[sql.SQL, tuple[object, ...]]: + """One row per call, `second` seconds into the window, ending a second later.""" + seeded: Final = tuple( + { + "request_id": f"{call.session}-{call.second}", + "call_type": call.call_type, + "session_id": call.session, + "status": call.status, + "start_time": (_COPY_START + timedelta(seconds=call.second)).replace(tzinfo=None).isoformat(), + } + for call in calls + ) + return sql.SQL( + 'INSERT INTO {table} (request_id, call_type, api_key, "startTime", "endTime", session_id, status) ' + "SELECT request_id, call_type, 'key-1', start_time, start_time + INTERVAL '1 second', session_id, status " + "FROM jsonb_to_recordset(%s::jsonb) AS calls(request_id text, call_type text, session_id text, status text, " + "start_time timestamp)" + ), (json.dumps(seeded),) + + +@asynccontextmanager +async def _spend_logs_copy(insert: sql.SQL, args: tuple[object, ...]) -> AsyncIterator[Prisma]: + """A private schema holding a copy of the migrated `LiteLLM_SpendLogs` with its indexes, seeded by `insert`.""" + schema: Final = f"integration_{uuid.uuid4().hex}" + url: Final = os.environ["DATABASE_URL"] + table: Final = sql.Identifier(schema, "LiteLLM_SpendLogs") + with psycopg.connect(url, autocommit=True) as setup: + setup.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + try: + setup.execute(sql.SQL('CREATE TABLE {} (LIKE "LiteLLM_SpendLogs" INCLUDING ALL)').format(table)) + setup.execute(insert.format(table=table), args) + setup.execute(sql.SQL("ANALYZE {}").format(table)) + parsed: Final = urlsplit(url) + query: Final = urlencode({**dict(parse_qsl(parsed.query)), "schema": schema}) + database: Final = Prisma(datasource={"url": urlunsplit(parsed._replace(query=query))}) + await database.connect() + try: + yield database + finally: + await database.disconnect() + finally: + setup.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema))) + + +class _PlanNode(BaseModel): + model_config = ConfigDict(frozen=True) + + relation_name: str | None = Field(default=None, alias="Relation Name") + actual_rows: float = Field(alias="Actual Rows") + rows_removed_by_filter: float = Field(default=0, alias="Rows Removed by Filter") + actual_loops: int = Field(alias="Actual Loops") + plans: tuple["_PlanNode", ...] = Field(default=(), alias="Plans") + + def spend_log_rows_scanned(self) -> int: + own: Final = ( + (self.actual_rows + self.rows_removed_by_filter) * self.actual_loops + if self.relation_name == "LiteLLM_SpendLogs" + else 0 + ) + return round(own) + sum(child.spend_log_rows_scanned() for child in self.plans) + + +class _Explained(BaseModel): + model_config = ConfigDict(frozen=True) + + plan: _PlanNode = Field(alias="Plan") + + +class _ExplainedReads: + """Runs each read through Prisma, first under EXPLAIN ANALYZE to count the `LiteLLM_SpendLogs` rows it scans.""" + + def __init__(self, database: Prisma) -> None: + self.database: Final = database + self.rows_scanned: tuple[int, ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + explained: Final = await self.database.query_raw(f"EXPLAIN (ANALYZE, FORMAT JSON) {query}", *args) + plan: Final = _Explained.model_validate(explained[0]["QUERY PLAN"][0]).plan + self.rows_scanned = (*self.rows_scanned, plan.spend_log_rows_scanned()) + return await self.database.query_raw(query, *args) + + +@pytest.mark.parametrize( + "rows_per_session, page, newest_session, rows_scanned_per_read, total", + [ + pytest.param(1, 1, ("r", 20_000), (102,), (100, True), id="first-page"), + pytest.param(1, 3, ("r", 19_980), (102,), (100, True), id="third-page"), + pytest.param(10, 1, ("s", 1_999), (102, 1_171), (100, True), id="second-read-sized-by-the-first-reads-density"), + pytest.param(100, 1, ("s", 199), (102, 6_439, 12_878), (100, True), id="third-read-at-least-doubled"), + pytest.param(1_000, 1, ("s", 19), (102, 12_878, 20_000), (20, False), id="last-read-runs-out-of-rows"), + ], +) +async def test_a_newest_first_page_scans_only_the_rows_its_reads_ask_for( + rows_per_session: int, + page: int, + newest_session: tuple[str, int], + rows_scanned_per_read: tuple[int, ...], + total: tuple[int, bool], +) -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session)) as database: + reads: Final = _ExplainedReads(database) + session_page: Final = await read_session_page( + reads, + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=page, + page_size=10, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + prefix, newest = newest_session + assert tuple(session.key for session in session_page.sessions) == tuple( + f"{prefix}{newest - n}" for n in range(10) + ) + assert (session_page.total, session_page.total_is_capped) == total + assert reads.rows_scanned == rows_scanned_per_read + + +async def test_a_page_cut_inside_a_tie_reads_on_until_the_tie_is_whole() -> None: + async with _spend_logs_copy(*_sessions_tied_at_the_newest_moment()) as database: + reads: Final = _ExplainedReads(database) + session_page: Final = await read_session_page( + reads, + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=1, + page_size=5, + status=None, + sort_desc=True, + session_cursor=None, + cap=10, + ) + assert tuple(session.key for session in session_page.sessions) == ("s29", "s28", "s27", "s26", "s25") + assert reads.rows_scanned == (12, 24, 48) + + +@dataclass(frozen=True, slots=True) +class _ExplainedPrisma: + db: _ExplainedReads + + +async def test_a_pages_representatives_are_found_by_session_not_by_scanning_the_window() -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session=10)) as database: + reads: Final = _ExplainedReads(database) + representatives: Final = await _fetch_session_representatives( + prisma_client=_ExplainedPrisma(reads), # pyright: ignore[reportArgumentType] # only .db is used + where_clause=_COPY_WINDOW, + sql_params=_COPY_WINDOW_PARAMS, + next_param_index=len(_COPY_WINDOW_PARAMS) + 1, + session_keys=(("s1999", "key-1"), ("s1998", "key-1")), + ) + assert tuple(row["request_id"] for row in representatives) == ("r20000", "r19990") + assert reads.rows_scanned == (20,), "only the two sessions' rows, not the window's 20,000" + + +async def test_a_page_number_far_past_the_cap_shows_no_session() -> None: + async with _spend_logs_copy(*_generated_sessions(rows_per_session=1)) as database: + session_page: Final = await read_session_page( + _ExplainedReads(database), + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + page=9_300_000_000_000_000, + page_size=1000, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert (session_page.sessions, session_page.total, session_page.total_is_capped) == ((), 100, True) + + +_OUTCOME_CALLS: Final = ( + _Call("agent", "asend_message", "failure", 10), + _Call("agent", "acompletion", "success", 20), + _Call("mcp", "acompletion", "failure", 5), + _Call("mcp", "call_mcp_tool", "success", 30), + _Call("unknown", "acompletion", None, 40), + _Call("failed", "acompletion", "success", 1), + _Call("failed", "acompletion", "failure", 50), + _Call("recovered", "acompletion", "failure", 2), + _Call("recovered", "acompletion", "success", 45), +) + + +@pytest.mark.parametrize( + "status, expected", + [("failure", ("failed", "mcp", "agent")), ("success", ("recovered", "unknown"))], +) +async def test_a_status_filter_keeps_sessions_whose_representative_call_has_that_status( + status: str, expected: tuple[str, ...] +) -> None: + """An agent message stands for its session, then the newest call that is not an MCP tool call, and a missing status + counts as success. The page stays ordered by each session's newest call.""" + async with _spend_logs_copy(*_calls(_OUTCOME_CALLS)) as database: + session_page: Final = await read_session_page( + _ExplainedReads(database), + where_clause=_COPY_WINDOW, + params=_COPY_WINDOW_PARAMS, + status=status, + page=1, + page_size=10, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert tuple(session.key for session in session_page.sessions) == expected + assert (session_page.total, session_page.total_is_capped) == (len(expected), False) diff --git a/tests/unit/proxy/spend_tracking/test_session_page.py b/tests/unit/proxy/spend_tracking/test_session_page.py new file mode 100644 index 00000000000..a86990bddbb --- /dev/null +++ b/tests/unit/proxy/spend_tracking/test_session_page.py @@ -0,0 +1,327 @@ +import itertools +import random +import re +from collections.abc import Sequence +from typing import Final, TypeAlias + +import pytest + +from litellm.proxy.spend_tracking.session_page import ( + Session, + _parse_session_cursor, + _Read, + _ReadRow, + _Request, + _session_page, + read_session_page, +) + +_Row: TypeAlias = tuple[int, str, str] # (startTime, session key, api_key) +_Ranked: TypeAlias = tuple[int, str, str] # (last activity, session key, api_key) + + +def _session_of(row: _Row) -> tuple[str, str]: + return row[1], row[2] + + +def _ranked(rows: Sequence[_Row], request: _Request) -> tuple[_Ranked, ...]: + """The sessions in ``rows`` in page order, after the request's cursor or offset.""" + by_session: Final = itertools.groupby(sorted(rows, key=_session_of), key=_session_of) + ranked: Final = tuple( + sorted( + ((max(start for start, _, _ in group), key, api_key) for (key, api_key), group in by_session), + reverse=request.sort_desc, + ) + ) + if request.cursor is None: + return ranked[request.offset :] + cursor: Final = (int(request.cursor[0]), request.cursor[1], request.cursor[2]) + return tuple(session for session in ranked if (session < cursor if request.sort_desc else session > cursor)) + + +class _Reads: + """Reads like the page query, whose LIMIT cuts tied rows lowest key first, the opposite of page order. It fails a + read no larger than the one before, since only growing reads end the loop.""" + + def __init__(self, rows: Sequence[_Row], request: _Request, every_row: bool = False) -> None: + self.newest_first: Final = tuple(sorted(rows, key=lambda row: (-row[0], row[1], row[2]))) + self.request: Final = request + self.every_row: Final = every_row + self.budgets: tuple[int | None, ...] = () + + async def __call__(self, budget: int | None) -> _Read: + assert not self.budgets or (self.budgets[-1] is not None and (budget is None or budget > self.budgets[-1])), ( + self.budgets, + budget, + ) + self.budgets = (*self.budgets, budget) + taken: Final = self.newest_first if budget is None or self.every_row else self.newest_first[:budget] + page: Final = _ranked(taken, self.request)[: self.request.page_limit + 1] if self.request.page_limit > 0 else () + return _Read( + candidates=tuple(Session(key, api_key, f"{at:06d}") for at, key, api_key in page), + last_settled=bool(page) and page[-1][0] > taken[-1][0], + rows_read=len(taken), + session_count=len(frozenset(_session_of(row) for row in taken)), + saw_every_row=budget is None or self.every_row or len(taken) < budget, + ) + + +def _session_rows(rng: random.Random, index: int, clock: int) -> tuple[_Row, ...]: + start, length = rng.randrange(0, clock), rng.choice((1, 1, 2, 3, 8, 40)) + return tuple((start + rng.randrange(0, 30), f"s{index:03d}", rng.choice(("k1", "k2"))) for _ in range(length)) + + +def _sessions(rng: random.Random) -> tuple[_Row, ...]: + """Sessions of one to forty rows over a short clock, so many rows share a time.""" + clock: Final = rng.choice((3, 40, 400)) + return tuple( + itertools.chain.from_iterable(_session_rows(rng, index, clock) for index in range(rng.randrange(1, 120))) + ) + + +def _random_cursor(rng: random.Random, ranked: Sequence[_Ranked]) -> tuple[str, str, str]: + at, key, api_key = ranked[rng.randrange(len(ranked))] + return f"{at:06d}", key, api_key + + +def _request(rng: random.Random, rows: Sequence[_Row]) -> _Request: + cap, page_size, sort_desc = rng.randrange(2, 60), rng.randrange(1, 12), rng.random() < 0.8 + mode: Final = rng.choice(("first", "offset", "cursor")) + full: Final = _ranked(rows, _Request(None, 0, 0, sort_desc, None, cap)) + cursor: Final = _random_cursor(rng, full) if mode == "cursor" and full else None + offset: Final = (rng.randrange(1, 2 * cap // page_size + 3) - 1) * page_size if mode == "offset" else 0 + return _Request(cursor, offset, min(page_size, cap - offset), sort_desc, None, cap) + + +@pytest.mark.parametrize("seed", range(60)) +@pytest.mark.asyncio +async def test_a_page_read_from_the_newest_rows_matches_grouping_every_row(seed: int) -> None: + rng: Final = random.Random(seed) + rows: Final = _sessions(rng) + for request in tuple(_request(rng, rows) for _ in range(12)): + grouped = await _session_page(_Reads(rows, request, every_row=True), request) + assert await _session_page(_Reads(rows, request), request) == grouped, request + + +@pytest.mark.parametrize( + "rows_per_session, expected_budgets", + [ + pytest.param(1, (102,), id="one-read-holds-the-cap"), + pytest.param(2, (102, 253), id="sized-by-the-first-reads-density"), + pytest.param(50, (102, 4293, 8586), id="doubles-when-the-density-misleads"), + pytest.param(500, (102, 12878, 62533), id="re-sized-by-each-reads-density"), + ], +) +@pytest.mark.asyncio +async def test_reads_grow_by_the_last_reads_rows_per_session_until_one_settles_the_page( + rows_per_session: int, expected_budgets: tuple[int, ...] +) -> None: + """150 sessions a row per second apart. A cap of 100 starts at 100 + 2 rows.""" + rows: Final = tuple((index, f"s{index // rows_per_session:05d}", "k") for index in range(150 * rows_per_session)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100) + reads: Final = _Reads(rows, request) + await _session_page(reads, request) + assert reads.budgets == expected_budgets + + +@pytest.mark.asyncio +async def test_a_session_tied_with_the_reads_oldest_row_is_not_trusted() -> None: + """Thirty one-row sessions share one time and the first read takes 12 of them. The page's sixth session sits at the + read's oldest time, where the LIMIT may have cut a session that ranks before it, so the page reads on.""" + rows: Final = tuple((0, f"s{index:02d}", "k") for index in range(30)) + request: Final = _Request(cursor=None, offset=0, page_limit=5, sort_desc=True, status=None, cap=10) + reads: Final = _Reads(rows, request) + page: Final = await _session_page(reads, request) + assert tuple(session.key for session in page.sessions) == ("s29", "s28", "s27", "s26", "s25") + assert reads.budgets == (12, 24, 48) + + +@pytest.mark.parametrize("session_count, expected_total", [(100, (100, False)), (101, (100, True))]) +@pytest.mark.asyncio +async def test_the_total_is_capped_only_past_the_cap(session_count: int, expected_total: tuple[int, bool]) -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(session_count)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=True, status=None, cap=100) + page: Final = await _session_page(_Reads(rows, request), request) + assert (page.total, page.total_is_capped) == expected_total + + +@pytest.mark.asyncio +async def test_a_page_number_near_the_bigint_limit_reads_once() -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500)) + offset: Final = (9_300_000_000_000_000 - 1) * 1000 + request: Final = _Request( + cursor=None, offset=offset, page_limit=min(1000, 100 - offset), sort_desc=True, status=None, cap=100 + ) + reads: Final = _Reads(rows, request) + page: Final = await _session_page(reads, request) + assert reads.budgets == (102,) + assert (page.sessions, page.total, page.total_is_capped) == ((), 100, True) + + +@pytest.mark.parametrize( + "sort_desc, status", + [pytest.param(False, None, id="oldest-first"), pytest.param(True, "failure", id="filtered-by-outcome")], +) +@pytest.mark.asyncio +async def test_a_page_that_needs_every_row_reads_them_once(sort_desc: bool, status: str | None) -> None: + rows: Final = tuple((index, f"s{index:05d}", "k") for index in range(500)) + request: Final = _Request(cursor=None, offset=0, page_limit=10, sort_desc=sort_desc, status=status, cap=100) + reads: Final = _Reads(rows, request) + await _session_page(reads, request) + assert reads.budgets == (None,) + + +def test_parse_session_cursor() -> None: + assert _parse_session_cursor(None) is None + assert _parse_session_cursor("no-delimiter") is None + assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None + assert _parse_session_cursor("|hashed-key|sess-1") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None + assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( + "2026-08-29 10:00:00", + "sess-1", + "hashed-key", + ) + assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( + "2026-08-29 10:00:00.123", + "sess|with|pipes", + "hashed-key", + ) + + +def _bound(sql: str, args: tuple[object, ...], pattern: str) -> tuple[object, ...]: + """The values bound to the ``$N`` placeholders that ``pattern``'s groups capture in ``sql``.""" + found: Final = re.search(pattern, sql) + assert found, f"{pattern!r} not found in:\n{sql}" + assert all(1 <= int(index) <= len(args) for index in found.groups()), f"{found.groups()} outside $1..${len(args)}" + return tuple(args[int(index) - 1] for index in found.groups()) + + +class _TableDb: + """A table of ``rows`` matching rows in ``sessions`` sessions, whose reads return no page row.""" + + def __init__(self, rows: int, sessions: int) -> None: + self.rows: Final = rows + self.sessions: Final = sessions + self.calls: tuple[tuple[str, tuple[object, ...]], ...] = () + + async def query_raw(self, query: str, *args: object) -> tuple[_ReadRow, ...]: + self.calls = (*self.calls, (query, args)) + budget: Final = re.search(r"DESC LIMIT (\d+)", query) + rows_read: Final = min(int(budget.group(1)), self.rows) if budget else self.rows + return ( + _ReadRow( + session_key=None, + api_key=None, + last_activity=None, + settled=None, + rows_read=rows_read, + session_count=self.sessions, + ), + ) + + +def _page_bounds(sql: str) -> tuple[int, int]: + """The LIMIT and OFFSET of the read's sessions.""" + found: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s*\) AS page", sql) + assert found, sql + return int(found.group(1)), int(found.group(2) or 0) + + +def _read_size(sql: str) -> int | None: + found: Final = re.search(r"DESC LIMIT (\d+)", sql) + return int(found.group(1)) if found else None + + +@pytest.mark.asyncio +async def test_a_read_that_fills_its_size_is_not_taken_for_every_row() -> None: + """A read that took every row it asked for may have stopped short, so its 50 sessions are no total: the reads + grow until one takes fewer rows than it asked for.""" + db: Final = _TableDb(rows=9_000, sessions=50) + page: Final = await read_session_page( + db, + where_clause="TRUE", + params=(), + page=1, + page_size=10, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert tuple(_read_size(sql) for sql, _ in db.calls) == (102, 258, 652, 1647, 4159, 10502) + assert (page.total, page.total_is_capped) == (50, False) + + +@pytest.mark.parametrize("page_number", [pytest.param(5, id="at-the-cap"), pytest.param(6, id="past-the-cap")]) +@pytest.mark.asyncio +async def test_a_page_starting_at_or_past_the_cap_reads_once_and_shows_no_session(page_number: int) -> None: + db: Final = _TableDb(rows=1_000, sessions=150) + page: Final = await read_session_page( + db, + where_clause="TRUE", + params=(), + page=page_number, + page_size=25, + status=None, + sort_desc=True, + session_cursor=None, + cap=100, + ) + assert len(db.calls) == 1, "the first read already holds more sessions than the cap" + assert _page_bounds(db.calls[0][0])[0] == 0 + assert (page.total, page.total_is_capped) == (100, True) + + +@pytest.mark.asyncio +async def test_the_read_binds_the_filters_and_the_cursor_to_their_placeholders() -> None: + db: Final = _TableDb(rows=50, sessions=40) + where, params = "model = $1 AND api_key = $2", ("gpt-4o", "key-a") + for page, page_size, session_cursor in ((1, 3, "2026-10-28 09:00:00|key-a|sess|9"), (3, 1, None)): + await read_session_page( + db, + where_clause=where, + params=params, + page=page, + page_size=page_size, + status=None, + sort_desc=True, + session_cursor=session_cursor, + cap=100, + ) + (cursor_sql, cursor_args), (offset_sql, _) = db.calls + for sql, args in db.calls: + assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)), ( + "every argument needs a placeholder and every placeholder an argument" + ) + assert _bound(sql, args, r"model = \$(\d+) AND api_key = \$(\d+)") == params + assert _bound( + cursor_sql, cursor_args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)" + ) == ("2026-10-28 09:00:00", "sess|9", "key-a") + assert _page_bounds(cursor_sql) == (4, 0) + assert _page_bounds(offset_sql) == (2, 2) + + +@pytest.mark.asyncio +async def test_a_status_filtered_read_binds_the_status_between_the_filters_and_the_cursor() -> None: + db: Final = _TableDb(rows=50, sessions=40) + await read_session_page( + db, + where_clause="model = $1", + params=("gpt-4o",), + status="failure", + page=1, + page_size=3, + sort_desc=True, + session_cursor="2026-10-28 09:00:00|key-a|sess-9", + cap=100, + ) + ((sql, args),) = db.calls + assert {int(index) for index in re.findall(r"\$(\d+)", sql)} == set(range(1, len(args) + 1)) + assert _bound(sql, args, r"model = \$(\d+)") == ("gpt-4o",) + assert _bound(sql, args, r"COALESCE\(status, 'success'\) = \$(\d+)") == ("failure",) + assert _bound(sql, args, r"\(last_activity, session_key, api_key\) < \(\$(\d+)::timestamp, \$(\d+), \$(\d+)\)") == ( + "2026-10-28 09:00:00", + "sess-9", + "key-a", + ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 6598b94e60f..f8d0ebb0f77 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5,7 +5,7 @@ import hashlib import json import re import sqlite3 -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import timezone from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -7151,37 +7151,19 @@ def test_scoped_spend_report_range_at_max_allowed(client, monkeypatch): app.dependency_overrides.pop(ps.user_api_key_auth, None) -def test_parse_session_cursor(): - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - _parse_session_cursor, - ) - - assert _parse_session_cursor(None) is None - assert _parse_session_cursor("no-delimiter") is None - assert _parse_session_cursor("2026-08-29 10:00:00|sess-1") is None - assert _parse_session_cursor("|hashed-key|sess-1") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|") is None - assert _parse_session_cursor("2026-08-29 10:00:00|hashed-key|sess-1") == ( - "2026-08-29 10:00:00", - "sess-1", - "hashed-key", - ) - assert _parse_session_cursor("2026-08-29 10:00:00.123|hashed-key|sess|with|pipes") == ( - "2026-08-29 10:00:00.123", - "sess|with|pipes", - "hashed-key", - ) - - SESSION_GROUP_KEY_SQL = "COALESCE(NULLIF(session_id, ''), request_id), api_key" -def _session_grouped_mock_prisma(session_page_rows, session_total, representative_rows): +def _session_grouped_mock_prisma( + session_page_rows: Sequence[Mapping[str, object]], + session_total: int, + representative_rows: Sequence[Mapping[str, object]], +) -> MagicMock: """Mock prisma dispatching the raw queries the keyset grouped path emits.""" - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": session_total}] + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: + if "rows_read" in sql_query: + return _session_read_rows(session_page_rows, session_total) if "DISTINCT ON" in sql_query: return representative_rows if "COALESCE(SUM(spend)" in sql_query: @@ -7199,7 +7181,7 @@ def _session_grouped_mock_prisma(session_page_rows, session_total, representativ "session_models": ["gpt-4o"], } ] - return session_page_rows + raise AssertionError(f"unexpected query:\n{sql_query}") mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7228,23 +7210,33 @@ def _session_page_row(session_key, last_activity): return {"session_key": session_key, "api_key": "hashed-key", "last_activity": last_activity} -def _session_grouped_paginating_prisma(sessions, counted_total=None): - """Mock prisma serving the grouped page query out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" +def _session_read_rows( + page_rows: Sequence[Mapping[str, object]], session_count: int +) -> tuple[Mapping[str, object], ...]: + """The read query's rows for ``page_rows``, as if the read held every matching row.""" + summary: Final = {"rows_read": 0, "session_count": session_count} + if not page_rows: + return ({"session_key": None, "api_key": None, "last_activity": None, "settled": None, **summary},) + return tuple({**row, "settled": True, **summary} for row in page_rows) - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": min(len(sessions) if counted_total is None else counted_total, params[-1])}] + +def _session_grouped_paginating_prisma(sessions: Sequence[tuple[str, str]]) -> MagicMock: + """Mock prisma serving the grouped page read out of ``sessions``, honoring the LIMIT and OFFSET it asks for.""" + + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: if "DISTINCT ON" in sql_query: return [_session_representative_row(f"req-{session_key}", session_key) for session_key in params[-2]] if "COALESCE(SUM(spend)" in sql_query: return [] - bounds = re.search(r"LIMIT \$(\d+)(?: OFFSET \$(\d+))?", sql_query) - limit = params[int(bounds.group(1)) - 1] - offset = params[int(bounds.group(2)) - 1] if bounds.group(2) else 0 - return [ - _session_page_row(session_key, last_activity) - for session_key, last_activity in sessions[offset : offset + limit] - ] + bounds: Final = re.search(r"LIMIT (\d+)(?: OFFSET (\d+))?\s+\) AS page", sql_query) + limit, offset = int(bounds.group(1)), int(bounds.group(2) or 0) + return _session_read_rows( + [ + _session_page_row(session_key, last_activity) + for session_key, last_activity in sessions[offset : offset + limit] + ], + len(sessions), + ) mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -7296,20 +7288,15 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc assert data["data"][1]["session_total_count"] == 1 emitted = [call.args for call in mock_prisma.db.query_raw.await_args_list] + assert len(emitted) == 3, "one read picks the page and counts it, then the representatives and their totals" page_query_sql = emitted[0][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in page_query_sql assert "OFFSET" not in page_query_sql - assert "HAVING" not in page_query_sql - assert emitted[0][-1] == 3, "page query fetches page_size + 1 sessions to detect has_more" + assert "LIMIT 3 " in page_query_sql, "the read fetches page_size + 1 sessions to detect has_more" - count_sql = emitted[1][0] - assert f"GROUP BY {SESSION_GROUP_KEY_SQL}" in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - - rep_call = emitted[2] + rep_call: Final = emitted[1] assert f"DISTINCT ON ({SESSION_GROUP_KEY_SQL})" in rep_call[0] assert ( - f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + f"ORDER BY {SESSION_GROUP_KEY_SQL}, " + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL ) in rep_call[0] assert rep_call[-2] == ["sess-1", "req-solo"] assert rep_call[-1] == ["hashed-key", "hashed-key"] @@ -7319,7 +7306,7 @@ async def test_ui_view_spend_logs_group_by_session_first_page(client, monkeypatc @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypatch): - """A session_cursor becomes a HAVING keyset predicate instead of an OFFSET.""" + """A session_cursor becomes a keyset predicate on the read's sessions instead of an OFFSET.""" page_rows = [_session_page_row("sess-2", "2026-08-29 07:00:00")] reps = [_session_representative_row("req-2", "sess-2")] mock_prisma = _session_grouped_mock_prisma(page_rows, 3, reps) @@ -7352,7 +7339,7 @@ async def test_ui_view_spend_logs_group_by_session_cursor_page(client, monkeypat page_query_call = mock_prisma.db.query_raw.await_args_list[0] page_query_sql = page_query_call.args[0] - assert f'HAVING (MAX("startTime"), {SESSION_GROUP_KEY_SQL}) <' in page_query_sql + assert "WHERE (last_activity, session_key, api_key) <" in page_query_sql assert "OFFSET" not in page_query_sql cursor_index = page_query_call.args.index("2026-08-29 09:00:00") assert page_query_call.args[cursor_index : cursor_index + 3] == ( @@ -7403,9 +7390,9 @@ async def test_ui_view_spend_logs_group_by_session_jumps_to_page_without_cursor( @pytest.mark.asyncio async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(client, monkeypatch): - """A page that runs out of sessions is the end of the list, so the total comes from it and nothing is counted.""" + """A page that runs out of sessions has read every matching row, so that one read gives the total.""" sessions = tuple((f"sess-{index:02d}", f"2026-08-29 10:{59 - index:02d}:00") for index in range(10)) - mock_prisma = _session_grouped_paginating_prisma(sessions, counted_total=999) + mock_prisma: Final = _session_grouped_paginating_prisma(sessions) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) monkeypatch.setattr( "litellm.proxy.spend_tracking.spend_management_endpoints._is_admin_view_safe", @@ -7429,10 +7416,12 @@ async def test_ui_view_spend_logs_group_by_session_short_page_totals_itself(clie ) assert response.status_code == 200, response.text data = response.json() - assert data["total"] == 10, "the count query's 999 would have won if it had been asked" + assert data["total"] == 10 assert data["total_is_capped"] is False assert data["total_pages"] == 1 assert len(data["data"]) == 10 + reads: Final = tuple(call for call in mock_prisma.db.query_raw.await_args_list if "rows_read" in call.args[0]) + assert len(reads) == 1, "a page that ends the list is not read again for its total" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7517,7 +7506,7 @@ async def test_ui_view_spend_logs_group_by_session_page_past_count_cap_is_empty( assert past_cap_data["data"] == [] assert past_cap_data["has_more"] is False assert past_cap_data["total"] == cap - assert mock_prisma.db.query_raw.await_count == 1, "only the bounded count query runs past the capped window" + assert mock_prisma.db.query_raw.await_count == 1, "only the bounded read runs past the capped window" finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) @@ -7820,14 +7809,14 @@ def test_session_representative_uses_completed_agent_outcome(parent_status, chil ) result = connection.execute( "SELECT request_id, status FROM logs ORDER BY " - + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL + " LIMIT 1" ).fetchone() assert result == ("parent", expected) connection.execute("DELETE FROM logs WHERE call_type = 'asend_message'") fallback = connection.execute( "SELECT request_id, status FROM logs ORDER BY " - + spend_management_endpoints._SESSION_REPRESENTATIVE_ORDER_SQL + + spend_management_endpoints.SESSION_REPRESENTATIVE_ORDER_SQL + " LIMIT 1" ).fetchone() assert fallback == ("llm", "success") diff --git a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py index 93fae093340..7a38c1d8457 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py +++ b/tests/unit/proxy/spend_tracking/test_spend_query_optimization.py @@ -6,7 +6,9 @@ GitHub Issue: #17487 """ import datetime +from collections.abc import Mapping, Sequence from datetime import timezone +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -508,20 +510,10 @@ async def test_global_spend_report_team_group_forwards_team_id(monkeypatch): @pytest.mark.asyncio async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): - """ - With group_by_session=true, /spend/logs/ui must page and count SESSIONS, - not raw calls: the page is one representative row per session (DISTINCT - ON the session group key, preferring non-MCP calls, newest first), the - bounded count counts groups, and on the default startTime sort the page is - selected by keyset (no OFFSET) so deep pages do not degrade. Otherwise the - UI collapses a server page of N calls into fewer visible rows while the - footer still claims N (issue #38060). - """ + """With group_by_session=true, /spend/logs/ui pages and counts sessions, not calls, one representative row per + session, so the footer's total matches the rows shown (issue #38060).""" from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.spend_tracking.spend_management_endpoints import ( - SPEND_LOGS_PAGINATION_COUNT_CAP, - ui_view_spend_logs, - ) + from litellm.proxy.spend_tracking.spend_management_endpoints import ui_view_spend_logs group_key = "COALESCE(NULLIF(session_id, ''), request_id), api_key" session_rows = [ @@ -533,12 +525,12 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): {"request_id": "req-2", "api_key": "k", "metadata": "{}", "session_id": None}, ] - async def mock_query_raw(sql_query, *params): - if "COUNT(*) AS total_count" in sql_query: - return [{"total_count": 60}] + async def mock_query_raw(sql_query: str, *params: object) -> Sequence[Mapping[str, object]]: + if "rows_read" in sql_query: + return [{**row, "settled": True, "rows_read": 0, "session_count": 60} for row in session_rows] if "AS session_representatives" in sql_query: return representative_rows - return session_rows + return [] mock_prisma = MagicMock() mock_prisma.db = MagicMock() @@ -565,22 +557,16 @@ async def test_spend_logs_ui_group_by_session_paginates_sessions(monkeypatch): user_api_key_dict=auth, group_by_session=True, session_cursor=None, + status_filter=None, ) emitted = [call[0] for call in mock_prisma.db.query_raw.call_args_list] page_sql = emitted[0][0] - assert f"GROUP BY {group_key}" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" + assert "GROUP BY session_key, api_key" in page_sql, f"page must select sessions, not calls. SQL was:\n{page_sql}" assert "OFFSET" not in page_sql, "the startTime page must be keyset-selected, not offset-selected" - assert emitted[0][-1] == 51, "the page query fetches page_size + 1 sessions to detect has_more" + assert "LIMIT 51 " in page_sql, "the page read fetches page_size + 1 sessions to detect has_more" - count_call = emitted[1] - count_sql = count_call[0] - assert f"GROUP BY {group_key}" in count_sql, f"grouped total must count sessions. SQL was:\n{count_sql}" - assert "COUNT(*) OVER ()" not in count_sql - assert "LIMIT" in count_sql and "FROM (" in count_sql, "the grouped count must stay bounded" - assert count_call[-1] == SPEND_LOGS_PAGINATION_COUNT_CAP + 1 - - rep_sql = emitted[2][0] + rep_sql: Final = emitted[1][0] assert f"DISTINCT ON ({group_key})" in rep_sql, f"page must return one row per session. SQL was:\n{rep_sql}" assert ( f"ORDER BY {group_key}, (call_type = 'asend_message') DESC, " From 083961731c1d2bd59567432556ead3771620f0af Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:32:12 -0800 Subject: [PATCH 29/34] fix(model_prices): consolidate verified provider metadata and pricing (#49361) * fix(databricks): mark audio and video input support on the Gemini 3.5 to 3.8 Flash rows * fix(model_prices): add Vertex AI and Together AI retirement dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): correct perplexity agent api pricing and add gemini-4-argon, nemotron-3.5-super Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): consolidate verified pricing and Aplomb metadata Absorb verified pricing from #49547 and #49548 and Aplomb 1 metadata from #49539. Preserve provider tiers and remove an identical duplicate retirement key. Co-authored-by: Liutasil501 Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(model_prices): sync generated EmpirioLabs endpoint metadata Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): verify Nebius and EmpirioLabs model metadata Verify canonical EmpirioLabs rows from #49695 against published model pages, preserving pricing tiers and excluding unverified variants and specialty billing. Add Nebius models and cached-input prices from the official catalog and correct the Voyage legacy input limit. Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): absorb Bedrock Opus 5 batch prices, Vertex Omni Flash limits and Gemini 2.5 shutdown dates Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): drop the EmpirioLabs rows and endpoint flags from this sync * fix(model_prices): merge main and dedupe the gemini-2.5 deprecation_date keys * fix(model_prices): set supports_video_input on the two new Nebius video models * fix(model_prices): add Azure AI Claude Mythos 5.1 and match Foundry max effort support Co-authored-by: Emerson Gomes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Adam Dalloul <47503782+Adam-Dalloul@users.noreply.github.com> Co-authored-by: Emerson Gomes --- ...odel_prices_and_context_window_backup.json | 340 +++++++++++++----- model_prices_and_context_window.json | 340 +++++++++++++----- ...azure_ai_foundry_catalog_model_metadata.py | 1 + 3 files changed, 507 insertions(+), 174 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 974f1bb0c8b..5a46a0891fc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2135,12 +2135,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2221,12 +2223,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "global.anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2307,12 +2311,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2393,12 +2399,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "eu-west-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2474,12 +2482,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2560,7 +2570,9 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -3949,7 +3961,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -3983,10 +3995,40 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, "deprecation_date": "2027-12-05" }, + "azure_ai/claude-mythos-5-1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": false, + "prompt_cache_min_tokens": 512, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" + }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", "supports_mid_conversation_system": true, @@ -22269,7 +22311,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22287,7 +22329,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22301,7 +22345,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22319,7 +22363,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-6-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22335,7 +22381,7 @@ "mode": "chat", "output_cost_per_token": 3.74997e-06, "output_dbu_cost_per_token": 5.3571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22354,7 +22400,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -22370,7 +22418,7 @@ "mode": "chat", "output_cost_per_token": 1.124998e-05, "output_dbu_cost_per_token": 0.000160714, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22388,7 +22436,9 @@ "input_cost_per_token_priority": 3.37498e-06, "output_cost_per_token_priority": 2.025002e-05, "cache_read_input_token_cost_priority": 3.3747e-07, - "cache_creation_input_token_cost_priority": 3.37498e-06 + "cache_creation_input_token_cost_priority": 3.37498e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -22404,7 +22454,7 @@ "mode": "chat", "output_cost_per_token": 3.12501e-06, "output_dbu_cost_per_token": 4.4643e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22424,7 +22474,9 @@ "input_cost_per_token_priority": 6.7501e-07, "output_cost_per_token_priority": 5.62499e-06, "cache_read_input_token_cost_priority": 6.748e-08, - "cache_creation_input_token_cost_priority": 6.7501e-07 + "cache_creation_input_token_cost_priority": 6.7501e-07, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -24167,7 +24219,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, - "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24241,7 +24293,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 9.5e-07, - "cache_read_input_token_cost": 2.16e-07, + "cache_read_input_token_cost": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -29639,6 +29691,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -30638,6 +30691,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -30744,6 +30798,7 @@ "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { + "deprecation_date": "2027-09-25", "gemini_audio_only_live": true, "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, @@ -32706,13 +32761,13 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview", "supported_endpoints": [ "/v1/chat/completions" ], @@ -32730,7 +32785,8 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -32918,6 +32974,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -44265,7 +44322,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 4e-08 }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_input_tokens": 131072, @@ -44491,7 +44549,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -44499,7 +44557,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-08 }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { "max_input_tokens": 1048576, @@ -44510,29 +44569,32 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/deepseek-ai/DeepSeek-V4-Pro-0813": { "input_cost_per_token": 1.32e-06, "litellm_provider": "nebius", - "max_input_tokens": 979000, + "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08 }, "nebius/deepseek-ai/DeepSeek-V4.1-Flash": { "input_cost_per_token": 3e-07, "litellm_provider": "nebius", - "max_input_tokens": 1048000, + "max_input_tokens": 1048576, "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "nebius/MiniMaxAI/MiniMax-M2.5": { "max_input_tokens": 196608, @@ -44555,7 +44617,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/moonshotai/Kimi-K2.6": { "max_input_tokens": 262144, @@ -44567,7 +44630,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.6e-07 }, "nebius/moonshotai/Kimi-K2.7-Code": { "max_input_tokens": 262144, @@ -44579,10 +44643,11 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.5e-07 }, "nebius/moonshotai/Kimi-K3": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -44591,7 +44656,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-07 }, "nebius/NousResearch/Hermes-4-405B": { "max_input_tokens": 131072, @@ -44602,7 +44668,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2e-07 }, "nebius/NousResearch/Hermes-4-70B": { "max_input_tokens": 131072, @@ -44646,7 +44713,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-08 }, "nebius/nvidia/Nemotron-3-Nano-Omni": { "max_input_tokens": 262144, @@ -44669,7 +44737,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { "max_input_tokens": 1048576, @@ -44680,7 +44749,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/nvidia/Nemotron-3_5-Lightning": { "max_input_tokens": 1048576, @@ -44691,7 +44761,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-08 }, "nebius/openai/gpt-oss-120b": { "max_input_tokens": 131072, @@ -44702,7 +44773,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/openbmb/MiniCPM-V-4_5": { "max_input_tokens": 32000, @@ -44712,7 +44784,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.3e-07 }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { "max_input_tokens": 262144, @@ -44722,7 +44795,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 5e-08 }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { "max_input_tokens": 262144, @@ -44732,7 +44806,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_input_tokens": 128000, @@ -44754,7 +44829,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/Qwen/Qwen3.8-27B": { "input_cost_per_token": 4.5e-07, @@ -44762,9 +44838,10 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1e-07 }, "nebius/zai-org/GLM-5.1": { "max_input_tokens": 202752, @@ -44775,7 +44852,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.2": { "max_input_tokens": 1048576, @@ -44786,21 +44864,23 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/zai-org/GLM-5.3": { "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.3-Flash": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, @@ -44808,8 +44888,9 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash", - "supports_vision": true + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "supports_vision": true, + "cache_read_input_token_cost": 3e-08 }, "nebius/BAAI/bge-en-icl": { "max_input_tokens": 32768, @@ -49813,7 +49894,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -49835,7 +49916,9 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05 }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -52922,6 +53005,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -54150,7 +54234,7 @@ "voyage/voyage-lite-01": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", - "max_input_tokens": 4096, + "max_input_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -64809,7 +64893,7 @@ "supports_vision": false }, "novita/deepseek/deepseek-v4-pro-0813": { - "cache_read_input_token_cost": 1.3200000000000002e-07, + "cache_read_input_token_cost": 4.4e-08, "input_cost_per_token": 1.32e-06, "litellm_provider": "novita", "max_input_tokens": 1048576, @@ -64921,13 +65005,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-venti": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.65e-07, + "input_cost_per_token": 8.25e-07, "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 2.475e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -65030,13 +65114,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-tall": { - "cache_read_input_token_cost": 8e-08, - "input_cost_per_token": 4.5000000000000003e-07, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 2.475e-07, "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.43e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -66461,9 +66545,9 @@ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "max_output_tokens": 1048576, "max_input_tokens": 1048576, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, - "cache_read_input_token_cost": 1.6e-08, + "cache_read_input_token_cost": 1.5e-08, "supports_prompt_caching": true, "litellm_provider": "deepinfra", "mode": "chat", @@ -66842,8 +66926,8 @@ "deepinfra/google/gemma-4-31B-it": { "max_output_tokens": 262144, "max_input_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.8e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, @@ -68554,7 +68638,10 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json" }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68829,7 +68916,10 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -69852,6 +69942,14 @@ "cache_read_input_token_cost": 3e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-argon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -69979,6 +70077,14 @@ "cache_read_input_token_cost": 1.15e-09, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/perplexity/nemotron-3.5-super-120b-a12b": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { "litellm_provider": "perplexity", "mode": "responses", @@ -70104,12 +70210,12 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_input_tokens": 57920, + "max_input_tokens": 131072, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", "supported_endpoints": [ "/v1beta/interactions" ], @@ -74505,12 +74611,14 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -75107,7 +75215,7 @@ "supports_function_calling": true, "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-fable-5-1": { @@ -75155,6 +75263,9 @@ "output_cost_per_token": 5e-05, "cache_read_input_token_cost": 1e-06, "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-6-luna": { @@ -76622,6 +76733,9 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "us-gov.anthropic.claude-haiku-5-5": { @@ -77096,6 +77210,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { + "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, @@ -77504,9 +77619,10 @@ "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 5e-08 }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 7e-09, @@ -81777,5 +81893,55 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-09, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/meta-models/muse-spark-open": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 1.5e-07, + "output_cost_per_token": 4.25e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/MiniMaxAI/MiniMax-M3.1-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/nvidia/Nemotron-3.5-Super-VL": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-08, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 974f1bb0c8b..5a46a0891fc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2135,12 +2135,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2221,12 +2223,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "global.anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.5e-06, + "output_cost_per_token_batches": 1.25e-05 }, "global.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2307,12 +2311,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-east-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-east-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "us.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2393,12 +2399,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/eu-west-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "eu-west-1" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "eu.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2474,12 +2482,14 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-southeast-2/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "au.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -2560,7 +2570,9 @@ "supports_output_config": true, "supports_parallel_tool_use_config": true, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "jp.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -3949,7 +3961,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512 }, "azure_ai/claude-fable-5-1": { @@ -3983,10 +3995,40 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true, + "supports_max_reasoning_effort": false, "prompt_cache_min_tokens": 512, "deprecation_date": "2027-12-05" }, + "azure_ai/claude-mythos-5-1": { + "input_cost_per_token": 1e-05, + "output_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 1.25e-05, + "cache_creation_input_token_cost_above_1hr": 2e-05, + "cache_read_input_token_cost": 2.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_adaptive_thinking": true, + "thinking_always_on": true, + "supports_mid_conversation_system": true, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_forced_tool_use": false, + "supports_function_calling": true, + "supports_native_structured_output": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_sampling_params": false, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "supports_max_reasoning_effort": false, + "prompt_cache_min_tokens": 512, + "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/claude-models" + }, "azure_ai/claude-opus-5": { "deprecation_date": "2027-07-08", "supports_mid_conversation_system": true, @@ -22269,7 +22311,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22287,7 +22329,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-7-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22301,7 +22345,7 @@ }, "mode": "chat", "output_cost_per_token": 3.74997e-06, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22319,7 +22363,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-6-flash": { "cache_creation_input_token_cost": 7.4998e-07, @@ -22335,7 +22381,7 @@ "mode": "chat", "output_cost_per_token": 3.74997e-06, "output_dbu_cost_per_token": 5.3571e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22354,7 +22400,9 @@ "input_cost_per_token_priority": 1.35002e-06, "output_cost_per_token_priority": 6.75003e-06, "cache_read_input_token_cost_priority": 1.3503e-07, - "cache_creation_input_token_cost_priority": 1.35002e-06 + "cache_creation_input_token_cost_priority": 1.35002e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash": { "cache_creation_input_token_cost": 1.87502e-06, @@ -22370,7 +22418,7 @@ "mode": "chat", "output_cost_per_token": 1.124998e-05, "output_dbu_cost_per_token": 0.000160714, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22388,7 +22436,9 @@ "input_cost_per_token_priority": 3.37498e-06, "output_cost_per_token_priority": 2.025002e-05, "cache_read_input_token_cost_priority": 3.3747e-07, - "cache_creation_input_token_cost_priority": 3.37498e-06 + "cache_creation_input_token_cost_priority": 3.37498e-06, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemini-3-5-flash-lite": { "cache_creation_input_token_cost": 3.7499e-07, @@ -22404,7 +22454,7 @@ "mode": "chat", "output_cost_per_token": 3.12501e-06, "output_dbu_cost_per_token": 4.4643e-05, - "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models", "supported_modalities": [ "text", "image", @@ -22424,7 +22474,9 @@ "input_cost_per_token_priority": 6.7501e-07, "output_cost_per_token_priority": 5.62499e-06, "cache_read_input_token_cost_priority": 6.748e-08, - "cache_creation_input_token_cost_priority": 6.7501e-07 + "cache_creation_input_token_cost_priority": 6.7501e-07, + "supports_audio_input": true, + "supports_video_input": true }, "databricks/databricks-gemma-3-12b": { "cache_creation_input_token_cost": 1.5001e-07, @@ -24167,7 +24219,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 5e-07, "output_cost_per_token": 2.15e-06, - "cache_read_input_token_cost": 4e-07, + "cache_read_input_token_cost": 3.5e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -24241,7 +24293,7 @@ "max_output_tokens": 163840, "input_cost_per_token": 2.5e-07, "output_cost_per_token": 9.5e-07, - "cache_read_input_token_cost": 2.16e-07, + "cache_read_input_token_cost": 1.3e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, @@ -29639,6 +29691,7 @@ "web_search_billing_unit": "per_query" }, "gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -30638,6 +30691,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "vertex_ai/gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -30744,6 +30798,7 @@ "supports_native_streaming": true }, "vertex_ai/gemini-3.8-live": { + "deprecation_date": "2027-09-25", "gemini_audio_only_live": true, "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, @@ -32706,13 +32761,13 @@ "input_cost_per_audio_token": 1.5e-06, "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65535, + "max_input_tokens": 131072, + "max_output_tokens": 57920, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview", "supported_endpoints": [ "/v1/chat/completions" ], @@ -32730,7 +32785,8 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_video_input": true, - "supports_vision": true + "supports_vision": true, + "deprecation_date": "2026-11-06" }, "gemini-3.5-flash": { "prompt_cache_min_tokens": 4096, @@ -32918,6 +32974,7 @@ "google_maps_grounding_cost_per_query": 0.014 }, "gemini-3.8-flash": { + "deprecation_date": "2027-09-02", "cache_read_input_token_cost": 7.5e-08, "cache_read_input_token_cost_batches": 3.75e-08, "cache_read_input_token_cost_flex": 3.75e-08, @@ -44265,7 +44322,8 @@ "mode": "chat", "supports_function_calling": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/google%2Fgemma-3-27b-it" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 4e-08 }, "nebius/meta-llama/Llama-3.3-70B-Instruct": { "max_input_tokens": 131072, @@ -44491,7 +44549,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash" }, "nebius/deepseek-ai/DeepSeek-V4-Flash-0731": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.4e-07, "output_cost_per_token": 2.8e-07, @@ -44499,7 +44557,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Flash-0731" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-08 }, "nebius/deepseek-ai/DeepSeek-V4-Pro": { "max_input_tokens": 1048576, @@ -44510,29 +44569,32 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/deepseek-ai/DeepSeek-V4-Pro-0813": { "input_cost_per_token": 1.32e-06, "litellm_provider": "nebius", - "max_input_tokens": 979000, + "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 3.96e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/deepseek-ai%2FDeepSeek-V4-Pro-0813", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 4e-08 }, "nebius/deepseek-ai/DeepSeek-V4.1-Flash": { "input_cost_per_token": 3e-07, "litellm_provider": "nebius", - "max_input_tokens": 1048000, + "max_input_tokens": 1048576, "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://tokenfactory.nebius.com/endpoints?modals=endpoint-details&model-id=deepseek-ai/DeepSeek-V4.1-Flash", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, "supports_reasoning": true, - "supports_vision": true + "supports_vision": true, + "cache_read_input_token_cost": 1e-08 }, "nebius/MiniMaxAI/MiniMax-M2.5": { "max_input_tokens": 196608, @@ -44555,7 +44617,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/MiniMaxAI%2FMiniMax-M3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/moonshotai/Kimi-K2.6": { "max_input_tokens": 262144, @@ -44567,7 +44630,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K2.6" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.6e-07 }, "nebius/moonshotai/Kimi-K2.7-Code": { "max_input_tokens": 262144, @@ -44579,10 +44643,11 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/moonshotai%2FKimi-K2.7-Code" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.5e-07 }, "nebius/moonshotai/Kimi-K3": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, @@ -44591,7 +44656,8 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/moonshotai%2FKimi-K3" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-07 }, "nebius/NousResearch/Hermes-4-405B": { "max_input_tokens": 131072, @@ -44602,7 +44668,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/NousResearch%2FHermes-4-405B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2e-07 }, "nebius/NousResearch/Hermes-4-70B": { "max_input_tokens": 131072, @@ -44646,7 +44713,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNVIDIA-Nemotron-3-Nano-30B-A3B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-08 }, "nebius/nvidia/Nemotron-3-Nano-Omni": { "max_input_tokens": 262144, @@ -44669,7 +44737,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2Fnemotron-3-super-120b-a12b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 6e-08 }, "nebius/nvidia/Nemotron-3-Ultra-550b-a55b": { "max_input_tokens": 1048576, @@ -44680,7 +44749,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3-Ultra-550b-a55b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/nvidia/Nemotron-3_5-Lightning": { "max_input_tokens": 1048576, @@ -44691,7 +44761,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/nvidia%2FNemotron-3_5-Lightning" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 3e-08 }, "nebius/openai/gpt-oss-120b": { "max_input_tokens": 131072, @@ -44702,7 +44773,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/openai%2Fgpt-oss-120b" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/openbmb/MiniCPM-V-4_5": { "max_input_tokens": 32000, @@ -44712,7 +44784,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_vision": true, - "source": "https://tokenfactory.nebius.com/models/catalog/image2text/openbmb%2FMiniCPM-V-4_5" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.3e-07 }, "nebius/Qwen/Qwen3-235B-A22B-Instruct-2507": { "max_input_tokens": 262144, @@ -44722,7 +44795,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-235B-A22B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 5e-08 }, "nebius/Qwen/Qwen3-30B-A3B-Instruct-2507": { "max_input_tokens": 262144, @@ -44732,7 +44806,8 @@ "litellm_provider": "nebius", "mode": "chat", "supports_function_calling": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3-30B-A3B-Instruct-2507" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1e-07 }, "nebius/Qwen/Qwen3-Next-80B-A3B-Thinking": { "max_input_tokens": 128000, @@ -44754,7 +44829,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.5-397B-A17B" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.2e-07 }, "nebius/Qwen/Qwen3.8-27B": { "input_cost_per_token": 4.5e-07, @@ -44762,9 +44838,10 @@ "max_input_tokens": 262144, "mode": "chat", "output_cost_per_token": 3e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/Qwen%2FQwen3.8-27B", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 1e-07 }, "nebius/zai-org/GLM-5.1": { "max_input_tokens": 202752, @@ -44775,7 +44852,8 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.1" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.2": { "max_input_tokens": 1048576, @@ -44786,21 +44864,23 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.2" + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "cache_read_input_token_cost": 1.4e-07 }, "nebius/zai-org/GLM-5.3": { "input_cost_per_token": 1.4e-06, "litellm_provider": "nebius", - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "mode": "chat", "output_cost_per_token": 4.4e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 2.6e-07 }, "nebius/zai-org/GLM-5.3-Flash": { - "max_input_tokens": 1024000, + "max_input_tokens": 1048576, "max_output_tokens": 1024000, "input_cost_per_token": 1.5e-07, "output_cost_per_token": 5e-07, @@ -44808,8 +44888,9 @@ "mode": "chat", "supports_function_calling": true, "supports_reasoning": true, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/zai-org%2FGLM-5.3-Flash", - "supports_vision": true + "source": "https://tokenfactory.nebius.com/api/public/models_info", + "supports_vision": true, + "cache_read_input_token_cost": 3e-08 }, "nebius/BAAI/bge-en-icl": { "max_input_tokens": 32768, @@ -49813,7 +49894,7 @@ "mode": "chat", "output_cost_per_token": 3e-05, "prompt_cache_min_tokens": 512, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json", "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, @@ -49835,7 +49916,9 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05 }, "us-gov.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -52922,6 +53005,7 @@ }, "vertex_ai/gemini-3.1-flash-lite-image": { + "deprecation_date": "2027-06-28", "cache_read_input_token_cost": 2.5e-08, "cache_read_input_token_cost_batches": 1.25e-08, "cache_read_input_token_cost_flex": 1.25e-08, @@ -54150,7 +54234,7 @@ "voyage/voyage-lite-01": { "input_cost_per_token": 1e-07, "litellm_provider": "voyage", - "max_input_tokens": 4096, + "max_input_tokens": 4000, "mode": "embedding", "output_cost_per_token": 0.0 }, @@ -64809,7 +64893,7 @@ "supports_vision": false }, "novita/deepseek/deepseek-v4-pro-0813": { - "cache_read_input_token_cost": 1.3200000000000002e-07, + "cache_read_input_token_cost": 4.4e-08, "input_cost_per_token": 1.32e-06, "litellm_provider": "novita", "max_input_tokens": 1048576, @@ -64921,13 +65005,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-venti": { - "cache_read_input_token_cost": 3e-07, - "input_cost_per_token": 1.5e-06, + "cache_read_input_token_cost": 1.65e-07, + "input_cost_per_token": 8.25e-07, "litellm_provider": "novita", "max_input_tokens": 1048576, "max_output_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.5e-06, + "output_cost_per_token": 2.475e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -65030,13 +65114,13 @@ "supports_vision": false }, "novita/mindai/macaron-v1-tall": { - "cache_read_input_token_cost": 8e-08, - "input_cost_per_token": 4.5000000000000003e-07, + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 2.475e-07, "litellm_provider": "novita", "max_input_tokens": 262144, "max_output_tokens": 32768, "mode": "chat", - "output_cost_per_token": 2.6e-06, + "output_cost_per_token": 1.43e-06, "source": "https://novita.ai/pricing", "supports_function_calling": true, "supports_prompt_caching": true, @@ -66461,9 +66545,9 @@ "deepinfra/deepseek-ai/DeepSeek-V4-Flash-0731": { "max_output_tokens": 1048576, "max_input_tokens": 1048576, - "input_cost_per_token": 8e-08, + "input_cost_per_token": 6e-08, "output_cost_per_token": 1.8e-07, - "cache_read_input_token_cost": 1.6e-08, + "cache_read_input_token_cost": 1.5e-08, "supports_prompt_caching": true, "litellm_provider": "deepinfra", "mode": "chat", @@ -66842,8 +66926,8 @@ "deepinfra/google/gemma-4-31B-it": { "max_output_tokens": 262144, "max_input_tokens": 262144, - "input_cost_per_token": 1.3e-07, - "output_cost_per_token": 3.8e-07, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 4e-07, "litellm_provider": "deepinfra", "mode": "chat", "supports_function_calling": true, @@ -68554,7 +68638,10 @@ "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "us-gov-west-1" - } + }, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-west-1/index.json" }, "bedrock/us-gov-west-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -68829,7 +68916,10 @@ "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true, - "supports_xhigh_reasoning_effort": true + "supports_xhigh_reasoning_effort": true, + "input_cost_per_token_batches": 3e-06, + "output_cost_per_token_batches": 1.5e-05, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/us-gov-east-1/index.json" }, "bedrock/us-gov-east-1/anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -69852,6 +69942,14 @@ "cache_read_input_token_cost": 3e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/google/gemini-4-argon": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1e-05, + "cache_read_input_token_cost": 1e-07, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", @@ -69979,6 +70077,14 @@ "cache_read_input_token_cost": 1.15e-09, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, + "perplexity/perplexity/nemotron-3.5-super-120b-a12b": { + "litellm_provider": "perplexity", + "mode": "responses", + "input_cost_per_token": 2e-07, + "output_cost_per_token": 5e-07, + "cache_read_input_token_cost": 4e-08, + "source": "https://docs.perplexity.ai/docs/agent-api/models" + }, "perplexity/perplexity/nemotron-3-ultra-550b-a55b": { "litellm_provider": "perplexity", "mode": "responses", @@ -70104,12 +70210,12 @@ "input_cost_per_token": 1.5e-06, "litellm_provider": "vertex_ai", "max_output_tokens": 57920, - "max_input_tokens": 57920, + "max_input_tokens": 131072, "mode": "chat", "output_cost_per_reasoning_token": 9e-06, "output_cost_per_token": 9e-06, "output_cost_per_video_token": 1.75e-05, - "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", + "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-1-1-flash", "supported_endpoints": [ "/v1beta/interactions" ], @@ -74505,12 +74611,14 @@ "input_cost_per_token": 5.5e-06, "cache_read_input_token_cost": 5.5e-07, "output_cost_per_token": 2.75e-05, - "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/ap-northeast-1/index.json", "token_count_model": { "model": "anthropic.claude-opus-5", "litellm_provider": "bedrock_mantle", "region": "ap-southeast-4" - } + }, + "input_cost_per_token_batches": 2.75e-06, + "output_cost_per_token_batches": 1.375e-05 }, "apac.anthropic.claude-opus-5-5": { "bedrock_converse_supports_strict_tools": false, @@ -75107,7 +75215,7 @@ "supports_function_calling": true, "input_cost_per_token": 2e-06, "output_cost_per_token": 1e-05, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/anthropic/claude-fable-5-1": { @@ -75155,6 +75263,9 @@ "output_cost_per_token": 5e-05, "cache_read_input_token_cost": 1e-06, "cache_creation_input_token_cost": 1.25e-05, + "input_cost_per_token_above_272k_tokens": 2e-05, + "output_cost_per_token_above_272k_tokens": 7.5e-05, + "cache_read_input_token_cost_above_272k_tokens": 2e-06, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "perplexity/openai/gpt-6-luna": { @@ -76622,6 +76733,9 @@ "input_cost_per_token": 1e-07, "output_cost_per_token": 5e-07, "cache_read_input_token_cost": 1e-08, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, "source": "https://docs.perplexity.ai/docs/agent-api/models" }, "us-gov.anthropic.claude-haiku-5-5": { @@ -77096,6 +77210,7 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-4-argon": { + "deprecation_date": "2027-10-13", "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, @@ -77504,9 +77619,10 @@ "max_output_tokens": 1048576, "mode": "chat", "output_cost_per_token": 1e-06, - "source": "https://tokenfactory.nebius.com/models/catalog/text2text/reflection%2Fbeam", + "source": "https://tokenfactory.nebius.com/api/public/models_info", "supports_function_calling": true, - "supports_reasoning": true + "supports_reasoning": true, + "cache_read_input_token_cost": 5e-08 }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-1017": { "cache_read_input_token_cost": 7e-09, @@ -81777,5 +81893,55 @@ "supports_response_schema": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true + }, + "nebius/deepseek-ai/DeepSeek-V4.1-Flash-1017": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-09, + "output_cost_per_token": 1.2e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/meta-models/muse-spark-open": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.25e-06, + "cache_read_input_token_cost": 1.5e-07, + "output_cost_per_token": 4.25e-06, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/MiniMaxAI/MiniMax-M3.1-Flash": { + "max_input_tokens": 1048576, + "input_cost_per_token": 1.5e-07, + "cache_read_input_token_cost": 1.5e-08, + "output_cost_per_token": 6e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" + }, + "nebius/nvidia/Nemotron-3.5-Super-VL": { + "max_input_tokens": 1048576, + "input_cost_per_token": 3e-07, + "cache_read_input_token_cost": 6e-08, + "output_cost_per_token": 9e-07, + "litellm_provider": "nebius", + "mode": "chat", + "supports_vision": true, + "supports_video_input": true, + "supports_reasoning": true, + "supports_function_calling": true, + "source": "https://tokenfactory.nebius.com/api/public/models_info" } } diff --git a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py index 9b20192c3f2..a5b66efe7cd 100644 --- a/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py +++ b/tests/unit/llms/azure_ai/test_azure_ai_foundry_catalog_model_metadata.py @@ -22,6 +22,7 @@ TOKEN_PRICED_NAMES: Final = ( "cohere-command-a", "grok-4-20-reasoning", "grok-4-20-non-reasoning", + "claude-mythos-5-1", ) GROK_4_20_NAMES: Final = ("grok-4-20-reasoning", "grok-4-20-non-reasoning") CATALOG_NAMES: Final = TOKEN_PRICED_NAMES + ("whisper",) From cba5bcb17e31906541afb1cc8e90e18a3b27a771 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 3 Nov 2026 11:51:44 -0800 Subject: [PATCH 30/34] feat(model-catalog): read successor_model, the provider's recommended replacement for a deprecating row (#49791) * feat(model-catalog): read successor_model, the provider's recommended replacement for a deprecating row * test(cost-map): allow successor_model in the cost map row schema --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/model-catalog/src/wire/attributes.rs | 4 ++++ tests/unit/test_utils.py | 1 + 2 files changed, 5 insertions(+) diff --git a/litellm-rust/crates/model-catalog/src/wire/attributes.rs b/litellm-rust/crates/model-catalog/src/wire/attributes.rs index 307f25f0bda..d2ba5f609ee 100644 --- a/litellm-rust/crates/model-catalog/src/wire/attributes.rs +++ b/litellm-rust/crates/model-catalog/src/wire/attributes.rs @@ -93,6 +93,8 @@ attributes! { search_free_requests: SearchFreeRequests, /// URL of the provider pricing/model page this entry was taken from. source: String, + /// Model the provider recommends migrating to before `deprecation_date`, as a catalog key when one exists. + successor_model: String, /// Audio container formats the model can return. supported_audio_formats: Vec, /// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions. @@ -303,6 +305,8 @@ mod tests { #[case::known_mode("mode", json!("chat"), true)] #[case::unknown_mode("mode", json!("chatt"), false)] #[case::negative_dbu("input_dbu_cost_per_token", json!(-1), false)] + #[case::successor_model("successor_model", json!("gpt-6-luna"), true)] + #[case::successor_model_not_a_name("successor_model", json!(["gpt-6-luna"]), false)] #[case::thinking_on_by_default("thinking_on_by_default", json!(true), true)] #[case::thinking_on_by_default_not_a_flag("thinking_on_by_default", json!("yes"), false)] #[case::reasoning_contexts("reasoning_contexts", json!(["reasoning", "non_reasoning"]), true)] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c1c2593c4ef..f12280545b7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1026,6 +1026,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_read_input_image_token_cost": {"type": "number"}, "audio_transcription_config": {"type": "string"}, "deprecation_date": {"type": "string"}, + "successor_model": {"type": "string"}, "input_cost_per_audio_per_second": {"type": "number"}, "input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"}, "google_maps_grounding_cost_per_query": {"type": "number"}, From f52e5057f9f352bc7c48437246a512e8f2124be8 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 3 Nov 2026 11:52:52 -0800 Subject: [PATCH 31/34] refactor(ui): move teams, users and orgs text onto i18n (#49484) * refactor(ui): move teams, users and orgs text onto i18n Moves the page-authored English on /teams, /users and /organizations, plus the shared member table, user search modal, model select and budget schedule helpers they render, into react-i18next t() calls backed by en.json. The rendered English is unchanged Helpers that files outside this batch or untouched tests still call keep their signatures and take an optional t that defaults to the shared instance, the same as usageUnits.ts. Text that existing tests pin, values compared against the API, code samples, CSV column names and date formats stay as they were * refactor(ui): escape member ids in team member dialogs and merge split org and team strings The reset spend and reset budget dialogs interpolate the member id into a Trans string, so an id containing strong, i, p or br tags rendered as markup. Escaping the values and unescaping text nodes keeps it literal, as it was before the move to i18n. Also merges the org budget lines and the team metadata hint into whole sentences and drops two constants nothing imports anymore * refactor(ui): pass the member id into the team member reset dialogs as an element Escaping the id still let Trans fill in placeholders like {{budget}} written inside it a second time. Rendering the id through its own component keeps it out of the translated string, so any id shows exactly as it did before the move to i18n --- .../organizations/OrganizationFilters.tsx | 6 +- .../_components/OrganizationsPanel.tsx | 39 +- .../_components/OrganizationsTable.tsx | 15 +- .../_components/OrganizationsTableColumns.tsx | 71 +- .../users/_components/BulkEditUsers.tsx | 79 +- .../DefaultUserSettingsForm.tsx | 176 +- .../default-user-settings/schema.ts | 17 +- .../users/_components/user_edit_view.tsx | 100 +- .../users/_components/view_users.tsx | 38 +- .../_components/view_users/UsersTable.tsx | 56 +- .../view_users/UsersTableColumns.tsx | 110 +- .../_components/view_users/user_info_view.tsx | 187 +- .../src/components/CreateUserButton.tsx | 98 +- .../components/ModelSelect/ModelSelect.tsx | 47 +- .../src/components/TeamSSOSettings.tsx | 82 +- ui/litellm-dashboard/src/components/Teams.tsx | 319 ++-- .../TeamsPage/TeamMemberSpendBudgetCell.tsx | 29 +- .../src/components/TeamsPage/TeamsTable.tsx | 48 +- .../components/TeamsPage/teamTableColumns.tsx | 91 +- .../components/bulk_create_users_button.tsx | 220 +-- .../Filters/FiltersButton.tsx | 11 +- .../Filters/ResetFiltersButton.tsx | 6 +- .../common_components/MemberTable.tsx | 50 +- .../common_components/user_search_modal.tsx | 56 +- .../org-create/OrgCreateDialog.tsx | 62 +- .../org-settings/OrgSettingsForm.tsx | 66 +- .../organization/org-settings/schema.ts | 44 +- .../organization/organization_view.tsx | 152 +- .../components/team/AvailableTeamsPanel.tsx | 6 +- .../components/team/AvailableTeamsTable.tsx | 31 +- .../team/AvailableTeamsTableColumns.tsx | 33 +- .../src/components/team/EditMembership.tsx | 49 +- .../src/components/team/GuardrailsSelect.tsx | 25 +- .../src/components/team/LoggingSettings.tsx | 90 +- .../src/components/team/MyUserTab.tsx | 52 +- .../team/ResetMemberBudgetsDialog.tsx | 21 +- .../components/team/TeamAdminSettingsForm.tsx | 30 +- .../src/components/team/TeamInfo.tsx | 823 ++++---- .../src/components/team/TeamMemberTab.tsx | 142 +- .../components/team/TeamVirtualKeysTable.tsx | 190 +- .../src/components/team/memberFormValues.ts | 3 +- .../components/team/member_permissions.tsx | 30 +- .../components/team/teamAdminEditAccess.ts | 46 +- .../src/components/team/teamModelAccess.ts | 27 +- .../components/team/useMemberBudgetReset.ts | 17 +- ui/litellm-dashboard/src/i18n/locales/en.json | 1651 +++++++++++++++++ ui/litellm-dashboard/src/utils/budgetUtils.ts | 24 +- 47 files changed, 3957 insertions(+), 1608 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx index 6ad2f00fdb0..cd5c796b569 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/OrganizationFilters.tsx @@ -2,6 +2,7 @@ import { FilterInput } from "@/components/common_components/Filters/FilterInput" import { FiltersButton } from "@/components/common_components/Filters/FiltersButton"; import { ResetFiltersButton } from "@/components/common_components/Filters/ResetFiltersButton"; import { Search, User } from "lucide-react"; +import { useTranslation } from "react-i18next"; interface OrganizationFiltersProps { filters: FilterState; @@ -23,6 +24,7 @@ const OrganizationFilters = ({ onChange, onReset, }: OrganizationFiltersProps) => { + const { t } = useTranslation(); const hasActiveFilters = !!(filters.org_id || filters.org_alias); return ( @@ -30,7 +32,7 @@ const OrganizationFilters = ({ {/* Search and Filter Controls */}
onChange("org_alias", value)} icon={Search} @@ -50,7 +52,7 @@ const OrganizationFilters = ({ {showFilters && (
onChange("org_id", value)} icon={User} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx index 2810902bef6..819aa6c3334 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsPanel.tsx @@ -4,6 +4,7 @@ import OrganizationFilters, { FilterState } from "@/app/(dashboard)/organization import { useQueryClient } from "@tanstack/react-query"; import { parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; import React, { useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { toast } from "@/lib/toast"; import { organizationDeleteCall } from "@/components/networking"; @@ -28,6 +29,7 @@ const ORGANIZATION_DETAIL_STATE = { const ORGANIZATION_DETAIL_URL_KEYS = { tab: ORGANIZATION_TAB_URL_KEY }; const OrganizationsPanel: React.FC = ({ userRole, accessToken, premiumUser }) => { + const { t } = useTranslation(); const [{ org: selectedOrgId }, setOrganizationDetail] = useQueryStates(ORGANIZATION_DETAIL_STATE, { history: "push", urlKeys: ORGANIZATION_DETAIL_URL_KEYS, @@ -78,7 +80,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces try { setIsDeleting(true); await organizationDeleteCall(accessToken, orgToDelete); - toast.success("Organization deleted successfully"); + toast.success(t("organizations.panel.toasts.deleted")); setIsDeleteModalOpen(false); setOrgToDelete(null); @@ -99,16 +101,19 @@ const OrganizationsPanel: React.FC = ({ userRole, acces return (

- This is a LiteLLM Enterprise feature, and requires a valid key to use. Get a trial key{" "} - - here - - . + + ), + }} + />

); @@ -118,7 +123,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces
{(userRole === "Admin" || userRole === "Org Admin") && ( )} @@ -133,7 +138,7 @@ const OrganizationsPanel: React.FC = ({ userRole, acces /> ) : ( <> -

Click on an organization ID to view its details.

+

{t("organizations.panel.listHint")}

= ({ userRole, acces @@ -27,12 +29,12 @@ function EmptyState({ searchActive }: { searchActive: boolean }) {
- {searchActive ? "No matching organizations" : "No organizations yet"} + {searchActive ? t("organizations.table.empty.noMatchesTitle") : t("organizations.table.empty.noneTitle")}
{searchActive - ? "No organizations match your search. Try a different name or ID." - : "Create an organization to group teams, models, and budgets."} + ? t("organizations.table.empty.noMatchesDescription") + : t("organizations.table.empty.noneDescription")}
); @@ -47,12 +49,13 @@ const OrganizationsTable: React.FC = ({ onEditClick, onDeleteClick, }) => { + const { t } = useTranslation(); const { sorting, onSortingChange, pagination, onPaginationChange } = useOrganizationsTableState(); const columns = useMemo(() => { - const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick }; + const deps = { userRole, onOrganizationClick, onEditClick, onDeleteClick, t }; return getOrganizationsTableColumns(deps); - }, [userRole, onOrganizationClick, onEditClick, onDeleteClick]); + }, [userRole, onOrganizationClick, onEditClick, onDeleteClick, t]); return ( = ({ sorting={sorting} onSortingChange={onSortingChange} isLoading={isLoading} - loadingMessage="Loading organizations…" + loadingMessage={t("organizations.table.loading")} noDataMessage={} size="compact" /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx index dcbf141314b..b4464392ea8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx @@ -1,7 +1,9 @@ "use client"; import { ColumnDef } from "@tanstack/react-table"; +import type { TFunction } from "i18next"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable"; import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; @@ -25,11 +27,18 @@ const getOrganizationBudget = (organization: Organization): OrganizationBudget = (organization.litellm_budget_table ?? {}) as OrganizationBudget; function OrganizationLimitsCell({ organization }: { organization: Organization }) { + const { t } = useTranslation(); const { tpm_limit, rpm_limit } = getOrganizationBudget(organization); return (
- TPM: {tpm_limit ?? "Unlimited"} - RPM: {rpm_limit ?? "Unlimited"} + + {t("organizations.table.limits.tpmLabel")} + {tpm_limit ?? t("organizations.table.limits.unlimited")} + + + {t("organizations.table.limits.rpmLabel")} + {rpm_limit ?? t("organizations.table.limits.unlimited")} +
); } @@ -41,10 +50,11 @@ interface OrganizationRowActionsProps { } function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: OrganizationRowActionsProps) { + const { t } = useTranslation(); return ( @@ -56,7 +66,7 @@ function OrganizationRowActions({ organization, onEditClick, onDeleteClick }: Or onClick={() => onEditClick(organization.organization_id)} > - Edit + {t("organizations.table.actions.edit")} onDeleteClick(organization.organization_id)} > - Delete + {t("organizations.table.actions.delete")} @@ -76,6 +86,7 @@ export interface OrganizationsTableColumnsDeps { onOrganizationClick: (organizationId: string) => void; onEditClick: (organizationId: string) => void; onDeleteClick: (organizationId: string) => void; + t: TFunction; } export const getOrganizationsTableColumns = ({ @@ -83,12 +94,13 @@ export const getOrganizationsTableColumns = ({ onOrganizationClick, onEditClick, onDeleteClick, + t, }: OrganizationsTableColumnsDeps): ColumnDef[] => [ { id: "organization_id", accessorKey: "organization_id", - meta: { title: "Organization ID" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.id") }, + header: ({ column }) => , size: 220, enableSorting: true, cell: ({ row }) => ( @@ -103,8 +115,8 @@ export const getOrganizationsTableColumns = ({ { id: "organization_alias", accessorKey: "organization_alias", - meta: { title: "Organization Name" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.name") }, + header: ({ column }) => , size: 200, enableSorting: true, cell: ({ row }) => { @@ -120,8 +132,8 @@ export const getOrganizationsTableColumns = ({ id: "created_at", accessorKey: "created_at", sortingFn: "datetime", - meta: { title: "Created" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.created") }, + header: ({ column }) => , size: 130, enableSorting: true, cell: ({ row }) => , @@ -129,8 +141,8 @@ export const getOrganizationsTableColumns = ({ { id: "spend", accessorKey: "spend", - meta: { title: "Spend (USD)", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.spend"), numeric: true }, + header: ({ column }) => , size: 120, enableSorting: true, cell: ({ row }) => , @@ -140,12 +152,17 @@ export const getOrganizationsTableColumns = ({ accessorFn: (organization) => getOrganizationBudget(organization).max_budget ?? Number.POSITIVE_INFINITY, sortUndefined: "last", sortDescFirst: true, - meta: { title: "Budget (USD)", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.budget"), numeric: true }, + header: ({ column }) => , size: 120, enableSorting: true, cell: ({ row }) => ( - + ), }, { @@ -156,21 +173,21 @@ export const getOrganizationsTableColumns = ({ : undefined, sortUndefined: "last", sortDescFirst: false, - meta: { title: "Models", skeleton: "chips" }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.models"), skeleton: "chips" }, + header: ({ column }) => , size: 260, enableSorting: true, cell: ({ row }) => , }, { id: "limits", - meta: { title: "TPM / RPM Limits" }, + meta: { title: t("organizations.table.columns.limits") }, header: ({ table }) => ( ), @@ -183,11 +200,15 @@ export const getOrganizationsTableColumns = ({ accessorFn: (organization) => organization.members?.length ?? 0, sortUndefined: "last", sortDescFirst: true, - meta: { title: "Members", numeric: true }, - header: ({ column }) => , + meta: { title: t("organizations.table.columns.members"), numeric: true }, + header: ({ column }) => , size: 100, enableSorting: true, - cell: ({ row }) => {row.original.members?.length ?? 0} Members, + cell: ({ row }) => ( + + {t("organizations.table.memberCount", { count: row.original.members?.length ?? 0 })} + + ), }, ...(["tpm_limit", "rpm_limit"] as const).map( (field): ColumnDef => ({ @@ -201,7 +222,7 @@ export const getOrganizationsTableColumns = ({ { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, - header: () => Actions, + header: () => {t("organizations.table.columns.actions")}, size: 64, enableSorting: false, enableHiding: false, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx index 67a648dacd1..b38dd5c6d5e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/BulkEditUsers.tsx @@ -1,4 +1,5 @@ import React, { useId, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; import { userBulkUpdateUserCall, teamBulkMemberAddCall, Member } from "@/components/networking"; import { UserEditView } from "./user_edit_view"; import { toast } from "@/lib/toast"; @@ -46,6 +47,7 @@ const BulkEditUserModal: React.FC = ({ userModels, allowAllUsers = false, }) => { + const { t } = useTranslation(); const { premiumUser } = useAuthorized(); const [loading, setLoading] = useState(false); const [selectedTeams, setSelectedTeams] = useState([]); @@ -89,7 +91,7 @@ const BulkEditUserModal: React.FC = ({ const handleSubmit = async (formValues: any) => { if (!accessToken) { - toast.fromError("Access token not found"); + toast.fromError(t("users.bulkEdit.toasts.noAccessToken")); return; } @@ -125,7 +127,7 @@ const BulkEditUserModal: React.FC = ({ const hasTeamAdditions = addToTeams && selectedTeams.length > 0; if (!hasUserUpdates && !hasTeamAdditions) { - toast.fromError("Please modify at least one field or select teams to add users to"); + toast.fromError(t("users.bulkEdit.toasts.nothingToUpdate")); return; } @@ -135,10 +137,10 @@ const BulkEditUserModal: React.FC = ({ if (hasUserUpdates) { if (updateAllUsers) { const result = await userBulkUpdateUserCall(accessToken, updatePayload, undefined, true); - successMessages.push(`Updated all users (${result.total_requested} total)`); + successMessages.push(t("users.bulkEdit.toasts.updatedAll", { total: result.total_requested })); } else { await userBulkUpdateUserCall(accessToken, updatePayload, userIds); - successMessages.push(`Updated ${userIds.length} user(s)`); + successMessages.push(t("users.bulkEdit.toasts.updated", { count: userIds.length })); } } @@ -190,11 +192,13 @@ const BulkEditUserModal: React.FC = ({ if (successfulTeams.length > 0) { const totalAdditions = successfulTeams.reduce((sum, r) => sum + r.successfulAdditions, 0); - successMessages.push(`Added users to ${successfulTeams.length} team(s) (${totalAdditions} total additions)`); + successMessages.push( + t("users.bulkEdit.toasts.addedToTeams", { count: successfulTeams.length, additions: totalAdditions }), + ); } if (failedTeams.length > 0) { - toast.warning(`Failed to add users to ${failedTeams.length} team(s)`); + toast.warning(t("users.bulkEdit.toasts.teamAddFailed", { count: failedTeams.length })); } } @@ -212,7 +216,7 @@ const BulkEditUserModal: React.FC = ({ onCancel(); } catch (error) { console.error("Bulk operation failed:", error); - toast.fromError("Failed to perform bulk operations"); + toast.fromError(t("users.bulkEdit.toasts.bulkFailed")); } finally { setLoading(false); } @@ -223,7 +227,7 @@ const BulkEditUserModal: React.FC = ({ - {updateAllUsers ? "Bulk Edit All Users" : `Bulk Edit ${selectedUsers.length} User(s)`} + {updateAllUsers ? t("users.bulkEdit.titleAll") : t("users.bulkEdit.title", { count: selectedUsers.length })} {allowAllUsers && ( @@ -233,17 +237,15 @@ const BulkEditUserModal: React.FC = ({ id={updateAllUsersId} checked={updateAllUsers} onCheckedChange={(checked) => setUpdateAllUsers(checked === true)} - aria-label="Update ALL users in the system" + aria-label={t("users.bulkEdit.updateAll")} />
{updateAllUsers && (
- - ⚠️ This will apply changes to ALL users in the system, not just the selected ones. - + {t("users.bulkEdit.updateAllWarning")}
)} @@ -251,15 +253,19 @@ const BulkEditUserModal: React.FC = ({ {!updateAllUsers && (
-
Selected Users ({selectedUsers.length}):
+
+ {t("users.bulkEdit.selectedUsers", { count: selectedUsers.length })} +
- User ID - Email - Current Role - Budget + {t("users.bulkEdit.table.userId")} + {t("users.bulkEdit.table.email")} + {t("users.bulkEdit.table.currentRole")} + + {t("users.bulkEdit.table.budget")} + @@ -268,12 +274,19 @@ const BulkEditUserModal: React.FC = ({ {user.user_id.length > 20 ? `${user.user_id.slice(0, 20)}...` : user.user_id} - {user.user_email || "No email"} + + {user.user_email || t("users.bulkEdit.table.noEmail")} + {possibleUIRoles?.[user.user_role]?.ui_label || user.user_role} - + ))} @@ -287,15 +300,14 @@ const BulkEditUserModal: React.FC = ({

- Instructions: Fill in the fields below with the values you want to apply to all selected - users. You can bulk edit: role, budget, models, and metadata. You can also add users to teams. + }} />

{/* Team Management Section */} - Team Management + {t("users.bulkEdit.teamManagement.title")}
@@ -304,10 +316,10 @@ const BulkEditUserModal: React.FC = ({ id={addToTeamsId} checked={addToTeams} onCheckedChange={(checked) => setAddToTeams(checked === true)} - aria-label="Add selected users to teams" + aria-label={t("users.bulkEdit.teamManagement.addToTeams")} />
@@ -315,12 +327,12 @@ const BulkEditUserModal: React.FC = ({ <>
= ({
) => setTeamBudget(event.target.value === "" ? null : Number(event.target.value)) @@ -348,13 +360,12 @@ const BulkEditUserModal: React.FC = ({ step={0.01} /> - Leave empty for unlimited budget within team limits + {t("users.bulkEdit.teamManagement.teamBudgetHint")}
- Users will be added with "user" role by default. All users will be added to each selected - team. + {t("users.bulkEdit.teamManagement.defaultRoleNote")} )} @@ -379,7 +390,9 @@ const BulkEditUserModal: React.FC = ({ {loading && (
- Updating {updateAllUsers ? "all users" : selectedUsers.length} user(s)... + {updateAllUsers + ? t("users.bulkEdit.updatingAll") + : t("users.bulkEdit.updating", { count: selectedUsers.length })}
)} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx index 0239844851f..a617d5f84cb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/default-user-settings/DefaultUserSettingsForm.tsx @@ -1,8 +1,10 @@ "use client"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import type { ParseKeys } from "i18next"; import * as React from "react"; import { useFieldArray, type Control } from "react-hook-form"; +import { useTranslation } from "react-i18next"; import { useInfiniteTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { ModelSelect, MODEL_SENTINEL_OPTIONS } from "@/components/ModelSelect/ModelSelect"; @@ -29,21 +31,21 @@ import { const NO_RESET = "never"; -const BUDGET_DURATION_OPTIONS = [ - { value: NO_RESET, label: "No reset" }, - { value: "1h", label: "hourly" }, - { value: "24h", label: "daily" }, - { value: "7d", label: "weekly" }, - { value: "30d", label: "monthly" }, -] as const; +const BUDGET_DURATION_OPTIONS: readonly { value: string; labelKey: ParseKeys }[] = [ + { value: NO_RESET, labelKey: "users.defaultSettings.budgetDurations.noReset" }, + { value: "1h", labelKey: "users.defaultSettings.budgetDurations.hourly" }, + { value: "24h", labelKey: "users.defaultSettings.budgetDurations.daily" }, + { value: "7d", labelKey: "users.defaultSettings.budgetDurations.weekly" }, + { value: "30d", labelKey: "users.defaultSettings.budgetDurations.monthly" }, +]; -const TEAM_ROLE_OPTIONS = [ - { value: "user", label: "User" }, - { value: "admin", label: "Admin" }, -] as const; +const TEAM_ROLE_OPTIONS: readonly { value: "user" | "admin"; labelKey: ParseKeys }[] = [ + { value: "user", labelKey: "users.defaultSettings.teamRoles.user" }, + { value: "admin", labelKey: "users.defaultSettings.teamRoles.admin" }, +]; -const MODEL_SENTINEL_LABELS: ReadonlyMap = new Map( - MODEL_SENTINEL_OPTIONS.map(({ value, label }) => [value, label]), +const MODEL_SENTINEL_LABEL_KEYS: ReadonlyMap = new Map( + MODEL_SENTINEL_OPTIONS.map(({ value, labelKey }) => [value, labelKey]), ); const TEAMS_PAGE_SIZE = 50; @@ -71,6 +73,7 @@ interface RoleOption { type SettingsControl = Control; const TeamPickerField = ({ control, index }: { control: SettingsControl; index: number }) => { + const { t } = useTranslation(); const [search, setSearch] = React.useState(""); const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteTeams( TEAMS_PAGE_SIZE, @@ -90,7 +93,7 @@ const TeamPickerField = ({ control, index }: { control: SettingsControl; index: ); return ( - + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( { + const { t } = useTranslation(); const { fields, append, remove } = useFieldArray({ control, name: "teams" }); + const teamRoleItems = TEAM_ROLE_OPTIONS.map((option) => ({ value: option.value, label: t(option.labelKey) })); return (
-

Default Teams

-

- New users are added to these teams. Only teams that already exist can be selected. -

+

{t("users.defaultSettings.fields.teams")}

+

{t("users.defaultSettings.teams.description")}

{fields.map((field, index) => (
-

Team {index + 1}

+

{t("users.defaultSettings.teams.rowTitle", { number: index + 1 })}

- + {({ ref, ...budgetField }) => ( - + )} - + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - onChange(selected ?? "user")}> { - {TEAM_ROLE_OPTIONS.map((option) => ( + {teamRoleItems.map((option) => ( {option.label} @@ -172,7 +186,7 @@ const TeamsField = ({ control }: { control: SettingsControl }) => { ))}
); @@ -191,30 +205,41 @@ interface SettingsViewProps { } const SettingsView = ({ values, roleOptions }: SettingsViewProps) => { + const { t } = useTranslation(); const roleLabel = roleOptions.find((option) => option.value === values.user_role)?.label ?? values.user_role; const durationValue = values.budget_duration === "" ? NO_RESET : values.budget_duration; - const durationLabel = - BUDGET_DURATION_OPTIONS.find((option) => option.value === durationValue)?.label ?? values.budget_duration; + const durationOption = BUDGET_DURATION_OPTIONS.find((option) => option.value === durationValue); + const durationLabel = durationOption === undefined ? values.budget_duration : t(durationOption.labelKey); + const notSet = t("users.defaultSettings.view.notSet"); return (
- {roleLabel === "" ? "Not set" : roleLabel} - {values.max_budget === "" ? "Not set" : values.max_budget} - {durationLabel} - + {roleLabel === "" ? notSet : roleLabel} + + {values.max_budget === "" ? notSet : values.max_budget} + + {durationLabel} + {values.models.length === 0 - ? "Not set" - : values.models.map((model) => MODEL_SENTINEL_LABELS.get(model) ?? model).join(", ")} + ? notSet + : values.models + .map((model) => { + const labelKey = MODEL_SENTINEL_LABEL_KEYS.get(model); + return labelKey === undefined ? model : t(labelKey); + }) + .join(", ")}
-

Default Teams

+

{t("users.defaultSettings.fields.teams")}

{values.teams.length === 0 ? ( -

None

+

{t("users.defaultSettings.view.none")}

) : ( values.teams.map((team) => (

{team.team_id} - {team.max_budget_in_team !== "" && <> · ${team.max_budget_in_team} max budget} + {team.max_budget_in_team !== "" && ( + <> · {t("users.defaultSettings.view.teamMaxBudget", { budget: team.max_budget_in_team })} + )} <> · {team.user_role}

)) @@ -233,20 +258,25 @@ interface SettingsFormProps { } const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, onSaved }: SettingsFormProps) => { + const { t } = useTranslation(); const queryClient = useQueryClient(); + const budgetDurationItems = BUDGET_DURATION_OPTIONS.map((option) => ({ + value: option.value, + label: t(option.labelKey), + })); const form = useZodForm(defaultUserSettingsSchema, { defaultValues: initialValues }); const { isDirty } = form.formState; const mutation = useMutation({ mutationFn: (values: DefaultUserSettingsSubmitValues) => updateSettings(buildBody(values)), onSuccess: (_result, values) => { - toast.success("Default user settings updated successfully"); + toast.success(t("users.defaultSettings.toasts.updated")); queryClient.invalidateQueries({ queryKey: SETTINGS_QUERY_KEY }); form.reset(values); onSaved(); }, onError: (error: unknown) => - toast.fromError(error instanceof Error ? error.message : "Failed to update default user settings"), + toast.fromError(error instanceof Error ? error.message : t("users.defaultSettings.toasts.updateFailed")), }); const onSubmit = form.handleSubmit((values) => mutation.mutate(values)); @@ -257,8 +287,8 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( } @@ -295,12 +325,12 @@ const SettingsForm = ({ initialValues, roleOptions, updateSettings, onCancel, on {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( } )} {!isBulkEdit && ( - + {({ ref, value, ...control }) => } )} - + {({ ref, value, ...control }) => } @@ -242,8 +247,8 @@ export function UserEditView({ control={form.control} name="user_role" label={labelWithHint( - "Global Proxy Role", - "This is the role that the user will globally on the proxy. This role is independent of any team/org specific roles.", + t("users.editView.fields.globalProxyRole"), + t("users.editView.fields.globalProxyRoleHint"), )} > {({ id, value, onChange }) => ( @@ -271,8 +276,8 @@ export function UserEditView({ control={form.control} name="models" label={labelWithHint( - "Personal Models", - "Select which models this user can access outside of team-scope. Choose 'All Proxy Models' to grant access to all models available on the proxy.", + t("users.editView.fields.personalModels"), + t("users.editView.fields.personalModelsHint"), )} > {({ value, onChange }) => ( @@ -280,7 +285,7 @@ export function UserEditView({ options={modelOptions} value={value} onValueChange={onChange} - placeholder="Select models" + placeholder={t("users.editView.fields.modelsPlaceholder")} disabled={!all_admin_roles.includes(userRole || "")} /> )} @@ -291,10 +296,10 @@ export function UserEditView({ name="max_budget" label={ <> - Max Budget (USD) + {t("users.editView.fields.maxBudget")} } @@ -308,19 +313,19 @@ export function UserEditView({ value={value ?? ""} onChange={(event) => onChange(event.target.value)} onWheel={(event) => event.currentTarget.blur()} - placeholder="Enter a numerical value" + placeholder={t("users.editView.fields.maxBudgetPlaceholder")} disabled={unlimitedBudget} /> )} - + {({ id, value, onChange }) => ( )} @@ -330,10 +335,7 @@ export function UserEditView({ {({ ref, value, onChange, ...control }) => ( onChange(event.target.value)} onWheel={(event) => event.currentTarget.blur()} - placeholder="Unlimited" + placeholder={t("users.editView.fields.rateLimitPlaceholder")} /> )} @@ -353,10 +355,7 @@ export function UserEditView({ {({ ref, value, onChange, ...control }) => ( onChange(event.target.value)} onWheel={(event) => event.currentTarget.blur()} - placeholder="Unlimited" + placeholder={t("users.editView.fields.rateLimitPlaceholder")} /> )} @@ -385,13 +384,19 @@ export function UserEditView({ onChange={setModelMaxBudget} availableModels={userModels} usage={userData.user_info?.model_max_budget_usage} - hint="Cap this user's spend on individual models, each with its own reset window. Applies across every key the user holds." + hint={t("users.editView.fields.modelMaxBudgetHint")} /> )} - + {({ ref, value, ...control }) => ( -