diff --git a/litellm/__init__.py b/litellm/__init__.py index af812dade3c..88e56c14187 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -207,7 +207,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]] [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] -post_call_rules: List[Callable] = [] +post_call_rules: List[Callable[..., object]] = [] turn_off_message_logging: Optional[bool] = False standard_logging_payload_excluded_fields: Optional[List[str]] = ( None # Fields to exclude from StandardLoggingPayload before callbacks receive it diff --git a/litellm/constants.py b/litellm/constants.py index 271e42ee661..39363cc02e4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -57,6 +57,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "model_group_retry_policy", "model_group_alias", "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", "enable_tag_filtering", "tag_routing_prefix", "optional_pre_call_checks", @@ -146,6 +147,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 # Streams exceeding this duration are terminated with a Timeout error. # None (default) = no limit. Set env var to a number of seconds to enable globally. LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS") +MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0 # Maximum number of base64 characters to keep in logging payloads. # Data URIs exceeding this are replaced with a size placeholder. @@ -1629,6 +1631,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" +CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 31388b00554..b3a0802db13 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1217,6 +1217,7 @@ class MidStreamFallbackError(ServiceUnavailableError): num_retries: int | None = None, generated_content: str = "", is_pre_first_chunk: bool = False, + emitted_disqualifying_content: bool = False, ): original_status: Final = getattr(original_exception, "status_code", None) self.status_code = int(original_status) if original_status is not None else 503 @@ -1229,6 +1230,7 @@ class MidStreamFallbackError(ServiceUnavailableError): self.num_retries = num_retries self.generated_content = generated_content self.is_pre_first_chunk = is_pre_first_chunk + self.emitted_disqualifying_content = emitted_disqualifying_content # Create a response if one wasn't provided if response is None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e766a2be9c3..fa009ca6327 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2295,6 +2295,36 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["combined_usage_object"] = usage self.model_call_details["response_cost"] = response_cost + def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None: + """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw.""" + self.model_call_details["continued_partial_usage"] = usage + self.model_call_details["continued_partial_cost"] = response_cost + self.model_call_details["continued_response_id"] = response_id + + def _add_continued_partial_spend(self, response: object) -> None: + partial_usage: Final = self.model_call_details.get("continued_partial_usage") + partial_cost: Final = self.model_call_details.get("continued_partial_cost") + own_usage: Final = getattr(response, "usage", None) + own_cost: Final = self.model_call_details.get("response_cost") + if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)): + return + from litellm.cost_calculator import BaseTokenUsageProcessor + + self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[own_usage, partial_usage] + ) + if not isinstance(own_cost, (int, float)): + return + self.model_call_details["response_cost"] = own_cost + partial_cost + if self.cost_breakdown is not None: + self.cost_breakdown = self.cost_breakdown | CostBreakdown( + total_cost=own_cost + partial_cost, + additional_costs={ + **self.cost_breakdown.get("additional_costs", {}), + "mid_stream_partial_cost": partial_cost, + }, + ) + def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" usage: Final = getattr(assembled, "usage", None) @@ -2590,6 +2620,15 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"model": model}) + def _with_continued_response_id( + self, + response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: + response_id: Final = self.model_call_details.get("continued_response_id") + if not isinstance(response_id, str) or getattr(response, "id", None) == response_id: + return response + return response.model_copy(update={"id": response_id}) + def _success_handler_helper_fn( self, result: object = None, @@ -2908,8 +2947,11 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = self.response_cost_calculator( result=complete_streaming_response ) + self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response) + logged_streaming_response: Final = self._with_continued_response_id( + self._with_client_facing_stream_model(complete_streaming_response) + ) self.model_call_details["complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( @@ -3445,12 +3487,15 @@ class Logging(LiteLLMLoggingBaseClass): self.model, ) self.model_call_details["response_cost"] = None + self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response) + logged_streaming_response: Final = self._with_continued_response_id( + self._with_client_facing_stream_model(complete_streaming_response) + ) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 81fc865a5bc..044123d8729 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1894,6 +1894,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS) +def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues: + """Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys.""" + return _message_without_fields(message, frozenset({"prefix"})) + + def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: """ Filters a value from a dictionary diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 07534e70a27..87e76fe39f7 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4303,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]]) +def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock: + return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block + + +def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock: + """Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it.""" + if message["role"] != "assistant": + return message + blocks: Final = message["content"] + last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None) + return BedrockMessageBlock( + role="assistant", + content=[ + _rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks) + ], + ) + + def _ensure_document_messages_have_text( contents: list[BedrockMessageBlock], ) -> list[BedrockMessageBlock]: @@ -4894,7 +4912,12 @@ class BedrockConverseMessagesProcessor: llm_provider=llm_provider, ) - return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents)) + converse_messages: Final = _ensure_document_messages_have_text( + _rename_duplicate_bedrock_document_names(contents) + ) + if not (to_claude and converse_messages): + return converse_messages + return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])] @staticmethod def translate_thinking_blocks_to_reasoning_content_blocks( @@ -5303,7 +5326,10 @@ def _bedrock_converse_messages_pt( llm_provider=llm_provider, ) - return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents)) + converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents)) + if not (to_claude and converse_messages): + return converse_messages + return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])] def make_valid_bedrock_tool_name(input_tool_name: str) -> str: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 46ad1b70abe..95d062a190c 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -242,6 +242,7 @@ class CustomStreamWrapper: self.completion_stream = completion_stream self.sent_first_chunk = False self.sent_last_chunk = False + self._failure_logging: asyncio.Task[None] | None = None self._closed = False self._stream_created_time: float = time.time() @@ -269,6 +270,7 @@ class CustomStreamWrapper: self.holding_chunk = "" self.complete_response = "" self.response_uptil_now = "" + self._emitted_disqualifying_content = False _model_info: Final[dict] = litellm_params.model_info or {} _api_base: Final = get_api_base( @@ -1129,6 +1131,7 @@ class CustomStreamWrapper: if self.merge_reasoning_content_in_choices is True: reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None) if reasoning_content: + self._emitted_disqualifying_content = True if self.sent_first_thinking_block is False: # Ensure content is not None before concatenation if model_response.choices[0].delta.content is None: @@ -1865,9 +1868,7 @@ class CustomStreamWrapper: if response.choices: choice = response.choices[0] if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) # HANDLE STREAM OPTIONS self.chunks.append(response) @@ -2084,9 +2085,7 @@ class CustomStreamWrapper: if processed_chunk.choices: choice = processed_chunk.choices[0] if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) # Add mcp_list_tools to first chunk if present if not self.sent_first_chunk and processed_chunk.choices: @@ -2156,9 +2155,7 @@ class CustomStreamWrapper: choice = processed_chunk.choices[0] if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) # RETURN RESULT self.chunks.append(processed_chunk) @@ -2172,7 +2169,7 @@ class CustomStreamWrapper: if self.logging_obj is not None: self._record_partial_usage_for_failure() ## LOGGING - asyncio.create_task( + self._failure_logging = asyncio.create_task( self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True) ) await self.aclose() @@ -2305,11 +2302,17 @@ class CustomStreamWrapper: if self.logging_obj is not None: self._record_partial_usage_for_failure() ## LOGGING - asyncio.create_task( + self._failure_logging = asyncio.create_task( self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True) ) self._handle_stream_fallback_error(e) + async def wait_for_failure_logging(self, timeout: float) -> None: + """Let this stream's failure log read its own deployment before a fallback reuses the logging object.""" + if self._failure_logging is None: + return + await asyncio.wait((self._failure_logging,), timeout=timeout) + def _record_partial_usage_for_failure(self) -> None: """ A stream that breaks mid-flight still billed the provider for the chunks @@ -2346,6 +2349,35 @@ class CustomStreamWrapper: recover_error, ) + # Delta fields a text-only prefill continuation cannot carry, so a stream + # that emitted any of them is not eligible for mid-stream continuation. + _CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = ( + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", + ) + + @classmethod + def _delta_disqualifies_continuation(cls, delta: object) -> bool: + get: Final = getattr(delta, "get", None) + if not callable(get): + return False + return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS) + + def _accumulate_streamed_delta(self, delta: object) -> None: + # Shared by the sync, async, and non-aiohttp iteration sites so answer + # text and the disqualifying-content latch stay in step across all three. + get: Final = getattr(delta, "get", None) + content: Final = get("content") if callable(get) else None + self.response_uptil_now += content if isinstance(content, str) else "" + if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta): + self._emitted_disqualifying_content = True + def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn": """ Common error handling for both __next__ and __anext__. @@ -2377,6 +2409,7 @@ class CustomStreamWrapper: original_exception=e.original_exception, generated_content=self.response_uptil_now, is_pre_first_chunk=not self.sent_first_chunk, + emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content, ) # Map to OpenAI exception format. Some providers' mappers (e.g. @@ -2437,6 +2470,7 @@ class CustomStreamWrapper: original_exception=mapped_exception, generated_content=self.response_uptil_now, is_pre_first_chunk=not self.sent_first_chunk, + emitted_disqualifying_content=self._emitted_disqualifying_content, ) @staticmethod diff --git a/litellm/llms/base_llm/videos/async_task_utils.py b/litellm/llms/base_llm/videos/async_task_utils.py new file mode 100644 index 00000000000..d2fb3ed7606 --- /dev/null +++ b/litellm/llms/base_llm/videos/async_task_utils.py @@ -0,0 +1,123 @@ +import base64 +from collections.abc import Mapping +from io import BufferedReader, BytesIO +from itertools import accumulate, repeat +from pathlib import Path, PurePath +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.images.utils import ImageEditRequestUtils +from litellm.types.videos.utils import decode_video_id_with_provider + +_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _fields(value: object) -> Mapping[str, object] | None: + try: + return _FIELDS_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]: + return _fields(getattr(logging_obj, "litellm_params", None)) or {} + + +def _id_safe_name(name: object) -> str | None: + return name if isinstance(name, str) and name and ";" not in name else None + + +def _stamped_models(metadata: object) -> tuple[str, str | None] | None: + fields: Final = _fields(metadata) + if fields is None: + return None + requested: Final = _id_safe_name(fields.get("model_group")) + served: Final = _id_safe_name(fields.get("deployment_model_name")) + if requested is None: + return None + return requested, None if served is None or served == requested or "*" in served else served + + +def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None: + """ + A later status or content request checks the key's model access against the model encoded in the id, and + routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the + group that served the task can send its polls back to the deployment that created it, while a key may be + scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one + the router was asked for on this hop, and the group that served it when that differs and is not a wildcard. + The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none. + """ + params: Final = _logged_litellm_params(logging_obj) + stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata")) + return next((models for models in stamped if models), None) + + +def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]: + """ + The served group survives into the returned id only when the router sent this poll to it, so an edited id + cannot write any name it likes back. + """ + video_id: Final = _logged_litellm_params(logging_obj).get("video_id") + if not isinstance(video_id, str): + return None, None + decoded: Final = decode_video_id_with_provider(video_id) + served: Final = decoded.get("served_model") + routed: Final = routed_video_models(logging_obj) + return decoded.get("model_id") or None, served if routed is None or served in routed else None + + +def _tuple_payload(file_obj: object) -> object: + if not isinstance(file_obj, tuple): + return file_obj + file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj) + return file_tuple[1] if len(file_tuple) >= 2 else None + + +def _file_content(file_obj: object) -> object: + unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj) + return next(item for item in unwrapped if not isinstance(item, tuple)) + + +def input_reference_bytes(file_obj: object) -> bytes: + content: Final = _file_content(file_obj) + if isinstance(content, PurePath): + return Path(content).read_bytes() + if isinstance(content, (BytesIO, BufferedReader)): + current_position: Final = content.tell() + _ = content.seek(0) + data: Final = content.read() + _ = content.seek(current_position) + return data + if isinstance(content, bytes): + return content + if isinstance(content, bytearray): + return bytes(content) + read: Final = getattr(content, "read", None) + if callable(read): + read_data: Final[object] = read() + if isinstance(read_data, bytes): + return read_data + raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple") + + +def _reference_object_url(reference: Mapping[str, object]) -> str: + image_url: Final = reference.get("image_url") + if isinstance(image_url, str) and image_url: + return image_url + raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported") + + +def input_reference_url(image: object) -> str: + """A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form + carries that URL, or a data URL, in ``image_url``.""" + if isinstance(image, str): + return image + reference: Final = _fields(image) + if reference is not None: + return _reference_object_url(reference) + content: Final = input_reference_bytes(image) + content_type: Final = ImageEditRequestUtils.get_image_content_type(content) + encoded: Final = base64.b64encode(content).decode("utf-8") + return f"data:{content_type};base64,{encoded}" diff --git a/litellm/llms/dashscope/videos/transformation.py b/litellm/llms/dashscope/videos/transformation.py index 17b685a0e1a..1db91cbb127 100644 --- a/litellm/llms/dashscope/videos/transformation.py +++ b/litellm/llms/dashscope/videos/transformation.py @@ -3,13 +3,10 @@ DashScope (Alibaba Model Studio) async video task API: create, poll and download delete or remix endpoint, so those raise. """ -import base64 import json from collections.abc import Mapping from datetime import datetime -from io import BufferedReader, BytesIO from math import log -from pathlib import Path, PurePath from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final @@ -19,9 +16,13 @@ from pydantic import BaseModel, Field, StrictBool, StrictInt, StrictStr, TypeAda import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.images.utils import ImageEditRequestUtils from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.videos.async_task_utils import ( + input_reference_url, + polled_video_models, + routed_video_models, +) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # house cached-client factory has no public alias and its stub leaves params untyped @@ -32,7 +33,6 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelInfo from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject from litellm.types.videos.utils import ( - decode_video_id_with_provider, encode_video_id_with_provider, extract_original_video_id, ) @@ -86,7 +86,6 @@ _EMPTY_PARAMS: Final[dict[str, object]] = {} # mutable-ok: BaseVideoConfig cont _TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_TaskResponse) _PARAMETERS_ADAPTER: Final = TypeAdapter(dict[str, object]) -_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) _STATUS_MAP: Final = MappingProxyType( { @@ -383,39 +382,6 @@ def _is_priced_tier(resolution: object, priced_tiers: tuple[str, ...]) -> bool: return isinstance(resolution, str) and resolution.strip().upper() in priced_tiers -def _read_all_bytes(file_obj: object) -> bytes: - if isinstance(file_obj, tuple): - file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj) - return _read_all_bytes(file_tuple[1] if len(file_tuple) >= 2 else None) - if isinstance(file_obj, PurePath): - return Path(file_obj).read_bytes() - if isinstance(file_obj, (BytesIO, BufferedReader)): - current_position: Final = file_obj.tell() - file_obj.seek(0) - content: Final = file_obj.read() - file_obj.seek(current_position) - return content - if isinstance(file_obj, bytes): - return file_obj - if isinstance(file_obj, bytearray): - return bytes(file_obj) - read: Final = getattr(file_obj, "read", None) - if callable(read): - data: Final = read() - if isinstance(data, bytes): - return data - raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple") - - -def _image_url(image: object) -> str: - if isinstance(image, str): - return image - content: Final = _read_all_bytes(image) - content_type: Final = ImageEditRequestUtils.get_image_content_type(content) - encoded: Final = base64.b64encode(content).decode("utf-8") - return f"data:{content_type};base64,{encoded}" - - def _media_reference_type(model: str) -> str | None: """Models without the field predate the media array and take the image on the flat ``img_url`` field.""" info: Final = _dashscope_model_info(model) @@ -558,58 +524,6 @@ def _video_url_from_task(task: _TaskResponse, raw_response: httpx.Response) -> s raise _video_error(raw_response, 502, "Video URL not found in task response. The task may not have succeeded yet.") -def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]: - litellm_params: Final[object] = getattr(logging_obj, "litellm_params", None) - try: - return _PARAMETERS_ADAPTER.validate_python(litellm_params) - except ValidationError: - return _EMPTY_PARAMS - - -def _polled_models(logging_obj: object) -> tuple[str | None, str | None]: - """ - The proxy routes by the models encoded in the polled id, so they must survive into the returned id. The served - group survives only when the router sent this poll to it, so an edited id cannot write any name it likes back. - """ - video_id: Final = _logged_litellm_params(logging_obj).get("video_id") - if not isinstance(video_id, str): - return None, None - decoded: Final = decode_video_id_with_provider(video_id) - served: Final = decoded.get("served_model") - routed: Final = _poll_models(logging_obj) - return decoded.get("model_id") or None, served if routed is None or served in routed else None - - -def _poll_models(logging_obj: object) -> tuple[str, str | None] | None: - """ - A later status or content request checks the key's model access against the model encoded in the id, and - routes by it. DashScope serves the same model name from several regional hosts, so only the group that - served the task can send its polls back to the region that created it, while a key may be scoped to the - alias, routing group or wildcard name it asked for instead. The id carries both names: the one the router - was asked for on this hop, and the group that served it when that differs and is not a wildcard. The router - stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none. - """ - params: Final = _logged_litellm_params(logging_obj) - stamped: Final = (_stamped_poll_models(params.get(key)) for key in ("litellm_metadata", "metadata")) - return next((models for models in stamped if models), None) - - -def _stamped_poll_models(metadata: object) -> tuple[str, str | None] | None: - try: - fields: Final = _PARAMETERS_ADAPTER.validate_python(metadata) - except ValidationError: - return None - requested: Final = _id_safe_name(fields.get("model_group")) - served: Final = _id_safe_name(fields.get("deployment_model_name")) - if requested is None: - return None - return requested, None if served is None or served == requested or "*" in served else served - - -def _id_safe_name(name: object) -> str | None: - return name if isinstance(name, str) and name and ";" not in name else None - - class DashScopeVideoConfig(BaseVideoConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseVideoConfig contract returns list return [ @@ -804,7 +718,7 @@ class DashScopeVideoConfig(BaseVideoConfig): input_reference: Final = video_create_optional_request_params.get("input_reference") if input_reference is None: return None, None - image_url: Final = _image_url(input_reference) + image_url: Final = input_reference_url(input_reference) media_type: Final = _media_reference_type(model) if media_type is None: return "img_url", image_url @@ -823,7 +737,7 @@ class DashScopeVideoConfig(BaseVideoConfig): requested: Final = self._requested_parameters(request_data) video_obj: Final = _video_object_from_task(task, raw_response, model=model, requested=requested) if custom_llm_provider and video_obj.id: - requested_model, served_model = _poll_models(logging_obj) or (model, None) + requested_model, served_model = routed_video_models(logging_obj) or (model, None) video_obj.id = encode_video_id_with_provider( video_obj.id, custom_llm_provider, requested_model, served_model ) @@ -869,7 +783,7 @@ class DashScopeVideoConfig(BaseVideoConfig): client: "HTTPHandler | None" = None, ) -> VideoObject: task: Final = _parse_task_response(raw_response) - model, served_model = _polled_models(logging_obj) + model, served_model = polled_video_models(logging_obj) video_obj: Final = _video_object_from_task(task, raw_response, model=model) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model, served_model) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 2145c19a71a..74bdd2cd733 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( merge_consecutive_system_messages, strip_litellm_internal_message_fields, strip_name_from_message, + strip_prefill_prefix_field, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( @@ -487,6 +488,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): Databricks does not support: - 'name' in user message. - litellm's internal `thinking_blocks` / `reasoning_content` on assistant messages. + - the `prefix` prefill flag Mistral and DeepSeek read, which Claude does not need to continue an assistant turn. - non-Claude json_object "json" checks read only string content, so a single text block is sent as a string. """ new_messages = [] @@ -496,7 +498,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): else: _message = message _message = strip_name_from_message(_message, allowed_name_roles=["user"]) - _message = strip_litellm_internal_message_fields(_message) + _message = strip_prefill_prefix_field(strip_litellm_internal_message_fields(_message)) # Move message-level cache_control into a content block when content is a string. if "cache_control" in _message and isinstance(_message.get("content"), str): _message = self._move_cache_control_into_string_content_block(_message) diff --git a/litellm/llms/volcengine/videos/__init__.py b/litellm/llms/volcengine/videos/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/volcengine/videos/transformation.py b/litellm/llms/volcengine/videos/transformation.py new file mode 100644 index 00000000000..a75a4185186 --- /dev/null +++ b/litellm/llms/volcengine/videos/transformation.py @@ -0,0 +1,904 @@ +import json +from collections.abc import Mapping, Sequence +from math import log +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeVar + +import httpx +from httpx._types import RequestFiles +from pydantic import BaseModel, StrictBool, StrictInt, StrictStr, TypeAdapter, ValidationError + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.videos.async_task_utils import ( + input_reference_url, + polled_video_models, + routed_video_models, +) +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # house cached-client factory has no public alias and its stub leaves params untyped + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped +) +from litellm.llms.volcengine.common_utils import get_volcengine_base_url +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ModelInfo +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id +from litellm.utils import get_model_info + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + + +_ParsedT: Final = TypeVar("_ParsedT") + + +class VolcEngineVideoError(BaseLLMException): + pass + + +class _TaskError(BaseModel, frozen=True): + code: str | None = None + message: str | None = None + + +class _TaskContent(BaseModel, frozen=True): + video_url: str | None = None + + +class _TaskUsage(BaseModel, frozen=True): + total_tokens: int | None = None + + +class _Task(BaseModel, frozen=True): + id: str + status: str + model: str | None = None + error: _TaskError | None = None + content: _TaskContent | None = None + usage: _TaskUsage | None = None + created_at: int | None = None + updated_at: int | None = None + resolution: str | None = None + ratio: str | None = None + duration: int | float | None = None + + +class _TaskCreateResponse(BaseModel, frozen=True): + id: str | None = None + error: _TaskError | None = None + + +class _TaskList(BaseModel, frozen=True): + items: Sequence[_Task] = () + total: int | None = None + + +class _ContentItem(BaseModel, frozen=True): + type: str | None = None + role: str | None = None + + +class _KnownFields(BaseModel, frozen=True): + resolution: StrictStr | None = None + ratio: StrictStr | None = None + seed: StrictInt | None = None + camera_fixed: StrictBool | None = None + watermark: StrictBool | None = None + generate_audio: StrictBool | None = None + + +VOLCENGINE_VIDEO_TASKS_PATH: Final = "/api/v3/contents/generations/tasks" +VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS: Final = 4 +VOLCENGINE_VIDEO_DEFAULT_RESOLUTION: Final = "720p" +_VOLCENGINE_API_MARKER: Final = "/api/" +_VOLCENGINE_COMPAT_SUFFIXES: Final = ("/api", "/v1") +_FRAMES_FIELD: Final = "volcengine_video_frames" + +_EMPTY_PARAMS: Final[dict[str, object]] = {} # mutable-ok: BaseVideoConfig contract empty params + +_TASK_ADAPTER: Final = TypeAdapter(_Task) +_TASK_CREATE_ADAPTER: Final = TypeAdapter(_TaskCreateResponse) +_TASK_LIST_ADAPTER: Final = TypeAdapter(_TaskList) +_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_CONTENT_ITEMS_ADAPTER: Final = TypeAdapter(tuple[dict[str, object], ...]) +_FRAMES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]]) +_EMPTY_FRAMES: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType({}) + +_STATUS_MAP: Final = MappingProxyType( + { + "queued": "queued", + "running": "in_progress", + "succeeded": "completed", + "failed": "failed", + "cancelled": "cancelled", + "expired": "failed", + } +) +_PENDING_STATUSES: Final = frozenset({"queued", "running"}) +_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "cancelled"}) + +_CREATE_PASSTHROUGH_KEYS: Final = ( + "resolution", + "ratio", + "duration", + "seed", + "camera_fixed", + "watermark", + "generate_audio", + "service_tier", + "priority", + "output_format", + "execution_expires_after", + "callback_url", + "safety_identifier", + "tools", + "omni_reference_task_type", +) + +_REQUEST_METADATA: Final = frozenset({"model", "prompt", "user", "extra_headers", "extra_query", "extra_body"}) +_UNSUPPORTED_BY_VOLCENGINE: Final = frozenset({"characters", "image"}) +_MAPPED_ELSEWHERE: Final = frozenset({*_REQUEST_METADATA, *_UNSUPPORTED_BY_VOLCENGINE, "seconds", "size", "parameters"}) +_READ_FROM_THE_REQUEST: Final = _MAPPED_ELSEWHERE - _UNSUPPORTED_BY_VOLCENGINE +_LITELLM_CONTROLS: Final = frozenset({"drop_params", "additional_drop_params", "organization"}) +_DROP_KEYS_ADAPTER: Final = TypeAdapter(frozenset[str]) + +# litellm/videos/utils.py merges the caller's raw ``extra_body`` back over the mapped params, which returns the keys +# this adapter maps itself a second time, as fields of their own. They are already accounted for, so they count as +# handled rather than unknown. +_MAPPED_EXTRA_BODY_COPIES: Final = frozenset({"size", "seconds", "user"}) +_CREATE_HANDLED_KEYS: Final = frozenset( + { + *_CREATE_PASSTHROUGH_KEYS, + "content", + "input_reference", + "frames", + *_MAPPED_EXTRA_BODY_COPIES, + *_LITELLM_CONTROLS, + } +) + +_VIDEO_INPUT_UNSUPPORTED: Final = ( + "{operation} is not supported through litellm: Ark bills a task with a video input at a different rate per " + "token, and litellm bills the video when the task is created, before that cost is known" +) +_INPUT_VIDEO_ROLE: Final = "reference_video" +_INPUT_VIDEO_TYPE: Final = "video_url" + +_RATIO_SET: Final = ("16:9", "9:16", "4:3", "3:4", "1:1", "21:9") +_RESOLUTION_SHORTEST_SIDE: Final = MappingProxyType({"480p": 480, "720p": 720, "1080p": 1080, "4k": 2160}) +_TIER_FRAME_ASPECT: Final = 16 / 9 + + +def _video_error(raw_response: httpx.Response, status_code: int, message: str) -> VolcEngineVideoError: + """The provider's own error status wins, and a body that failed on a 2xx answers the caller's status instead, + since a 2xx error has the proxy retry a call the provider already rejected.""" + return VolcEngineVideoError( + status_code=raw_response.status_code if raw_response.status_code >= 400 else status_code, + message=message, + headers=raw_response.headers, + ) + + +def _parse(adapter: TypeAdapter[_ParsedT], raw_response: httpx.Response) -> _ParsedT: + try: + return adapter.validate_python(raw_response.json()) + except (json.JSONDecodeError, ValidationError) as e: + raise _video_error(raw_response, 502, f"Unexpected Volcengine video response: {e}") from e + + +def _unsupported(model: str, message: str) -> UnsupportedParamsError: + return UnsupportedParamsError(message=message, model=model, llm_provider="volcengine") + + +def _droppable(model: str, message: str) -> UnsupportedParamsError: + return _unsupported(model, f"{message}, set drop_params=True to ignore it") + + +def _normalized_api_base(api_base: str) -> str: + """The Ark credential is shared with the chat, responses and Anthropic surfaces, so its api_base may carry any of + their paths, and the video paths carry their own version segment.""" + trimmed: Final = api_base.rstrip("/") + marker_index: Final = trimmed.rfind(_VOLCENGINE_API_MARKER) + if marker_index != -1: + return trimmed[:marker_index] + suffix: Final = next((suffix for suffix in _VOLCENGINE_COMPAT_SUFFIXES if trimmed.endswith(suffix)), None) + return trimmed[: -len(suffix)] if suffix is not None else trimmed + + +def _size_dimensions(size: str) -> tuple[int, int] | None: + width_str, separator, height_str = size.partition("x") + if not separator or not (width_str.isdigit() and height_str.isdigit()): + return None + if not int(width_str) or not int(height_str): + return None + return int(width_str), int(height_str) + + +def _ratio_value(ratio: str) -> float: + width_str, _, height_str = ratio.partition(":") + return int(width_str) / int(height_str) + + +def _ratio_from_size(size: str) -> str | None: + """Ark only takes its named ratios, and its own tiers do not reduce onto them (854x480 is 427:240), so the + closest ratio wins.""" + dimensions: Final = _size_dimensions(size) + if dimensions is None: + return None + width, height = dimensions + return min(_RATIO_SET, key=lambda candidate: abs(log(width / height / _ratio_value(candidate)))) + + +def _resolution_from_size(size: str) -> str | None: + """Ark keeps a tier's pixel count across its ratios (960x960 and 1280x720 are both 720p), so the tier whose 16:9 + frame has the closest pixel count wins.""" + dimensions: Final = _size_dimensions(size) + if dimensions is None: + return None + width, height = dimensions + return min( + _RESOLUTION_SHORTEST_SIDE, + key=lambda tier: abs(log(width * height / (_RESOLUTION_SHORTEST_SIDE[tier] ** 2 * _TIER_FRAME_ASPECT))), + ) + + +def _duration_param(seconds: object) -> int | None: + if isinstance(seconds, bool): + return None + if isinstance(seconds, int): + return seconds + if not isinstance(seconds, (str, float)): + return None + try: + parsed: Final = float(seconds) + except ValueError: + return None + return int(parsed) if parsed.is_integer() else None + + +def _duration_fields(seconds: object, model: str, drop_params: bool) -> Mapping[str, object]: + if seconds is None: + return _EMPTY_PARAMS + duration: Final = _duration_param(seconds) + if duration is not None: + return MappingProxyType({"duration": duration}) + if drop_params: + return _EMPTY_PARAMS + raise _droppable(model, f"seconds must be a whole number of seconds, got {seconds!r}") + + +def _size_fields(size: object, model: str, drop_params: bool) -> Mapping[str, object]: + if size is None: + return _EMPTY_PARAMS + ratio: Final = _ratio_from_size(size) if isinstance(size, str) else None + resolution: Final = _resolution_from_size(size) if isinstance(size, str) else None + if ratio is not None and resolution is not None: + return MappingProxyType({"ratio": ratio, "resolution": resolution}) + if drop_params: + return _EMPTY_PARAMS + raise _droppable(model, f"size must be WIDTHxHEIGHT, got {size!r}") + + +def _parameters_fields(parameters: object, model: str, drop_params: bool) -> Mapping[str, object]: + if parameters is None: + return _EMPTY_PARAMS + try: + fields: Final = _FIELDS_ADAPTER.validate_python(parameters) + except ValidationError: + if drop_params: + return _EMPTY_PARAMS + raise _droppable( + model, f"parameters must be an object of Ark request fields, got {type(parameters).__name__}" + ) from None + misplaced: Final = tuple(sorted(key for key in fields if key in _READ_FROM_THE_REQUEST)) + if misplaced and not drop_params: + raise _droppable(model, f"send {', '.join(misplaced)} as top-level request fields, not inside parameters") + return MappingProxyType({key: value for key, value in fields.items() if key not in _READ_FROM_THE_REQUEST}) + + +def _user_fields(user: object, model: str, drop_params: bool) -> Mapping[str, object]: + if isinstance(user, str) and user: + return MappingProxyType({"safety_identifier": user}) + if user is None or isinstance(user, str) or drop_params: + return _EMPTY_PARAMS + raise _droppable(model, f"user must be a string, got {type(user).__name__}") + + +def _additional_drop_keys(params: Mapping[str, object]) -> frozenset[str]: + try: + return _DROP_KEYS_ADAPTER.validate_python(params.get("additional_drop_params") or ()) + except ValidationError: + return frozenset() + + +def _without(params: Mapping[str, object], keys: frozenset[str]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in params.items() if key not in keys}) + + +def _malformed_fields(fields: Mapping[str, object]) -> tuple[tuple[str, str], ...]: + try: + _ = _KnownFields.model_validate(dict(fields)) + except ValidationError as e: + return tuple((str(error["loc"][0]), error["msg"]) for error in e.errors() if error["loc"]) + return () + + +def _drops_params(litellm_params: GenericLiteLLMParams) -> bool: + return litellm_params.drop_params is True or litellm.drop_params is True + + +def _model_info(model: str) -> ModelInfo | None: + try: + return get_model_info(model=model, custom_llm_provider="volcengine") + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + return None + + +def _pricing_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + """A deployment priced through ``model_info.base_model``, like an ``ep-`` endpoint id, bills at that model's + tiers.""" + model_info: Final = _FIELDS_ADAPTER.validate_python(litellm_params["model_info"] or _EMPTY_PARAMS) + base_model: Final = model_info.get("base_model") + return base_model if isinstance(base_model, str) and base_model else model + + +def _priced_tiers(model: str) -> tuple[str, ...] | None: + """A tier the model has no price for is one Ark does not offer it at, and litellm would bill it at the base rate. + A model with no tier price listed, missing from the cost map or registered unpriced by the router, has nothing + to check against.""" + info: Final = _model_info(model) + if info is None: + return None + return ( + tuple(tier for tier in _RESOLUTION_SHORTEST_SIDE if info.get(f"output_cost_per_second_{tier}") is not None) + or None + ) + + +def _delivered_frames(model: str | None) -> Mapping[str, Mapping[str, str]]: + info: Final = _model_info(model) if model else None + provider_specific: Final[object] = info.get("provider_specific_entry") if info is not None else None + frames: Final[object] = provider_specific.get(_FRAMES_FIELD) if isinstance(provider_specific, Mapping) else None + try: + return _FRAMES_ADAPTER.validate_python(frames) if frames is not None else _EMPTY_FRAMES + except ValidationError: + return _EMPTY_FRAMES + + +def _size_from_geometry(resolution: str | None, ratio: str | None, model: str | None) -> str | None: + """Ark echoes a tier and a ratio rather than pixels, and its rendered frame does not always follow the tier's + nominal size, so the frame listed for the model wins over scaling the tier's shortest side up the ratio.""" + if not resolution or not ratio or ":" not in ratio: + return None + tier: Final = resolution.strip().lower() + normalized_ratio: Final = ratio.strip() + listed: Final = _delivered_frames(model).get(tier, MappingProxyType({})).get(normalized_ratio) + if listed is not None: + return listed + shortest_side: Final = _RESOLUTION_SHORTEST_SIDE.get(tier) + width_str, _, height_str = normalized_ratio.partition(":") + if shortest_side is None or not (width_str.isdigit() and height_str.isdigit()): + return None + ratio_width: Final = int(width_str) + ratio_height: Final = int(height_str) + if not ratio_width or not ratio_height: + return None + shortest_ratio_side: Final = min(ratio_width, ratio_height) + return f"{-(-shortest_side * ratio_width // shortest_ratio_side)}x{-(-shortest_side * ratio_height // shortest_ratio_side)}" + + +def _is_input_video(item: _ContentItem) -> bool: + return item.type == _INPUT_VIDEO_TYPE or item.role == _INPUT_VIDEO_ROLE + + +def _supplied_items(supplied: object, model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]: + try: + items: Final = _CONTENT_ITEMS_ADAPTER.validate_python(supplied) + entries: Final = tuple(_ContentItem.model_validate(item) for item in items) + except ValidationError: + if drop_params: + return () + raise _droppable(model, "content must be an array of Ark content items") from None + if any(entry.type == "text" for entry in entries) and not drop_params: + raise _droppable( + model, + "A text item in content is not supported, put the text in prompt since it is the only text input " + "guardrails scan before the request reaches Ark", + ) + if any(_is_input_video(entry) for entry in entries) and not drop_params: + raise _droppable(model, _VIDEO_INPUT_UNSUPPORTED.format(operation="Video input (a reference_video item)")) + kept: Final = (item for item, entry in zip(items, entries) if entry.type != "text" and not _is_input_video(entry)) + return tuple(MappingProxyType(item) for item in kept) + + +def _reference_items(params: Mapping[str, object], model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]: + """A caller-supplied ``content`` array wins over ``input_reference``, since its roles are the only way to express + reference-to-video.""" + supplied: Final = params.get("content") + if supplied is not None: + return _supplied_items(supplied, model, drop_params) + reference: Final = params.get("input_reference") + if reference is None: + return () + try: + url: Final = input_reference_url(reference) + except ValueError as e: + if drop_params: + return () + raise _droppable(model, str(e)) from None + return (MappingProxyType({"type": "image_url", "image_url": {"url": url}, "role": "first_frame"}),) + + +def _sent_content( + prompt: str, params: Mapping[str, object], model: str, drop_params: bool +) -> tuple[Mapping[str, object], ...]: + """The prompt always leads, as it is the only text the input guardrails scan.""" + return (MappingProxyType({"type": "text", "text": prompt}), *_reference_items(params, model, drop_params)) + + +def _seconds_label(seconds: object) -> str | None: + if isinstance(seconds, float) and seconds.is_integer(): + return str(int(seconds)) + return None if seconds is None else str(seconds) + + +def _requested_usage(request_data: Mapping[str, object]) -> Mapping[str, object]: + """The create call is the only billed one and Ark's create response carries no usage, so the requested duration + and tier are what the video bills at.""" + duration: Final = _duration_param(request_data.get("duration")) + resolution: Final = request_data.get("resolution") + return MappingProxyType( + { + key: value + for key, value in ( + ("duration_seconds", float(duration) if duration is not None and duration > 0 else None), + ("video_resolution", resolution.strip().lower() if isinstance(resolution, str) else None), + ) + if value is not None + } + ) + + +def _task_usage(task: _Task) -> Mapping[str, object]: + resolution: Final = task.resolution.strip().lower() if task.resolution else None + return MappingProxyType( + { + key: value + for key, value in ( + ("duration_seconds", float(task.duration) if task.duration is not None else None), + ("video_resolution", resolution), + ("total_tokens", task.usage.total_tokens if task.usage is not None else None), + ) + if value is not None + } + ) + + +def _task_status(task: _Task, raw_response: httpx.Response) -> str: + status: Final = _STATUS_MAP.get(task.status.strip().lower()) + if status is None: # defaulting it would hand back a job the caller polls forever + raise _video_error(raw_response, 502, f"Unknown Volcengine task status: {task.status}") + return status + + +def _video_object_from_task(task: _Task, raw_response: httpx.Response, model: str | None) -> VideoObject: + status: Final = _task_status(task, raw_response) + usage: Final = _task_usage(task) + seconds: Final = usage.get("duration_seconds") + error: Final[dict[str, object] | None] = ( # mutable-ok: VideoObject.error is a dict field + {key: value for key, value in (("code", task.error.code), ("message", task.error.message)) if value is not None} + if task.error is not None + else None + ) + return VideoObject( + id=task.id, + object="video", + status=status, + created_at=task.created_at, + completed_at=task.updated_at if status in _TERMINAL_STATUSES else None, + error=error, + seconds=_seconds_label(seconds), + size=_size_from_geometry(task.resolution, task.ratio, task.model), + model=model or task.model, + usage=dict(usage) if usage else None, + ) + + +def _video_url_from_task(task: _Task, raw_response: httpx.Response) -> str: + """A download before the video is ready is the caller polling too early, not a fault of the deployment, so it + answers 400, which litellm neither retries nor counts toward a deployment cooldown. A 404 would cool the + deployment down, and a 409 or a 5xx would be retried against a job that is simply not finished.""" + if task.content is not None and task.content.video_url: + return task.content.video_url + status: Final = task.status.strip().lower() + if status in _PENDING_STATUSES: + raise _video_error( + raw_response, 400, f"Video is still processing (status: {status}). Please wait and try again." + ) + if task.error is not None and (task.error.message or task.error.code): + raise _video_error(raw_response, 400, f"Video generation failed: {task.error.message or task.error.code}") + if status == "expired": + raise _video_error(raw_response, 400, "Task expired. Ark keeps task records for 7 days after creation.") + if status in ("failed", "cancelled"): + raise _video_error(raw_response, 400, f"Video generation ended as {status} with no video URL.") + raise _video_error(raw_response, 502, "Video URL not found in task response.") + + +class VolcEngineVideoConfig(BaseVideoConfig): + """Volcengine Ark (Doubao-Seedance) on Ark's async task API: create, poll, download from the signed + ``content.video_url``, and list.""" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseVideoConfig contract returns list + return [ + "model", + "prompt", + "input_reference", + "seconds", + "size", + "user", + "extra_headers", + "content", + "parameters", + *_CREATE_PASSTHROUGH_KEYS, + ] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: BaseVideoConfig contract + requested: Final = _without(video_create_optional_params, _additional_drop_keys(video_create_optional_params)) + unsupported: Final = next( + (key for key, value in requested.items() if key in _UNSUPPORTED_BY_VOLCENGINE and value is not None), + None, + ) + if unsupported is not None and not drop_params: + raise _droppable(model, f"'{unsupported}' is not supported by Volcengine video generation") + passthrough: Final = MappingProxyType( + {key: value for key, value in requested.items() if value is not None and key not in _MAPPED_ELSEWHERE} + ) + return { + **_user_fields(requested.get("user"), model, drop_params), + **_size_fields(requested.get("size"), model, drop_params), + **_duration_fields(requested.get("seconds"), model, drop_params), + **passthrough, + **_parameters_fields(requested.get("parameters"), model, drop_params), + } + + def _resolve_api_key(self, api_key: str | None) -> str: + resolved_api_key: Final = api_key or get_secret_str("ARK_API_KEY") or get_secret_str("VOLCENGINE_API_KEY") + if resolved_api_key is None: + raise ValueError( + "Volcengine API key is required. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass " + "api_key parameter." + ) + return resolved_api_key + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract; handler expects a mutable headers dict + model: str, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> dict[str, str]: # mutable-ok: BaseVideoConfig contract + resolved_api_key: Final = self._resolve_api_key( + api_key + or (litellm_params.api_key if litellm_params is not None and litellm_params.api_key else None) + or litellm.api_key + ) + headers.update({"Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}) + return headers + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: BaseVideoConfig contract + ) -> str: + return _normalized_api_base( + get_volcengine_base_url(api_base or get_secret_str("VOLCENGINE_API_BASE") or get_secret_str("ARK_API_BASE")) + ) + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: dict[str, object], # mutable-ok: BaseVideoConfig contract + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + ) -> tuple[dict[str, object], RequestFiles, str]: # mutable-ok: BaseVideoConfig contract + """ + Ark bills a video when the task finishes, while litellm bills it here at create, so every request has to + carry a length and a tier whose cost is known now. Ark-picked length (``duration: -1``) and a video input + (edits, extensions and reference videos, billed at a different rate per token) are rejected for that reason, + and an omitted length is sent as OpenAI's documented default of 4 seconds, since the 2.5 family's own default + is ``-1``. + """ + drop_params: Final = _drops_params(litellm_params) + dropped_by_config: Final = _additional_drop_keys(video_create_optional_request_params) + kept: Final = _without(video_create_optional_request_params, dropped_by_config) + params: Final = _without( + { + **{key: value for key, value in kept.items() if key != "parameters"}, + **_parameters_fields(kept.get("parameters"), model, drop_params), + }, + dropped_by_config, + ) + content: Final = _sent_content(prompt, params, model, drop_params) + if params.get("frames") is not None and not drop_params: + raise _droppable(model, "frames is not supported through litellm, pass seconds instead") + requested_duration: Final = params.get("duration") + parsed_duration: Final = None if requested_duration is None else _duration_param(requested_duration) + if requested_duration is not None and (parsed_duration is None or parsed_duration <= 0) and not drop_params: + raise _droppable( + model, + f"duration must be a positive whole number of seconds, got {requested_duration!r}. Ark-picked length " + "(-1) is not supported through litellm, since the video is billed when the task is created, before " + "Ark has picked its length", + ) + duration: Final = ( + parsed_duration + if parsed_duration is not None and parsed_duration > 0 + else VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS + ) + + unknown_fields: Final = tuple( + sorted(key for key, value in params.items() if value is not None and key not in _CREATE_HANDLED_KEYS) + ) + if unknown_fields and not drop_params: + raise _droppable(model, f"litellm does not send {', '.join(unknown_fields)} to Ark video generation") + requested_fields: Final = MappingProxyType( + {key: params[key] for key in _CREATE_PASSTHROUGH_KEYS if params.get(key) is not None} + ) + malformed: Final = _malformed_fields(requested_fields) + if malformed and not drop_params: + raise _droppable(model, "; ".join(f"invalid {field}: {message}" for field, message in malformed)) + resolution: Final = requested_fields.get("resolution") + priced_tiers: Final = _priced_tiers(_pricing_model(model, litellm_params)) + drops_resolution: Final = ( + priced_tiers is not None and isinstance(resolution, str) and resolution.strip().lower() not in priced_tiers + ) + if drops_resolution and not drop_params: + raise _droppable( + model, f"{model} is not offered at resolution {resolution!r}, only at {', '.join(priced_tiers or ())}" + ) + dropped: Final = frozenset({*(field for field, _ in malformed), *(("resolution",) if drops_resolution else ())}) + kept_fields: Final = MappingProxyType( + {key: value for key, value in requested_fields.items() if key not in dropped} + ) + default_tier: Final = ( + MappingProxyType({"resolution": VOLCENGINE_VIDEO_DEFAULT_RESOLUTION}) + if "resolution" not in kept_fields + and priced_tiers is not None + and VOLCENGINE_VIDEO_DEFAULT_RESOLUTION in priced_tiers + else _EMPTY_PARAMS + ) + + request_data: Final[dict[str, object]] = { # mutable-ok: request body dict, JSON-serialized by the handler + "model": model, + "content": [dict(item) for item in content], + **default_tier, + **kept_fields, + "duration": duration, + } + return request_data, (), f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}" + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + request_data: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> VideoObject: + task: Final = _parse(_TASK_CREATE_ADAPTER, raw_response) + if not task.id: + message: Final = task.error.message if task.error is not None and task.error.message else "no task id" + raise _video_error(raw_response, 400, f"Volcengine did not return a video task id: {message}") + requested: Final = request_data or _EMPTY_PARAMS + usage: Final = _requested_usage(requested) + seconds: Final = usage.get("duration_seconds") + requested_model, served_model = routed_video_models(logging_obj) or (model, None) + resolution: Final = requested.get("resolution") + ratio: Final = requested.get("ratio") + return VideoObject( + id=encode_video_id_with_provider(task.id, custom_llm_provider, requested_model, served_model) + if custom_llm_provider + else task.id, + object="video", + status="queued", + model=model, + seconds=_seconds_label(seconds), + size=_size_from_geometry( + resolution if isinstance(resolution, str) else None, ratio if isinstance(ratio, str) else None, model + ), + usage=dict(usage) if usage else None, + ) + + @staticmethod + def _task_url(video_id: str, api_base: str) -> str: + original_task_id: Final = extract_original_video_id(video_id) + encoded_task_id: Final = encode_url_path_segment(original_task_id, field_name="video_id") + return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}/{encoded_task_id}" + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + return self._task_url(video_id, api_base), _EMPTY_PARAMS + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, + ) -> VideoObject: + task: Final = _parse(_TASK_ADAPTER, raw_response) + model, served_model = polled_video_models(logging_obj) + video_obj: Final = _video_object_from_task(task, raw_response, model) + if custom_llm_provider: + video_obj.id = encode_video_id_with_provider(task.id, custom_llm_provider, model, served_model) + return video_obj + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + variant: str | None = None, + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + return self._task_url(video_id, api_base), _EMPTY_PARAMS + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + client: "HTTPHandler | None" = None, + ) -> bytes: + video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response) + httpx_client: Final[HTTPHandler] = client if client is not None else _get_httpx_client() + video_response: Final = httpx_client.get(video_url) # pyright: ignore[reportUnknownMemberType] # HTTPHandler.get stub leaves params/headers untyped + video_response.raise_for_status() + return video_response.content + + async def async_transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + client: "AsyncHTTPHandler | None" = None, + ) -> bytes: + video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response) + async_httpx_client: Final[AsyncHTTPHandler] = ( + client if client is not None else get_async_httpx_client(llm_provider=litellm.LlmProviders.VOLCENGINE) + ) + video_response: Final = await async_httpx_client.get(video_url) # pyright: ignore[reportUnknownMemberType] # HTTPHandler.get stub leaves params/headers untyped + video_response.raise_for_status() + return video_response.content + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + extra_body: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video remix")) + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("Video remix is not supported by Volcengine.") + + def transform_video_edit_request( + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + video_file: object | None = None, + extra_body: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + prefetched_source_data: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, Mapping[str, object], RequestFiles | None]: + raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video edit")) + + def transform_video_extension_request( + self, + prompt: str, + video_id: str, + seconds: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + extra_body: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video extension")) + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + """Ark pages by page_num/page_size with filters and has no cursor or order.""" + unsupported: Final = tuple(name for name, value in (("after", after), ("order", order)) if value is not None) + if unsupported and not _drops_params(litellm_params): + raise _droppable( + "", f"Ark's task list pages by page_num in extra_query and has no {' or '.join(unsupported)}" + ) + return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}", { + **({"page_size": str(limit)} if limit is not None else {}), + **(extra_query or {}), + } + + def transform_video_list_response( # pyright: ignore[reportIncompatibleMethodOverride] # base declares dict[str, str] but the payload is a heterogeneous list body + self, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + ) -> dict[str, object]: # mutable-ok: OpenAI list body served as JSON by the proxy + list_payload: Final = _parse(_TASK_LIST_ADAPTER, raw_response) + videos: Final = tuple(_video_object_from_task(task, raw_response, None) for task in list_payload.items) + data: Final = [ + video.model_dump() + | ( + {"id": encode_video_id_with_provider(video.id, custom_llm_provider, video.model)} + if custom_llm_provider + else {} + ) + for video in videos + ] + return { + "object": "list", + "data": data, + "total": list_payload.total if list_payload.total is not None else len(data), + "has_more": False, + **({"first_id": data[0]["id"], "last_id": data[-1]["id"]} if data else {}), + } + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + raise NotImplementedError("Video delete is not supported by Volcengine.") + + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + ) -> VideoObject: + raise NotImplementedError("Video delete is not supported by Volcengine.") + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseVideoConfig contract + ) -> BaseLLMException: + return VolcEngineVideoError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b744032fb6c..49d2ce90af4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2773,7 +2773,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2808,7 +2808,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2843,7 +2843,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2878,7 +2878,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2913,7 +2913,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2948,7 +2948,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -3743,7 +3743,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -15494,7 +15494,7 @@ }, "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -20832,7 +20832,7 @@ "output_cost_per_token": 2.5000010000000002e-05, "output_dbu_cost_per_token": 0.000357143, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -21047,7 +21047,7 @@ "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -49176,7 +49176,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -51006,7 +51006,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -51179,7 +51179,7 @@ "supported_regions": [ "global" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -51200,7 +51200,7 @@ "supported_regions": [ "us-central1" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -58822,7 +58822,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -60467,6 +60467,61 @@ } ] }, + "volcengine/doubao-seedance-2-5-260628": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.216, + "output_cost_per_second_480p": 0.096075, + "output_cost_per_second_720p": 0.216, + "output_cost_per_second_1080p": 0.5346, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.141942857, + "output_cost_per_second_480p": 0.066003429, + "output_cost_per_second_720p": 0.141942857, + "output_cost_per_second_1080p": 0.354085714, + "output_cost_per_second_4k": 0.722057143, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-fast-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.114171429, + "output_cost_per_second_480p": 0.053089714, + "output_cost_per_second_720p": 0.114171429, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-mini-260615": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.070971429, + "output_cost_per_second_480p": 0.033001714, + "output_cost_per_second_720p": 0.070971429, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -72334,7 +72389,7 @@ "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6bd31a42e9e..675bf0830f9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -27,6 +27,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm._uuid import uuid from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -187,6 +188,30 @@ def _caller_trace_field(data: Mapping[str, object], metadata_variable_name: str, return requester_value if isinstance(requester_value, str) and requester_value else None +def _caller_body_session_id(data: Mapping[str, object], metadata_variable_name: str) -> str | None: + top_level: Final = data.get("litellm_session_id") + if isinstance(top_level, str) and top_level: + return top_level + return _caller_trace_field(data, metadata_variable_name, "session_id") + + +def _caller_session_beside_trace_header( + normalized_headers: Mapping[str, str], data: Mapping[str, object], metadata_variable_name: str +) -> str | None: + session_headers: Final = {key: value for key, value in normalized_headers.items() if key != "x-litellm-trace-id"} + return get_chain_id_from_headers(session_headers) or _caller_body_session_id(data, metadata_variable_name) + + +def _drop_client_caller_session(request_body: Mapping[str, object]) -> None: + for metadata_key in ("metadata", "litellm_metadata"): + if isinstance(metadata := request_body.get(metadata_key), dict): + metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None) + + +def _requester_metadata_snapshot(metadata: Mapping[str, object]) -> Mapping[str, object]: + return copy.deepcopy({key: value for key, value in metadata.items() if key != CALLER_SESSION_ID_METADATA_KEY}) + + def _stampable_key_hash(user_api_key_dict: UserAPIKeyAuth) -> str | None: """Only proxy-validated keys are stamped, proven by the unforgeable via_virtual_key marker AND a known non-secret shape: the sha256 hex digest @@ -1623,6 +1648,8 @@ class LiteLLMProxyRequestSetup: from litellm.proxy._types import LitellmMetadataFromRequestHeaders metadata_from_headers: Final = LitellmMetadataFromRequestHeaders() + request_body: Final = cast(Mapping[str, object], data) # cast-ok: request body is a str-keyed JSON object + _drop_client_caller_session(request_body) spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) if spend_logs_metadata is not None: metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata @@ -1631,6 +1658,9 @@ class LiteLLMProxyRequestSetup: # Finally update the requests metadata with the `metadata_from_headers` ######################################################################################### + normalized_headers: Final = MappingProxyType( + {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)} + ) agent_id_from_header: Final = headers.get("x-litellm-agent-id") # Explicit litellm headers take precedence; fall back to any x-*-session-id header. chain_id: Final = get_chain_id_from_headers(dict(headers)) @@ -1639,6 +1669,13 @@ class LiteLLMProxyRequestSetup: metadata_from_headers["agent_id"] = agent_id_from_header verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header) + spend_log_session_id: Final = ( + _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name) + if litellm.request_correlation_in_logs + and chain_id is not None + and chain_id == normalized_headers.get("x-litellm-trace-id") + else None + ) if chain_id: metadata_from_headers["trace_id"] = chain_id metadata_from_headers["session_id"] = chain_id @@ -1663,7 +1700,6 @@ class LiteLLMProxyRequestSetup: # - but lets a caller's existing traceparent/baggage headers (from # real OTel instrumentation) correlate with litellm's own logs instead # of generating an unrelated trace_id. - normalized_headers: Final = MappingProxyType({k.lower(): v for k, v in headers.items() if isinstance(k, str)}) if "litellm_trace_id" not in data: traceparent: Final = normalized_headers.get("traceparent") if isinstance(traceparent, str): @@ -1689,11 +1725,7 @@ class LiteLLMProxyRequestSetup: baggage: Final = normalized_headers.get("baggage") if isinstance(baggage, str): session_id_from_baggage: Final = _session_id_from_baggage(baggage) - caller_session_id: Final = _caller_trace_field( - cast(Mapping[str, object], data), # cast-ok: request body is a str-keyed JSON object - _metadata_variable_name, - "session_id", - ) + caller_session_id: Final = _caller_trace_field(request_body, _metadata_variable_name, "session_id") if session_id_from_baggage and ( caller_session_id is None or caller_session_id == session_id_from_baggage ): @@ -1702,7 +1734,11 @@ class LiteLLMProxyRequestSetup: verbose_proxy_logger.debug("Extracted session_id from W3C baggage header") if isinstance(data[_metadata_variable_name], dict): - data[_metadata_variable_name].update(metadata_from_headers) + data[_metadata_variable_name].update( + {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id} + if spend_log_session_id + else metadata_from_headers + ) return data @staticmethod @@ -2417,7 +2453,7 @@ async def add_litellm_data_to_request( # them — from leaking into requester_metadata where guardrails and audit # paths may read from it. if "metadata" in data and isinstance(data["metadata"], dict): - data[_metadata_variable_name]["requester_metadata"] = copy.deepcopy(data["metadata"]) + data[_metadata_variable_name]["requester_metadata"] = _requester_metadata_snapshot(data["metadata"]) if _metadata_variable_name == "litellm_metadata": data[_metadata_variable_name].update( _promoted_trace_control_fields( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 49644f1aa12..9ebea589cc9 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -33,6 +34,8 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) +from litellm.integrations.otel.model.metadata import caller_session_id +from litellm.integrations.otel.model.trace_controls import TraceControls from litellm.litellm_core_utils.classifier_logging import ( CLASSIFIER_AUDIT_FIELDS, classifier_audit_fields, @@ -570,11 +573,11 @@ def get_logging_payload( elif isinstance(_usage, dict): usage = _usage - # A request that failed mid-stream has no usable response_obj usage, but the - # streaming handler may have recovered the usage from the chunks already - # delivered. Honor that override so the partial usage lands in spend tracking. + # A request that failed mid-stream has no usable response_obj usage, and a stream + # continued on a fallback only has the fallback's. combined_usage_object holds the + # usage of every call the client's answer took, so it wins in both cases. _combined_usage: Final = kwargs.get("combined_usage_object") - if not usage and isinstance(_combined_usage, litellm.Usage): + if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage): usage = _combined_usage.model_dump() id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata) @@ -930,6 +933,16 @@ def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) - return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX) +def _caller_session_for_spend_log(kwargs: Mapping[str, object], metadata: Mapping[str, object] | None) -> str | None: + beside_trace_header: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) if metadata else None + caller_session: Final = ( + beside_trace_header + if isinstance(beside_trace_header, str) and beside_trace_header + else caller_session_id(kwargs, TraceControls()) + ) + return caller_session if caller_session and "\x00" not in caller_session else None + + def _get_session_id_for_spend_log( kwargs: Mapping[str, object], metadata: Mapping[str, object] | None, @@ -937,9 +950,12 @@ def _get_session_id_for_spend_log( omit_when_missing: bool, batch_trace_session_id: str | None = None, ) -> str | None: - """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may - be a copied trace id. Batch call types carry a deterministic session derived from the batch id, which outranks - the per-request trace ids because those differ between the create call and the cost poller's row.""" + """Resolve the session id for the spend log row: `omit` honors only metadata.session_id, + batch sessions outrank everything, then the conversation id the caller sent (also one an x-litellm-trace-id header + displaced from the request's session), then trace ids. + + A session minted by `missing_session_id: generate` or copied from the request's trace id is not a conversation, so + it yields to the trace id, which the Responses bridge points at the previous turn's session to keep a chain whole.""" if omit_when_missing: session_id: Final = metadata.get("session_id") if metadata else None return str(session_id) if session_id else None @@ -948,11 +964,14 @@ def _get_session_id_for_spend_log( if batch_trace_session_id is not None: return batch_trace_session_id - if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None: - return str(standard_logging_payload.get("trace_id")) - if kwargs.get("litellm_trace_id") is not None: - return str(kwargs.get("litellm_trace_id")) - return str(uuid.uuid4()) + if litellm.request_correlation_in_logs and (caller_session := _caller_session_for_spend_log(kwargs, metadata)): + return caller_session + match standard_logging_payload: + case {"trace_id": trace_id} if trace_id is not None: + return str(trace_id) + case _: + litellm_trace_id: Final = kwargs.get("litellm_trace_id") + return str(litellm_trace_id) if litellm_trace_id is not None else str(uuid.uuid4()) def _get_request_duration_ms(start_time: datetime, end_time: datetime) -> int | None: diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 732c32ff2cb..e78ec5e5d3e 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -281,7 +281,13 @@ class ResponsesSessionHandler: SQL query - SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id + SELECT session_id, api_key, "user", team_id FROM spend_logs WHERE request_id = previous_response_id, then + SELECT * FROM spend_logs with that session_id and the same owner + + Only rows with the referenced response's owner are returned: the same user and team, and for a key with + neither, the same key. The session id can be caller supplied, so matching on it alone would mix owners. A + user's or team's regenerated or second key keeps the history; a key with neither does not, and turns written + by another owner (an admin continuing the chain) are not replayed. A just-finished turn gets a short second chance: the worker that served it may still be writing its spend log when the follow-up arrives, and an empty result @@ -303,13 +309,24 @@ class ResponsesSessionHandler: query: Final = """ WITH matching_session AS ( - SELECT session_id + SELECT session_id, api_key, "user", team_id FROM "LiteLLM_SpendLogs" WHERE request_id = $1 ) SELECT * - FROM "LiteLLM_SpendLogs" - WHERE session_id IN (SELECT session_id FROM matching_session) + FROM "LiteLLM_SpendLogs" AS logs + WHERE EXISTS ( + SELECT 1 + FROM matching_session + WHERE matching_session.session_id = logs.session_id + AND matching_session."user" IS NOT DISTINCT FROM logs."user" + AND matching_session.team_id IS NOT DISTINCT FROM logs.team_id + AND ( + COALESCE(matching_session."user", '') <> '' + OR COALESCE(matching_session.team_id, '') <> '' + OR matching_session.api_key = logs.api_key + ) + ) ORDER BY "endTime" ASC; """ diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 2e7fb6da194..587cdfcee25 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -39,7 +39,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching import InMemoryCache -from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + REDACTED_BY_LITELLM, + REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, +) from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( @@ -563,6 +567,14 @@ class ChatCompletionSession(TypedDict, total=False): ########### End of Initialize Classes used for Responses API ########### +def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata" + metadata: Final = request_kwargs.get(key) + if not isinstance(metadata, dict): + return MappingProxyType({}) + return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: @@ -1078,8 +1090,6 @@ class LiteLLMCompletionResponsesConfig: else: # Both are empty - this likely means function_call_output had empty/invalid call_id # Provide a helpful error message - import litellm - raise litellm.BadRequestError( message=( f"Unable to create messages for completion request. " @@ -1094,8 +1104,14 @@ class LiteLLMCompletionResponsesConfig: ) litellm_completion_request["messages"] = combined_messages - litellm_completion_request["litellm_trace_id"] = chat_completion_session.get("litellm_session_id") - return litellm_completion_request + chain_session: Final = chat_completion_session.get("litellm_session_id") + litellm_completion_request["litellm_trace_id"] = chain_session + chain_session_metadata: Final[Mapping[str, object]] = ( + _chain_session_metadata(chain_session, request_kwargs) + if chain_session and litellm.request_correlation_in_logs + else MappingProxyType({}) + ) + return {**litellm_completion_request, **chain_session_metadata} @staticmethod def _transform_response_input_param_to_chat_completion_message( diff --git a/litellm/router.py b/litellm/router.py index 151a8a75dc3..3c969610e83 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -38,7 +38,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache, partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast import anyio import httpx @@ -68,6 +68,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, INTERNAL_CALL_ORIGIN_METADATA_KEY, + MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, @@ -75,7 +76,7 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) -from litellm.integrations.custom_guardrail import is_guardrail_intervention +from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span @@ -194,6 +195,7 @@ from litellm.router_utils.clientside_credential_handler import ( is_clientside_credential, ) from litellm.router_utils.common_utils import ( + MID_STREAM_CONTINUATION_KWARG, filter_team_based_models, filter_web_search_deployments, format_fallback_outcome_message, @@ -233,7 +235,10 @@ from litellm.router_utils.fallback_event_handlers import ( check_non_standard_fallback_format, clear_pre_routing_selection, committed_retry_budget_for_request, + continuation_fallback_targets, fallback_lookup_groups, + fallback_target_name, + fallback_target_overrides_request_params, fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, get_pre_routing_selection, @@ -258,6 +263,17 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_MARKER, + ContinuationOutput, + ContinuationPrefillDeploymentCheck, + ContinuedPartial, + constrains_continuation, + continuation_deployments, + continuation_output_ceilings, + continued_text, + deployment_can_continue, +) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, warn_on_unknown_model_group_affinity_flags, @@ -303,6 +319,8 @@ from litellm.router_utils.routing_groups import ( ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch from litellm.scheduler import FlowItem, Scheduler +from litellm.secret_managers.main import str_to_bool +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.litellm_params import RoutingStrategyName from litellm.types.llms.openai import ( AllMessageValues, @@ -371,6 +389,7 @@ from litellm.utils import ( EmbeddingResponse, ModelResponse, Rules, + TextCompletionResponse, function_setup, get_llm_provider, get_non_default_completion_params, @@ -527,6 +546,15 @@ _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeA _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) _DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected" +_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + +def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]: + """Continue the last text block in place so the prefill reads as the text the caller already received.""" + last: Final = blocks[-1] if blocks else None + if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str): + return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}] + return [*blocks, {"type": "text", "text": text}] def _is_fallback_hop(request_kwargs: Mapping[str, object]) -> bool: @@ -1119,6 +1147,7 @@ class Router: health_check_ignore_transient_errors: bool = False, background_health_check_model_groups: Sequence[str] | None = None, enable_weighted_failover: bool = False, + enable_mid_stream_fallback_continuation: bool = False, fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, @@ -1160,6 +1189,7 @@ class Router: deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600. ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. + enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). Returns: @@ -1355,6 +1385,7 @@ class Router: self.fallback_on_unsupported_params = fallback_on_unsupported_params self.enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover + self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation) self.health_check_ignore_transient_errors = health_check_ignore_transient_errors self.background_health_check_model_groups: frozenset[str] | None = ( frozenset(background_health_check_model_groups) @@ -1549,6 +1580,14 @@ class Router: default_pre_call_checks: Final[OptionalPreCallChecks] = [] self.add_optional_pre_call_checks(default_pre_call_checks) + if self.enable_mid_stream_fallback_continuation: + self._register_continuation_prefill_check() + + @staticmethod + def _register_continuation_prefill_check() -> None: + # Process-global on purpose: Router.discard() must not drop a filter another router still needs. + litellm.logging_callback_manager.add_litellm_callback(ContinuationPrefillDeploymentCheck()) + def discard(self): """ Pseudo-destructor to be invoked to clean up global data structures when router is no longer used. @@ -3143,15 +3182,53 @@ class Router: usage: Final = cast(Usage | None, getattr(fallback_item, "usage", None)) usage_objects: Final = [usage] if usage is not None else [] - if ( - complete_response_object_usage is not None - and hasattr(complete_response_object_usage, "usage") - and complete_response_object_usage.usage is not None - ): + if complete_response_object_usage is not None: usage_objects.append(complete_response_object_usage) combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=usage_objects) setattr(fallback_item, "usage", combined_usage) + @staticmethod + def _price_cut_partial( + cut_stream: CustomStreamWrapper, + partial_response: ModelResponse | TextCompletionResponse | None, + ) -> "ContinuedPartial | None": + """Price the cut stream's partial, plus any partial it inherited, before a fallback can reuse its logging.""" + from litellm.cost_calculator import BaseTokenUsageProcessor + + cut_logging: Final = cut_stream.logging_obj + partial_usage: Final = getattr(partial_response, "usage", None) + if partial_response is None or not isinstance(partial_usage, Usage): + return None + inherited_usage: Final = cut_logging.model_call_details.get("continued_partial_usage") + inherited_cost: Final = cut_logging.model_call_details.get("continued_partial_cost") + inherited_id: Final = cut_logging.model_call_details.get("continued_response_id") + priced_partial: Final = partial_response.model_copy( + update={"model": cut_stream.model} if cut_stream.model else {} + ) + priced_partial._hidden_params = { # pyright: ignore[reportPrivateUsage] # no public accessor + key: value for key, value in get_hidden_params_dict(partial_response).items() if key != "response_cost" + } + return ContinuedPartial( + usage=BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[partial_usage, inherited_usage] + if isinstance(inherited_usage, Usage) + else [partial_usage] + ), + response_cost=(cut_logging.response_cost_calculator(result=priced_partial) or 0.0) + + (inherited_cost if isinstance(inherited_cost, float) else 0.0), + response_id=inherited_id if isinstance(inherited_id, str) else partial_response.id, + ) + + @staticmethod + def _carry_partial_spend_to_fallback(fallback_response: object, cut_partial: "ContinuedPartial | None") -> None: + """Bill what the cut legs consumed on the continuation's success log.""" + fallback_logging: Final = getattr(fallback_response, "logging_obj", None) + if cut_partial is None or not isinstance(fallback_logging, LiteLLMLogging): + return + fallback_logging.record_continued_partial( + usage=cut_partial.usage, response_cost=cut_partial.response_cost, response_id=cut_partial.response_id + ) + @staticmethod def _prepare_fallback_hidden_params( fallback_response: object, @@ -3201,7 +3278,7 @@ class Router: self, model_response: CustomStreamWrapper, messages: list[dict[str, str]], - initial_kwargs: dict, + initial_kwargs: dict[str, Any], deployment_slot: contextlib.AsyncExitStack | None = None, first_chunk: _FirstChunk[AsyncIterator[object]] | None = None, ) -> CustomStreamWrapper: @@ -3264,16 +3341,22 @@ class Router: with anyio.CancelScope(shield=True): await close_model_response() await held_slot.aclose() - if not e.is_pre_first_chunk and ( - e.generated_content or _stream_chunks_have_generated_content(model_response.chunks) - ): - if e.original_exception is not None: - raise e.original_exception from e - raise + committed: Final = bool( + not e.is_pre_first_chunk + and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) + ) + cut_hidden_params: Final = get_hidden_params(model_response) or {} + continue_after_content: Final = committed and self._mid_stream_continuation_eligible( + e=e, request_kwargs=initial_kwargs, cut_deployment_id=cut_hidden_params.get("model_id") + ) + if committed and not continue_after_content: + self._raise_original_mid_stream_error(e) from litellm.main import stream_chunk_builder - complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks) + complete_response_object: Final = stream_chunk_builder( + chunks=model_response.chunks, messages=messages if continue_after_content else None + ) complete_response_object_usage: Final = cast( Usage | None, getattr(complete_response_object, "usage", None), @@ -3289,7 +3372,41 @@ class Router: "content_policy_fallbacks", self.content_policy_fallbacks ) initial_kwargs["original_function"] = self._acompletion - initial_kwargs["messages"] = messages + continuation_messages: Final = ( + self._build_completion_continuation_input(messages, e.generated_content) + if continue_after_content + else None + ) + continuation_output: Final = ( + ContinuationOutput( + e.generated_content, + continuation_messages[-1], + model_group, + complete_response_object, + checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG) + is not MID_STREAM_CONTINUATION_MARKER, + ) + if continuation_messages is not None + else None + ) + cut_partial: Final = ( + Router._price_cut_partial(model_response, complete_response_object) + if continuation_output is not None + else None + ) + if continue_after_content: + emitted_tokens: Final = int( + getattr(complete_response_object_usage, "completion_tokens", 0) or 0 + ) + reduced_ceilings: Final = continuation_output_ceilings(initial_kwargs, emitted_tokens) + if reduced_ceilings is None or continuation_messages is None: + self._raise_original_mid_stream_error(e) + await model_response.wait_for_failure_logging(timeout=MID_STREAM_FAILURE_LOG_WAIT_SECONDS) + initial_kwargs.update(reduced_ceilings) + initial_kwargs["messages"] = continuation_messages + initial_kwargs[MID_STREAM_CONTINUATION_KWARG] = MID_STREAM_CONTINUATION_MARKER + else: + initial_kwargs["messages"] = messages self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs) fallback_response = await self.async_function_with_fallbacks_common_utils( e=e, @@ -3305,11 +3422,17 @@ class Router: # If fallback returns a streaming response, iterate over it if hasattr(fallback_response, "__aiter__"): + Router._carry_partial_spend_to_fallback(fallback_response, cut_partial) prepared_fallback_hidden_params = Router._adopt_fallback_response_headers( wrapper_ref, fallback_response ) fallback_headers_are_settled = False - async for fallback_item in fallback_response: + fallback_items: Final = ( + continuation_output.splice(fallback_response) + if continuation_output is not None + else fallback_response + ) + async for fallback_item in fallback_items: if not fallback_headers_are_settled: fallback_headers_are_settled = True # a fallback that failed over again only repoints itself once it yields @@ -3322,7 +3445,10 @@ class Router: and isinstance(fallback_item, ModelResponseStream) and hasattr(fallback_item, "usage") ): - self._combine_fallback_usage(fallback_item, complete_response_object_usage) + self._combine_fallback_usage( + fallback_item, + complete_response_object_usage if continuation_output is not None else None, + ) yield fallback_item else: # If fallback returns a non-streaming response, yield None @@ -3479,6 +3605,97 @@ class Router: total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0), ) + @staticmethod + def _raise_original_mid_stream_error(e: "MidStreamFallbackError") -> NoReturn: + """Surface the provider error the stream wrapper carried instead of the internal MidStreamFallbackError.""" + if e.original_exception is not None: + raise e.original_exception from e + raise e + + def _params_sent_to_deployment( + self, request_kwargs: Mapping[str, object], deployment: Deployment | None + ) -> Mapping[str, object]: + """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" + deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} + defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None} + deployment_tool_choice: Final = ( + {"tool_choice": deployment_params["tool_choice"]} + if deployment_params.get("tool_choice") is not None + else {} + ) + return MappingProxyType({**deployment_params, **defaults, **deployment_tool_choice, **request_kwargs}) + + def _names_deployment_that_cannot_continue(self, target: object) -> bool: + name: Final = fallback_target_name(target) + if name is None or name in self.model_names or not self.has_model_id(name): + return False + deployment: Final = self.get_deployment(model_id=name) + return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True)) + + def _mid_stream_continuation_eligible( + self, + e: "MidStreamFallbackError", + request_kwargs: Mapping[str, object], + cut_deployment_id: object = None, + ) -> bool: + """Whether a stream that broke after plain assistant text may continue via prefill.""" + if not self.enable_mid_stream_fallback_continuation: + return False + if not e.generated_content or e.emitted_disqualifying_content: + return False + if isinstance(e.original_exception, litellm.ContentPolicyViolationError): + return False + cut_deployment: Final = ( + self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None + ) + if isinstance(cut_deployment_id, str) and cut_deployment is None: + return False + if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): + return False + requested_group: Final = request_kwargs.get("model") + lookup_groups: Final = fallback_lookup_groups( + request_kwargs, requested_group if isinstance(requested_group, str) else None + ) + targets: Final = continuation_fallback_targets(request_kwargs.get("fallbacks", self.fallbacks), lookup_groups) + if any(fallback_target_overrides_request_params(target) for target in targets): + return False + if any(self._names_deployment_that_cannot_continue(target) for target in targets): + return False + return not any( + isinstance(callback, CustomGuardrail) + and callback._event_hook_is_event_type( # pyright: ignore[reportPrivateUsage] # the only check of a guardrail's hooks + GuardrailEventHooks.post_call + ) + for callback in litellm.callbacks + ) + + @staticmethod + def _build_completion_continuation_input( + messages: Sequence[Mapping[str, object]], + generated_content: str, + ) -> Sequence[Mapping[str, object]] | None: + """Append the partial output as an assistant prefill, or extend a trailing `prefix: True` assistant turn in + place so the request never ends in two assistant messages. None when the request ends in a plain assistant + turn, in a prefill turn whose content has a shape this cannot extend, or when the cut came mid-echo of the + prefill.""" + last: Final = messages[-1] if messages else None + if last is None or last.get("role") != "assistant": + return [*messages, {"role": "assistant", "content": generated_content, "prefix": True}] + if last.get("prefix") is not True: + return None + new_text: Final = continued_text(last, generated_content) + if new_text is None: + return None + content: Final = last.get("content") + if content is None or isinstance(content, str): + return [*messages[:-1], {**last, "content": f"{content or ''}{new_text}", "prefix": True}] + try: + blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content) + except ValidationError: + return None + extended: Final = _extend_trailing_text_block(blocks, new_text) + return [*messages[:-1], {**last, "content": extended, "prefix": True}] + @staticmethod def _build_responses_continuation_input( input_val: Union[str, "ResponseInputParam"] | None, @@ -3880,13 +4097,6 @@ class Router: raise e.original_exception from e raise - from litellm.main import stream_chunk_builder - - complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks) - complete_response_object_usage: Final = cast( - Usage | None, - getattr(complete_response_object, "usage", None), - ) try: model_group: Final = cast(str, initial_kwargs.get("model")) fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", router_self.fallbacks) @@ -3932,7 +4142,7 @@ class Router: and isinstance(fallback_item, ModelResponseStream) and hasattr(fallback_item, "usage") ): - router_self._combine_fallback_usage(fallback_item, complete_response_object_usage) + router_self._combine_fallback_usage(fallback_item, None) yield fallback_item else: yield None @@ -13004,6 +13214,7 @@ class Router: "retry_policy", "model_group_alias", "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", "enable_tag_filtering", "tag_routing_prefix", "fallback_on_unsupported_params", @@ -13060,12 +13271,10 @@ class Router: rebuild_routing_groups = True elif var == "optional_pre_call_checks": self.set_optional_pre_call_checks(kwargs[var]) + elif var == "enable_mid_stream_fallback_continuation": + self._set_mid_stream_fallback_continuation(kwargs[var]) elif var == "retry_policy": - value = kwargs[var] - if isinstance(value, dict): - value = RetryPolicy(**value) - if value is None or isinstance(value, RetryPolicy): - setattr(self, var, value) + self._set_retry_policy(kwargs[var]) else: value = kwargs[var] # only run routing strategy init if it has changed @@ -13092,6 +13301,20 @@ class Router: self._routing_groups_input = routing_groups_input verbose_router_logger.debug("Updated Router settings: %s", self.get_settings()) + @staticmethod + def _is_flag_enabled(value: object) -> bool: + return value is True or (isinstance(value, str) and str_to_bool(value) is True) + + def _set_mid_stream_fallback_continuation(self, enabled: object) -> None: + self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enabled) + if self.enable_mid_stream_fallback_continuation: + self._register_continuation_prefill_check() + + def _set_retry_policy(self, value: object) -> None: + policy: Final = RetryPolicy.model_validate(value) if isinstance(value, dict) else value + if policy is None or isinstance(policy, RetryPolicy): + self.retry_policy = policy + def _get_client(self, deployment, kwargs, client_type=None): """ Returns the appropriate client based on the given deployment, kwargs, and client_type. @@ -15119,9 +15342,8 @@ class Router: request_kwargs=request_kwargs, ) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) - pick_attributes: Final = _deployment_pick_attributes( - model, _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) - ) + routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) + pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -15159,7 +15381,9 @@ class Router: ) healthy_deployments = _pre_cooldown_deployments - healthy_deployments = self._filter_blocked_deployments(healthy_deployments) + healthy_deployments = continuation_deployments( + self._filter_blocked_deployments(healthy_deployments), routing_kwargs + ) # filter pre-call checks if self.enable_pre_call_checks and (messages is not None or input is not None): diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index e12304c6ed1..c4e4d9ccc0b 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -27,7 +27,8 @@ from litellm.types.utils import LlmProviders, ProviderSpecificModelInfo _V = TypeVar("_V") -ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors"}) +MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" +ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 1f37c6c8cb1..06bd3329cc1 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -615,6 +615,32 @@ def get_fallback_model_group( return fallback_model_group, generic_fallback_idx +def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]: + group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple( # mutable-ok: mirrors the fallbacks config shape + entry for entry in entries if isinstance(entry, Mapping) and entry + ) + chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups) + return chain if isinstance(chain, list) else () + + +def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]: + entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else () + return ( + entries + if check_non_standard_fallback_format(fallbacks=entries) + else _resolved_fallback_chain(entries, lookup_groups) + ) + + +def fallback_target_overrides_request_params(target: object) -> bool: + return isinstance(target, Mapping) and "model" in target and len(target) > 1 + + +def fallback_target_name(target: object) -> str | None: + name: Final = target.get("model") if isinstance(target, Mapping) else target + return name if isinstance(name, str) else None + + PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id") PROVIDER_SCOPED_RESOURCE_FUNCTION_NAMES: Final = frozenset( { diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py new file mode 100644 index 00000000000..8f211ef67db --- /dev/null +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -0,0 +1,264 @@ +"""Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave +the output unconstrained.""" + +from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import accumulate +from os.path import commonprefix +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.rules import Rules +from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage +from litellm.utils import supports_assistant_prefill + + +class _ContinuationMarker: + """Only the router constructs one, so a JSON request body cannot forge the continuation flag.""" + + +MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker() + +_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) +_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) +_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens") + + +def _str_keyed(value: object) -> Mapping[str, object]: + try: + return _STR_KEYED_DICT_ADAPTER.validate_python(value) + except ValidationError: + return _NO_FIELDS + + +def deployment_supports_prefill(deployment: object) -> bool: + deployment_map: Final = _str_keyed(deployment) + model_info: Final = _str_keyed(deployment_map.get("model_info")) + declared: Final = model_info.get("supports_assistant_prefill") + if isinstance(declared, bool): + return declared + litellm_params: Final = _str_keyed(deployment_map.get("litellm_params")) + candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model")) + model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + return model is not None and supports_assistant_prefill(model=model) + + +def constrains_continuation(params: Mapping[str, object]) -> bool: + sent: Final = {**params, **_str_keyed(params.get("extra_body"))} + choice_count: Final = sent.get("n") + response_format: Final = sent.get("response_format") + tool_choice: Final = sent.get("tool_choice") + return ( + (isinstance(choice_count, int) and choice_count > 1) + or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) + or tool_choice == "required" + or isinstance(tool_choice, Mapping) + or sent.get("merge_reasoning_content_in_choices") is True + ) + + +def deployment_can_continue(deployment: object) -> bool: + litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")) + return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params) + + +def continuation_deployments( + healthy_deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list + marker: Final = (request_kwargs or {}).get(MID_STREAM_CONTINUATION_KWARG) + if not isinstance(marker, _ContinuationMarker): + return healthy_deployments + return [deployment for deployment in healthy_deployments if deployment_can_continue(deployment)] + + +class ContinuationPrefillDeploymentCheck(CustomLogger): + async def async_filter_deployments( + self, + model: str, + healthy_deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: Sequence[AllMessageValues] | None, + request_kwargs: Mapping[str, object] | None = None, + parent_otel_span: Span | None = None, + ) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list + return continuation_deployments(healthy_deployments, request_kwargs) + + +def _delta_text(chunk: ModelResponseStream) -> str: + if not chunk.choices: + return "" + content: Final = chunk.choices[0].delta.content + return content if isinstance(content, str) else "" + + +_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + +def _prefill_text(prefill_message: Mapping[str, object]) -> str: + content: Final = prefill_message.get("content") + if isinstance(content, str): + return content + try: + blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content) + except ValidationError: + return "" + return "".join( + text for block in blocks if block.get("type") == "text" and isinstance(text := block.get("text"), str) + ) + + +def _resendable_whitespace(prefill: str) -> str: + return prefill[len(prefill.rstrip()) :] if prefill.strip() else "" + + +def continued_text(prefill_message: Mapping[str, object], generated_content: str) -> str | None: + prefill: Final = _prefill_text(prefill_message) + if generated_content and len(generated_content) < len(prefill) and prefill.startswith(generated_content): + return None + echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0 + after_echo: Final = generated_content[echo:] + return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :] + + +def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]: + return MappingProxyType( + {key: value - emitted_tokens for key in _OUTPUT_CEILING_KEYS if isinstance(value := params.get(key), int)} + ) + + +def continuation_output_ceilings( + request_kwargs: Mapping[str, object], emitted_tokens: int +) -> Mapping[str, object] | None: + """The caller's output ceilings, top-level and in extra_body, minus the tokens already emitted, or None once + nothing is left.""" + extra_body: Final = _str_keyed(request_kwargs.get("extra_body")) + top_level: Final = _reduced_ceilings(request_kwargs, emitted_tokens) + in_extra_body: Final = _reduced_ceilings(extra_body, emitted_tokens) + if any(ceiling <= 0 for ceiling in (*top_level.values(), *in_extra_body.values())): + return None + if not in_extra_body: + return top_level + return MappingProxyType({**top_level, "extra_body": {**extra_body, **in_extra_body}}) + + +def _with_first_delta(chunk: ModelResponseStream, update: Mapping[str, object]) -> ModelResponseStream: + choice: Final = chunk.choices[0] + delta: Final = choice.delta.model_copy(update=update) + return chunk.model_copy(update={"choices": [choice.model_copy(update={"delta": delta}), *chunk.choices[1:]]}) + + +def _without_leading_text(chunk: ModelResponseStream, count: int) -> ModelResponseStream: + if count <= 0: + return chunk + return _with_first_delta(chunk, {"content": _delta_text(chunk)[count:]}) + + +def _without_role(chunk: ModelResponseStream) -> ModelResponseStream: + if not chunk.choices or chunk.choices[0].delta.role is None: + return chunk + return _with_first_delta(chunk, {"role": None}) + + +def _stream_identity(primary_response: object) -> Mapping[str, object]: + if not isinstance(primary_response, ModelResponse): + return _NO_FIELDS + return MappingProxyType({"id": primary_response.id, "created": primary_response.created}) + + +def _drop_leading_text(chunks: tuple[ModelResponseStream, ...], count: int) -> tuple[ModelResponseStream, ...]: + lengths: Final = tuple(len(_delta_text(chunk)) for chunk in chunks) + offsets: Final = tuple(accumulate(lengths, initial=0))[:-1] + return tuple( + _without_leading_text(chunk, min(length, count - offset)) + for chunk, offset, length in zip(chunks, offsets, lengths, strict=True) + ) + + +@dataclass(frozen=True, slots=True) +class ContinuedPartial: + usage: Usage + response_cost: float + response_id: str + + +class ContinuationOutput: + """Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the + prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing + whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and + runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A + continuation nested in another leaves the rules to the outermost one, the only one holding the whole text.""" + + def __init__( + self, + emitted_text: str, + prefill_message: Mapping[str, object], + model: str, + primary_response: object, + checks_post_call_rules: bool = True, + ) -> None: + self._text = emitted_text + self._checks_post_call_rules: Final = checks_post_call_rules + self._prefill_text: Final = _prefill_text(prefill_message) + self._model: Final = model + self._identity: Final = _stream_identity(primary_response) + self._rules: Final = Rules() + self._held: tuple[ModelResponseStream, ...] = () + self._echo_settled = not self._prefill_text + self._seam = _resendable_whitespace(self._prefill_text) + + async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]: + async for item in stream: + if not isinstance(item, ModelResponseStream): + yield item + continue + for released in self._observed(self._past_seam(self._released(item))): + yield self._in_primary_stream(released) + for released in self._observed(self._past_seam(self._flushed())): + yield self._in_primary_stream(released) + + def _in_primary_stream(self, chunk: ModelResponseStream) -> ModelResponseStream: + return _without_role(chunk.model_copy(update=self._identity) if self._identity else chunk) + + def _flushed(self) -> tuple[ModelResponseStream, ...]: + held: Final = self._held + self._held = () + self._echo_settled = True + return held + + def _released(self, chunk: ModelResponseStream) -> tuple[ModelResponseStream, ...]: + if self._echo_settled: + return (chunk,) + held: Final = (*self._held, chunk) + text: Final = "".join(_delta_text(held_chunk) for held_chunk in held) + if len(text) < len(self._prefill_text) and self._prefill_text.startswith(text): + self._held = held + return () + self._held = () + self._echo_settled = True + if not text.startswith(self._prefill_text): + return held + return _drop_leading_text(held, len(self._prefill_text)) + + def _past_seam(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]: + return tuple(self._without_seam(chunk) for chunk in chunks) + + def _without_seam(self, chunk: ModelResponseStream) -> ModelResponseStream: + text: Final = _delta_text(chunk) + matched: Final = len(commonprefix((text, self._seam))) + self._seam = self._seam[matched:] if matched == len(text) else "" + return _without_leading_text(chunk, matched) + + def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]: + if not self._checks_post_call_rules or not litellm.post_call_rules: + return chunks + for chunk in chunks: + self._text += _delta_text(chunk) + self._rules.post_call_rules(input=self._text, model=self._model) + return chunks diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0bcb732a0f5..7df8cc7452f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4045,7 +4045,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) -DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_regex_lookaround"}) +DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( + {"supports_regex_lookaround", "supports_assistant_prefill"} +) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) diff --git a/litellm/utils.py b/litellm/utils.py index 65408663f6f..d2ca9faef41 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3060,6 +3060,20 @@ def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None ) +def supports_assistant_prefill(model: str, custom_llm_provider: str | None = None) -> bool: + """ + Whether the model can continue a prefilled assistant message (Anthropic's + ``prefix: True`` trick and equivalents). Missing metadata reads as False, + so a mid-stream fallback continuation is only routed to a model known to + support it. + """ + return _supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_assistant_prefill", + ) + + def supports_thinking_cache_preservation(model: str, custom_llm_provider: str | None = None) -> bool: return supports_factory( model=model, @@ -9933,6 +9947,10 @@ class ProviderConfigManager: return MinimaxVideoConfig() elif LlmProviders.EDENAI == provider: return litellm.EdenAIVideoConfig() + elif LlmProviders.VOLCENGINE == provider: + from litellm.llms.volcengine.videos.transformation import VolcEngineVideoConfig + + return VolcEngineVideoConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b744032fb6c..49d2ce90af4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2773,7 +2773,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2808,7 +2808,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2843,7 +2843,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2878,7 +2878,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2913,7 +2913,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2948,7 +2948,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -3743,7 +3743,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -15494,7 +15494,7 @@ }, "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -20832,7 +20832,7 @@ "output_cost_per_token": 2.5000010000000002e-05, "output_dbu_cost_per_token": 0.000357143, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -21047,7 +21047,7 @@ "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -49176,7 +49176,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -51006,7 +51006,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -51179,7 +51179,7 @@ "supported_regions": [ "global" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -51200,7 +51200,7 @@ "supported_regions": [ "us-central1" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -58822,7 +58822,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -60467,6 +60467,61 @@ } ] }, + "volcengine/doubao-seedance-2-5-260628": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.216, + "output_cost_per_second_480p": 0.096075, + "output_cost_per_second_720p": 0.216, + "output_cost_per_second_1080p": 0.5346, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.141942857, + "output_cost_per_second_480p": 0.066003429, + "output_cost_per_second_720p": 0.141942857, + "output_cost_per_second_1080p": 0.354085714, + "output_cost_per_second_4k": 0.722057143, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-fast-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.114171429, + "output_cost_per_second_480p": 0.053089714, + "output_cost_per_second_720p": 0.114171429, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-mini-260615": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.070971429, + "output_cost_per_second_480p": 0.033001714, + "output_cost_per_second_720p": 0.070971429, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, @@ -72334,7 +72389,7 @@ "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 45104d40018..0325fe1f796 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2766,7 +2766,8 @@ "batches": false, "rerank": false, "a2a": true, - "interactions": true + "interactions": true, + "video_generations": true } }, "voyage": { diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index a3ed70f4a56..6762e4cd35c 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -114,6 +114,12 @@ ignored_function_names = [ "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py "_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py "_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py + "_register_continuation_prefill_check", # Tested through the continuation tests in test_router.py + "_set_mid_stream_fallback_continuation", # Tested through the update_settings continuation test in test_router.py + "_is_flag_enabled", # Tested through the string-flag continuation tests in test_router.py + "_params_sent_to_deployment", # Tested through the configured-constraint continuation tests in test_router.py + "_names_deployment_that_cannot_continue", # Tested through the deployment-id continuation tests in test_router.py + "_set_retry_policy", # Tested through update_settings in test_router_retry_policy_update.py "_resolve_pre_call_check_input_tokens", # Tested through the token-count failure pre-call check tests in test_router.py "_params_neither_dropped_nor_allowed", # Tested through the allowed-or-dropped param pre-call check tests in test_router.py "_deployment_rejects_params", # Tested through the unsupported-param pre-call check tests in test_router.py diff --git a/tests/integration/spend/test_spend_log_caller_session.py b/tests/integration/spend/test_spend_log_caller_session.py new file mode 100644 index 00000000000..213583d21a1 --- /dev/null +++ b/tests/integration/spend/test_spend_log_caller_session.py @@ -0,0 +1,783 @@ +import itertools +import json +import re +import threading +import uuid +from urllib.parse import urlsplit +from collections import deque +from collections.abc import Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import openai +import psutil +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.budget import readiness_seconds +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * readiness_seconds(2) + 2 * graceful_stop_seconds() + 240) + +MARKER: Final = re.compile(rb"callersess-[0-9a-f]{32}") +CALLER_SESSION_KEY: Final = "litellm_caller_session_id" +PATHS: Final = MappingProxyType( + {"chat": "/v1/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"} +) +ENDPOINT_STREAMS: Final = tuple(itertools.product(PATHS, (False, True))) +Headers = Mapping[str, str] | Sequence[tuple[str, str]] + + +def _marker() -> str: + return "callersess-" + uuid.uuid4().hex + + +def _token(label: str) -> str: + return f"{label}-{uuid.uuid4().hex}" + + +def _frame(payload: Mapping[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(payload).encode() + b"\n\n" + + +def _chat_reply(text: str, stream: bool) -> Reply: + identity: Final = "chatcmpl-" + uuid.uuid4().hex + usage: Final[dict[str, JsonValue]] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"} + ], + "usage": usage, + } + ).encode() + ) + chunk: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + } + return Reply( + content_type="text/event-stream", + chunks=( + _frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}}]}), + _frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": usage}), + b"data: [DONE]\n\n", + ), + ) + + +def _upstream(request: Request) -> Reply: + markers: Final = tuple(found.group(0).decode() for found in MARKER.finditer(request.body)) + if request.method != "POST" or not request.target.endswith("/chat/completions") or not markers: + return Reply(status=404, body=b'{"error":"unexpected request"}') + return _chat_reply(f"answer {markers[-1]}", JSON_OBJECT.validate_json(request.body).get("stream") is True) + + +def _user_markers(body: Mapping[str, JsonValue]) -> tuple[str, ...]: + user_contents: Final = tuple( + json.dumps(object_value(message)["content"]) + for message in list_value(body.get("messages", [])) + if object_value(message)["role"] == "user" + ) + return tuple(found.group(0).decode() for content in user_contents if (found := MARKER.search(content.encode()))) + + +def _spans(batch: Request) -> Iterator[dict[str, JsonValue]]: + for resource in list_value(JSON_OBJECT.validate_json(batch.body)["resourceSpans"]): + for scope in list_value(object_value(resource)["scopeSpans"]): + yield from (object_value(span) for span in list_value(object_value(scope)["spans"])) + + +def _attributes(span: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return { + string_value(object_value(attribute)["key"]): object_value(attribute)["value"] + for attribute in list_value(span.get("attributes", [])) + } + + +def _batch_attributes(batches: Sequence[Request]) -> Iterator[dict[str, JsonValue]]: + return (_attributes(span) for span in itertools.chain.from_iterable(_spans(batch) for batch in batches)) + + +@dataclass(frozen=True, slots=True) +class Collector: + wire: Wire + accepted: deque[Request] + guard: threading.Lock + + def span_attributes(self, response_id: str) -> tuple[dict[str, JsonValue], ...]: + with self.guard: + batches: Final = tuple(self.accepted) + attributes: Final = tuple(_batch_attributes(batches)) + return tuple( + found + for found in attributes + if object_value(found.get("gen_ai.response.id", {})).get("stringValue") == response_id + ) + + def conversation_ids(self, conversations: frozenset[str]) -> tuple[str, ...]: + with self.guard: + batches: Final = tuple(self.accepted) + responded: Final = ( + object_value(found.get("gen_ai.conversation.id", {})).get("stringValue") + for found in _batch_attributes(batches) + if "gen_ai.response.id" in found + ) + return tuple( + conversation + for conversation in responded + if isinstance(conversation, str) and conversation in conversations + ) + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + scenario: Scenario + model: str + upstream: Wire + process: OwnedProxy + + def body(self, endpoint: str, marker: str, extra: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + match endpoint: + case "chat": + return {"model": self.model, "messages": [{"role": "user", "content": marker}], **extra} + case "messages": + return { + "model": self.model, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker}], + **extra, + } + case _: + return {"model": self.model, "input": marker, **extra} + + def post( + self, + endpoint: str, + marker: str, + fields: Mapping[str, JsonValue] = MappingProxyType({}), + *, + stream: bool = False, + headers: Headers = (), + key: str | None = None, + ) -> httpx.Response: + payload: Final = self.body(endpoint, marker, fields) | ({"stream": True} if stream else {}) + request_headers: Final = ( + ("Authorization", f"Bearer {self.proxy.key if key is None else key}"), + *(headers.items() if isinstance(headers, Mapping) else headers), + ) + with self.proxy.client.stream("POST", PATHS[endpoint], json=payload, headers=request_headers) as response: + response.read() + return response + + def send( + self, + endpoint: str, + marker: str, + fields: Mapping[str, JsonValue] = MappingProxyType({}), + *, + stream: bool = False, + headers: Headers = (), + key: str | None = None, + ) -> httpx.Response: + response: Final = self.post(endpoint, marker, fields, stream=stream, headers=headers, key=key) + assert response.status_code == 200, response.text + return response + + def upstream_bodies(self, marker: str) -> tuple[dict[str, JsonValue], ...]: + bodies: Final = tuple(JSON_OBJECT.validate_json(request.body) for request in self.upstream.drain()) + return tuple(body for body in bodies if _user_markers(body)[-1:] == (marker,)) + + +@dataclass(frozen=True, slots=True) +class Rigs: + flag_off: Rig + flag_on: Rig + sink: Collector + + +def _spend_rows(call_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT session_id, request_id, metadata::text AS metadata_text FROM "LiteLLM_SpendLogs" ' + "WHERE litellm_call_id=%s", + (call_id,), + ) + + +def _spend_row(response: httpx.Response) -> dict[str, JsonValue]: + call_id: Final = response.headers["x-litellm-call-id"] + rows: Final = eventually(lambda: _spend_rows(call_id), lambda values: len(values) >= 1, seconds=70) + assert len(rows) == 1, rows + return rows[0] + + +def _session(response: httpx.Response) -> JsonValue: + return _spend_row(response)["session_id"] + + +def _model(scenario: Scenario, upstream: Wire) -> str: + return scenario.model( + model="hosted_vllm/gpt-4o-mini", + api_base=f"{upstream.url}/v1", + use_chat_completions_api=True, + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + + +def _rig_config( + directory: Path, name: str, litellm_settings: Mapping[str, JsonValue], extra: Mapping[str, JsonValue] +) -> Path: + shared: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **shared, + "general_settings": {**object_value(shared["general_settings"]), "store_prompts_in_spend_logs": True}, + "litellm_settings": {**object_value(shared["litellm_settings"]), **litellm_settings}, + **extra, + } + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rigs(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rigs]: + accepted: Final[deque[Request]] = deque() # mutable-ok: sink thread appends each accepted OTLP batch + guard: Final = threading.Lock() + + def sink(request: Request) -> Reply: + with guard: + accepted.append(request) + return Reply() + + directory: Final = tmp_path_factory.mktemp("caller-session") + with ExitStack() as stack: + upstream: Final = stack.enter_context(wire_server(_upstream)) + collector: Final = Collector(stack.enter_context(wire_server(sink)), accepted, guard) + shared: Final = stack.enter_context(gateway_from_environment()) + flag_off: Final = stack.enter_context( + owned_proxy_process(shared, directory, {}, config=_rig_config(directory, "flag-off", {}, {}), workers=2) + ) + flag_on: Final = stack.enter_context( + owned_proxy_process( + shared, + directory, + {"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"}, + config=_rig_config( + directory, + "flag-on", + {"request_correlation_in_logs": True, "callbacks": ["otel"]}, + { + "callback_settings": { + "otel": {"exporter": "http/json", "endpoint": collector.wire.url, "mapper_names": ["genai"]} + } + }, + ), + workers=2, + ) + ) + off_scenario: Final = stack.enter_context(flag_off.gateway.scenario()) + on_scenario: Final = stack.enter_context(flag_on.gateway.scenario()) + yield Rigs( + Rig(flag_off.gateway, off_scenario, _model(off_scenario, upstream), upstream, flag_off), + Rig(flag_on.gateway, on_scenario, _model(on_scenario, upstream), upstream, flag_on), + collector, + ) + + +@pytest.mark.parametrize(("endpoint", "stream"), ENDPOINT_STREAMS) +def test_spend_log_session_is_the_body_session_beside_a_trace_header(rigs: Rigs, endpoint: str, stream: bool) -> None: + marker: Final = _marker() + session: Final = _token("convo") + response: Final = rigs.flag_on.send( + endpoint, + marker, + stream=stream, + headers={"x-litellm-trace-id": _token("trace")}, + fields={"litellm_session_id": session}, + ) + + row: Final = _spend_row(response) + upstream: Final = rigs.flag_on.upstream_bodies(marker) + + assert row["session_id"] == session, row + assert CALLER_SESSION_KEY not in string_value(row["metadata_text"]), row + assert len(upstream) == 1, upstream + assert CALLER_SESSION_KEY not in json.dumps(upstream[0]), upstream + + +def test_sdk_clients_record_the_body_session_beside_a_trace_header(rigs: Rigs) -> None: + rig: Final = rigs.flag_on + base_url: Final = str(rig.proxy.client.base_url) + traces: Final = tuple(_token("trace") for _ in range(3)) + sessions: Final = tuple(_token("convo") for _ in range(3)) + markers: Final = tuple(_marker() for _ in range(3)) + chat_client: Final = openai.OpenAI(base_url=base_url + "/v1", api_key=rig.proxy.key, max_retries=0) + messages_client: Final = anthropic.Anthropic(base_url=base_url, api_key=rig.proxy.key, max_retries=0) + + with chat_client.chat.completions.with_streaming_response.create( + model=rig.model, + messages=[{"role": "user", "content": markers[0]}], + stream=True, + extra_headers={"x-litellm-trace-id": traces[0]}, + extra_body={"litellm_session_id": sessions[0]}, + ) as chat_stream: + chat_call: Final = chat_stream.headers["x-litellm-call-id"] + chat_text: Final = "".join( + chunk.choices[0].delta.content or "" for chunk in chat_stream.parse() if chunk.choices + ) + message: Final = messages_client.messages.with_raw_response.create( + model=rig.model, + max_tokens=16, + messages=[{"role": "user", "content": markers[1]}], + extra_headers={"x-litellm-trace-id": traces[1]}, + extra_body={"litellm_session_id": sessions[1]}, + ) + with chat_client.responses.with_streaming_response.create( + model=rig.model, + input=markers[2], + stream=True, + extra_headers={"x-litellm-trace-id": traces[2]}, + extra_body={"litellm_session_id": sessions[2]}, + ) as responses_stream: + responses_call: Final = responses_stream.headers["x-litellm-call-id"] + completed: Final = tuple(event for event in responses_stream.parse() if event.type == "response.completed") + + first_block: Final = message.parse().content[0] + assert chat_text == f"answer {markers[0]}", chat_text + assert isinstance(first_block, anthropic.types.TextBlock) and first_block.text == f"answer {markers[1]}", message + assert len(completed) == 1, completed + calls: Final = (chat_call, message.headers["x-litellm-call-id"], responses_call) + landed: Final = tuple( + eventually(lambda call=call: _spend_rows(call), lambda values: len(values) == 1, seconds=70)[0]["session_id"] + for call in calls + ) + assert landed == sessions, (landed, calls) + + +def test_conversation_span_keeps_the_trace_header_and_no_internal_key(rigs: Rigs) -> None: + trace: Final = _token("trace") + response: Final = rigs.flag_on.send( + "chat", _marker(), headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": _token("convo")} + ) + response_id: Final = string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + spans: Final = eventually(lambda: rigs.sink.span_attributes(response_id), lambda found: len(found) == 1, seconds=30) + + assert spans[0]["gen_ai.conversation.id"] == {"stringValue": trace}, spans + assert CALLER_SESSION_KEY not in json.dumps(spans[0]), spans + + +def test_a_continued_chains_conversation_spans_keep_each_turns_trace_header(rigs: Rigs) -> None: + traces: Final = (_token("trace"), _token("trace")) + first: Final = _turn(rigs.flag_on, _marker(), headers={"x-litellm-trace-id": traces[0]}) + _turn( + rigs.flag_on, + _marker(), + first, + headers={"x-litellm-trace-id": traces[1]}, + fields={"litellm_session_id": _token("convo")}, + ) + + conversations: Final = eventually( + lambda: rigs.sink.conversation_ids(frozenset(traces)), lambda found: len(found) == 2, seconds=30 + ) + + assert sorted(conversations) == sorted(traces), conversations + + +@pytest.mark.parametrize("endpoint", list(PATHS)) +def test_flag_off_keeps_the_trace_id_beside_a_body_session(rigs: Rigs, endpoint: str) -> None: + trace: Final = _token("trace") + + response: Final = rigs.flag_off.send( + endpoint, _marker(), headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": _token("convo")} + ) + + assert _session(response) == trace + + +@pytest.mark.parametrize("endpoint", list(PATHS)) +def test_response_cache_hit_records_the_body_session_beside_a_trace_header(rigs: Rigs, endpoint: str) -> None: + marker: Final = _marker() + sessions: Final = (_token("convo"), _token("convo")) + responses: Final = tuple( + rigs.flag_on.send( + endpoint, marker, headers={"x-litellm-trace-id": _token("trace")}, fields={"litellm_session_id": session} + ) + for session in sessions + ) + + assert tuple(_session(response) for response in responses) == sessions + assert len(rigs.flag_on.upstream_bodies(marker)) == 1 + + +def test_metadata_session_beside_a_new_trace_per_turn_groups_both_turns(rigs: Rigs) -> None: + session: Final = _token("convo") + responses: Final = tuple( + rigs.flag_on.send( + "chat", + _marker(), + headers={"x-litellm-trace-id": _token("trace")}, + fields={"metadata": {"session_id": session}}, + ) + for _ in range(2) + ) + + assert tuple(_session(response) for response in responses) == (session, session) + now: Final = datetime.now(timezone.utc) + page: Final = rigs.flag_on.proxy.get( + "/spend/logs/ui", + params={ + "session_id": session, + "start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + "end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + }, + ) + listed: Final = sorted(string_value(object_value(row)["request_id"]) for row in list_value(page["data"])) + assert listed == sorted( + string_value(JSON_OBJECT.validate_json(response.content)["id"]) for response in responses + ), page + + +def _shape(shape: str, trace: str, session: str, header: str) -> tuple[Headers, dict[str, JsonValue], str]: + traced: Final = {"x-litellm-trace-id": trace} + match shape: + case "session-header-wins": + return {"x-litellm-session-id": header}, {"litellm_session_id": session}, header + case "trace-header-only": + return traced, {}, trace + case "body-session-only": + return {}, {"litellm_session_id": session}, session + case "int-session": + return traced, {"litellm_session_id": 7}, trace + case "list-session": + return traced, {"litellm_session_id": [session]}, trace + case "empty-session": + return traced, {"litellm_session_id": ""}, trace + case "null-session": + return traced, {"litellm_session_id": None}, trace + case "nul-session": + return traced, {"litellm_session_id": session + "\x00x"}, trace + case "two-trace-headers": + return ( + (("x-litellm-trace-id", trace), ("x-litellm-trace-id", header)), + {"litellm_session_id": session}, + session, + ) + case _: + return traced, {"litellm_session_id": session + "s" * 5120}, session + "s" * 5120 + + +@pytest.mark.parametrize( + "shape", + [ + "session-header-wins", + "trace-header-only", + "body-session-only", + "int-session", + "list-session", + "empty-session", + "null-session", + "nul-session", + "two-trace-headers", + "5kb-session", + ], +) +def test_spend_log_session_for_each_session_shape(rigs: Rigs, shape: str) -> None: + headers, fields, expected = _shape(shape, _token("trace"), _token("convo"), _token("header")) + neighbor_trace: Final = _token("neighbor") + + response: Final = rigs.flag_on.send("chat", _marker(), fields, headers=headers) + neighbor: Final = rigs.flag_on.send("chat", _marker(), headers={"x-litellm-trace-id": neighbor_trace}) + + assert _session(response) == expected + assert _session(neighbor) == neighbor_trace + + +@pytest.mark.parametrize("endpoint", ["chat", "messages"]) +def test_client_sent_internal_session_key_is_ignored(rigs: Rigs, endpoint: str) -> None: + forged: Final = _token("forged") + trace: Final = _token("trace") + forged_metadata: Final[dict[str, JsonValue]] = { + "metadata" if endpoint == "chat" else "litellm_metadata": {CALLER_SESSION_KEY: forged} + } + + untraced: Final = rigs.flag_on.send(endpoint, _marker(), forged_metadata) + traced: Final = rigs.flag_on.send(endpoint, _marker(), forged_metadata, headers={"x-litellm-trace-id": trace}) + + assert _session(untraced) not in (forged, None) + assert _session(traced) == trace + + +def test_identical_requests_each_log_once_with_the_body_session(rigs: Rigs) -> None: + session: Final = _token("convo") + trace: Final = _token("trace") + marker: Final = _marker() + + responses: Final = tuple( + rigs.flag_on.send("chat", marker, headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": session}) + for _ in range(3) + ) + + assert len({response.headers["x-litellm-call-id"] for response in responses}) == 3 + assert tuple(_session(response) for response in responses) == (session, session, session) + + +def _pass_through_route(rig: Rig, target: str) -> str: + path: Final = f"/caller-session-{uuid.uuid4().hex}" + created: Final = rig.proxy.post("/config/pass_through_endpoint", {"path": path, "target": target, "auth": True}) + endpoint_id: Final = string_value(object_value(list_value(created["endpoints"])[0])["id"]) + rig.scenario.cleanups.callback( + lambda: rig.proxy.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": endpoint_id}) + ) + return path + + +@pytest.mark.parametrize("field", ["litellm_session_id", "metadata"]) +def test_pass_through_spend_log_session_is_the_body_session_beside_a_trace_header(rigs: Rigs, field: str) -> None: + marker: Final = _marker() + session: Final = _token("convo") + rig: Final = rigs.flag_on + path: Final = _pass_through_route(rig, f"{rig.upstream.url}/v1/chat/completions") + key: Final = rig.scenario.key(allowed_passthrough_routes=[path]) + fields: Final[dict[str, JsonValue]] = ( + {"litellm_session_id": session} if field == "litellm_session_id" else {"metadata": {"session_id": session}} + ) + + response: Final = rig.proxy.request( + "POST", + path, + rig.body("chat", marker, fields), + key=key, + headers={"x-litellm-trace-id": _token("trace")}, + ) + + rows: Final = eventually( + lambda: read_rows( + 'SELECT session_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (sha256(key.encode()).hexdigest(),) + ), + lambda found: len(found) >= 1, + seconds=70, + ) + assert response.status_code == 200, response.text + assert f"answer {marker}" in response.text + assert len(rig.upstream_bodies(marker)) == 1 + assert rows == [{"session_id": session}] + + +def _turn( + rig: Rig, + marker: str, + previous: str | None = None, + *, + key: str | None = None, + headers: Headers = (), + fields: Mapping[str, JsonValue] = MappingProxyType({}), +) -> str: + chained: Final[dict[str, JsonValue]] = {"previous_response_id": previous} if previous else {} + response: Final = rig.send("responses", marker, {**chained, **fields}, headers=headers, key=key) + _spend_row(response) + return string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + +def _turn_fields(shape: str, session: str) -> tuple[dict[str, JsonValue], dict[str, JsonValue]]: + match shape: + case "session-every-turn": + return {"litellm_session_id": session}, {"litellm_session_id": session} + case "session-from-turn-2": + return {}, {"litellm_session_id": session} + case _: + return {"litellm_session_id": _token("convo")}, {"litellm_session_id": _token("convo")} + + +@pytest.mark.parametrize( + ("correlated", "shape", "replays_turn_1"), + [ + (True, "session-every-turn", True), + (True, "session-from-turn-2", True), + (True, "new-session-per-turn", True), + (False, "session-from-turn-2", False), + ], +) +def test_bridged_chain_history(rigs: Rigs, correlated: bool, shape: str, replays_turn_1: bool) -> None: + rig: Final = rigs.flag_on if correlated else rigs.flag_off + first_fields, second_fields = _turn_fields(shape, _token("convo")) + markers: Final = (_marker(), _marker(), _marker()) + first: Final = _turn(rig, markers[0], headers={"x-litellm-trace-id": _token("trace")}, fields=first_fields) + second: Final = _turn(rig, markers[1], first, headers={"x-litellm-trace-id": _token("trace")}, fields=second_fields) + + _turn(rig, markers[2], second) + + replayed: Final = rig.upstream_bodies(markers[2]) + assert len(replayed) == 1, replayed + assert _user_markers(replayed[0]) == (markers if replays_turn_1 else markers[1:]) + + +def test_bridged_chains_sharing_a_metadata_session_replay_each_others_turns(rigs: Rigs) -> None: + shared: Final[dict[str, JsonValue]] = {"metadata": {"session_id": _token("convo")}} + markers: Final = (_marker(), _marker(), _marker()) + first: Final = _turn(rigs.flag_on, markers[0], fields=shared) + _turn(rigs.flag_on, markers[1], fields=shared) + + _turn(rigs.flag_on, markers[2], first) + + replayed: Final = rigs.flag_on.upstream_bodies(markers[2]) + assert len(replayed) == 1, replayed + assert _user_markers(replayed[0]) == markers + + +@dataclass(frozen=True, slots=True) +class Owners: + first: str + second: str + + +def _owners(scenario: Scenario, model: str, shape: str) -> Owners: + user: Final = _token("user") + match shape: + case "other-user": + return Owners( + scenario.key(user_id=user, models=[model]), scenario.key(user_id=_token("user"), models=[model]) + ) + case "same-user": + return Owners(scenario.key(user_id=user, models=[model]), scenario.key(user_id=user, models=[model])) + case "same-team": + team: Final = scenario.team() + return Owners(scenario.key(team_id=team, models=[model]), scenario.key(team_id=team, models=[model])) + case _: + return Owners(scenario.key(models=[model]), scenario.key(models=[model])) + + +@pytest.mark.parametrize( + ("shape", "replays_first_owner"), + [("other-user", False), ("same-user", True), ("same-team", True), ("ownerless-keys", False)], +) +def test_bridged_history_follows_the_previous_responses_owner( + rigs: Rigs, shape: str, replays_first_owner: bool +) -> None: + rig: Final = rigs.flag_off + owners: Final = _owners(rig.scenario, rig.model, shape) + session: Final = _token("convo") + markers: Final = (_marker(), _marker(), _marker()) + _turn(rig, markers[0], key=owners.first, fields={"litellm_session_id": session}) + second: Final = _turn(rig, markers[1], key=owners.second, fields={"litellm_session_id": session}) + + _turn(rig, markers[2], second, key=owners.second) + + replayed: Final = rig.upstream_bodies(markers[2]) + assert len(replayed) == 1, replayed + assert _user_markers(replayed[0]) == (markers if replays_first_owner else markers[1:]) + + +BURST: Final = ENDPOINT_STREAMS * 4 + + +def test_concurrent_burst_across_endpoints_logs_each_session_once(rigs: Rigs) -> None: + sessions: Final = tuple(_token("convo") for _ in BURST) + + def one(index: int) -> httpx.Response: + endpoint, stream = BURST[index] + return rigs.flag_on.send( + endpoint, + _marker(), + stream=stream, + headers={"x-litellm-trace-id": _token("trace")}, + fields={"litellm_session_id": sessions[index]}, + ) + + with ThreadPoolExecutor(max_workers=12) as pool: + responses: Final = tuple(pool.map(one, range(len(BURST)))) + + assert tuple(_session(response) for response in responses) == sessions + + +def _open_upstream_connections(pid: int, upstream: Wire) -> int: + port: Final = urlsplit(upstream.url).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +def _post_or_none(rig: Rig, session: str) -> httpx.Response | None: + try: + return rig.post( + "chat", _marker(), headers={"x-litellm-trace-id": _token("trace")}, fields={"litellm_session_id": session} + ) + except httpx.HTTPError: + return None + + +def test_killing_one_worker_mid_burst_keeps_logging_the_body_session(rigs: Rigs) -> None: + release: Final = threading.Event() + arrived: Final[SimpleQueue[bytes]] = SimpleQueue() + + def held(request: Request) -> Reply: + arrived.put(request.body) + assert release.wait(timeout=60), "the burst was never released" + return _upstream(request) + + process: Final = rigs.flag_on.process + sessions: Final = tuple(_token("convo") for _ in range(20)) + after_session: Final = _token("convo") + with wire_server(held) as wire: + rig: Final = replace(rigs.flag_on, model=_model(rigs.flag_on.scenario, wire), upstream=wire) + workers: Final = eventually(process.worker_pids, lambda found: len(found) == 2, seconds=30) + with ThreadPoolExecutor(max_workers=len(sessions)) as pool: + pending: Final = tuple(pool.submit(_post_or_none, rig, session) for session in sessions) + eventually(arrived.qsize, lambda size: size == len(sessions), seconds=60) + held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire) for pid in workers}) + victim, survivor = sorted(workers, key=held_by.__getitem__) + with process.replacing_worker(victim) as kill: + kill.freeze() + kill() + release.set() + results: Final = tuple(future.result() for future in pending) + after: Final = rig.send( + "chat", + _marker(), + headers={"x-litellm-trace-id": _token("trace")}, + fields={"litellm_session_id": after_session}, + ) + + served: Final = tuple( + (response, session) + for response, session in zip(results, sessions, strict=True) + if response is not None and response.status_code == 200 + ) + assert sum(held_by.values()) == len(sessions), held_by + assert len(served) == held_by[survivor] >= len(sessions) // 2, (held_by, len(served)) + assert _session(after) == after_session + assert tuple(_session(response) for response, _ in served) == tuple(session for _, session in served) diff --git a/tests/integration/streaming/test_mid_stream_fallback_continuation.py b/tests/integration/streaming/test_mid_stream_fallback_continuation.py new file mode 100644 index 00000000000..8ec52f26dde --- /dev/null +++ b/tests/integration/streaming/test_mid_stream_fallback_continuation.py @@ -0,0 +1,749 @@ +from __future__ import annotations + +import functools +import hashlib +import itertools +import json +import math +import operator +import threading +import uuid +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import openai +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows, scratch_database +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from openai.types.chat import ChatCompletionChunk +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(4 * graceful_stop_seconds() + 120) + +SETTING: Final = "enable_mid_stream_fallback_continuation" +PARTIAL: Final = ("Alpha beta", " gamma") +JOINED_PARTIAL: Final = "".join(PARTIAL) +RELAY_TEXT: Final = " delta" +BACKUP_TEXT: Final = " omega." +PLAIN_TEXT: Final = "plain answer" +TOOL_PREAMBLE: Final = "Let me check" +BACKUP_USAGE: Final = (40, 6) +MAX_TOKENS: Final = 200 +PRIMARY_PRICES: Final = (0.001, 0.002) +BACKUP_PRICES: Final = (0.003, 0.005) +SHARED_BACKEND: Final = "openai/gpt-4o-mini" +SEPARATE_BACKEND: Final = "openai/gpt-4o" +WEATHER_TOOL: Final[Mapping[str, JsonValue]] = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +TOOL_CALL_DELTA: Final[Mapping[str, JsonValue]] = { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, +} +NOTHING_SEEN: Final[tuple[ChatCompletionChunk, ...]] = () +ALLOW_EVERYTHING: Final = "def apply_guardrail(inputs, request_data, input_type):\n return allow()\n" +EVENT_BATCH: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +@functools.cache +def cut_gate(marker: str, hop: str, attempt: int) -> threading.Event: + return threading.Event() + + +@functools.cache +def attempts(marker: str, hop: str) -> Iterator[int]: + return itertools.count() + + +def frame(identity: str, delta: Mapping[str, JsonValue], finish: str | None = None) -> bytes: + body: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish}], + } + return b"data: " + json.dumps(body).encode() + b"\n\n" + + +def complete_stream(identity: str, text: str, usage: tuple[int, int]) -> Reply: + usage_frame: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [], + "usage": {"prompt_tokens": usage[0], "completion_tokens": usage[1], "total_tokens": sum(usage)}, + } + return Reply( + content_type="text/event-stream", + chunks=( + frame(identity, {"role": "assistant", "content": text}), + frame(identity, {}, finish="stop"), + b"data: " + json.dumps(usage_frame).encode() + b"\n\n", + b"data: [DONE]\n\n", + ), + ) + + +def cut_after_caller_sees(marker: str, hop: str, *frames: bytes) -> Reply: + return Reply( + content_type="text/event-stream", + chunks=(b"".join(frames), b"data: never sent\n\n"), + abort_after=1, + gate_after_first=cut_gate(marker, hop, next(attempts(marker, hop))), + ) + + +def first_user_text(request: Request) -> str: + messages: Final = JSON_OBJECT.validate_json(request.body)["messages"] + assert isinstance(messages, list), messages + first: Final = object_value(messages[0])["content"] + assert isinstance(first, str), first + return first + + +def marker_of(request: Request) -> str: + return first_user_text(request).split()[-1] + + +def is_chat(request: Request) -> bool: + return request.target.endswith("/chat/completions") + + +def model_list() -> Reply: + return Reply(body=b'{"object": "list", "data": []}') + + +def primary(request: Request) -> Reply: + if not is_chat(request): + return model_list() + scenario: Final = first_user_text(request).split()[0] + marker: Final = marker_of(request) + identity: Final = f"chatcmpl-{marker}-primary" + if scenario == "reject": + return Reply(status=500, body=b'{"error": {"message": "primary is down", "type": "server_error"}}') + if scenario == "tool-cut": + return cut_after_caller_sees( + marker, + "primary", + frame(identity, {"role": "assistant", "content": TOOL_PREAMBLE}), + frame(identity, {"tool_calls": [dict(TOOL_CALL_DELTA)]}), + ) + return cut_after_caller_sees( + marker, + "primary", + frame(identity, {"role": "assistant", "content": PARTIAL[0]}), + frame(identity, {"content": PARTIAL[1]}), + ) + + +def relay(request: Request) -> Reply: + if not is_chat(request): + return model_list() + marker: Final = marker_of(request) + return cut_after_caller_sees( + marker, "relay", frame(f"chatcmpl-{marker}-relay", {"role": "assistant", "content": RELAY_TEXT}) + ) + + +def backup(request: Request) -> Reply: + if not is_chat(request): + return model_list() + return complete_stream(f"chatcmpl-{marker_of(request)}-backup", BACKUP_TEXT, BACKUP_USAGE) + + +def no_prefill(request: Request) -> Reply: + if not is_chat(request): + return model_list() + return complete_stream(f"chatcmpl-{marker_of(request)}-plain", PLAIN_TEXT, (9, 2)) + + +def accept_events(request: Request) -> Reply: + return Reply() + + +def registered_guardrail(name: str, mode: str) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "custom_code", + "mode": mode, + "default_on": False, + "custom_code": ALLOW_EVERYTHING, + }, + } + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + database_url: str + primary: Wire + relay: Wire + backup: Wire + no_prefill: Wire + events: Wire + + +def joined_text(chunks: Iterable[ChatCompletionChunk]) -> str: + return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + + +def has_tool_call(chunks: Iterable[ChatCompletionChunk]) -> bool: + return any(choice.delta.tool_calls for chunk in chunks for choice in chunk.choices) + + +@dataclass(frozen=True, slots=True) +class Release: + gate: threading.Event + ready: Callable[[tuple[ChatCompletionChunk, ...]], bool] + + +def after_text(marker: str, hop: str, text: str, attempt: int = 0) -> Release: + return Release(cut_gate(marker, hop, attempt), lambda seen: joined_text(seen) == text) + + +@dataclass(frozen=True, slots=True) +class Streamed: + chunks: tuple[ChatCompletionChunk, ...] + error: openai.APIError | None + + @property + def text(self) -> str: + return joined_text(self.chunks) + + @property + def ids(self) -> frozenset[str]: + return frozenset(chunk.id for chunk in self.chunks) + + @property + def role_deltas(self) -> int: + return sum(1 for chunk in self.chunks for choice in chunk.choices if choice.delta.role is not None) + + @property + def finish_reasons(self) -> tuple[str, ...]: + return tuple(choice.finish_reason for chunk in self.chunks for choice in chunk.choices if choice.finish_reason) + + @property + def usage(self) -> tuple[int, int]: + reported: Final = tuple(chunk.usage for chunk in self.chunks if chunk.usage is not None) + assert reported, self.chunks + return (reported[-1].prompt_tokens, reported[-1].completion_tokens) + + +def deployment( + name: str, + wire: Wire, + prices: tuple[float, float], + *, + backend: str = SHARED_BACKEND, + prefill: bool | None = None, +) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": backend, + "api_base": wire.url + "/v1", + "api_key": "synthetic-continuation-key", + "input_cost_per_token": prices[0], + "output_cost_per_token": prices[1], + }, + **({"model_info": {"supports_assistant_prefill": prefill}} if prefill is not None else {}), + } + + +def write_config( + directory: Path, + model_entries: Sequence[Mapping[str, JsonValue]], + fallbacks: Sequence[Mapping[str, JsonValue]], + router_settings: Mapping[str, JsonValue], + guardrails: Sequence[Mapping[str, JsonValue]], +) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "model_list": [dict(entry) for entry in model_entries], + "litellm_settings": { + **object_value(base["litellm_settings"]), + "callbacks": ["generic_api"], + "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, + }, + "guardrails": [dict(entry) for entry in guardrails], + "router_settings": { + **object_value(base["router_settings"]), + "num_retries": 0, + "fallbacks": [dict(entry) for entry in fallbacks], + **router_settings, + }, + } + path: Final = directory / f"continuation-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def booted_rig( + directory: Path, + fallbacks: Sequence[Mapping[str, JsonValue]], + router_settings: Mapping[str, JsonValue], + workers: int, + guardrails: Sequence[Mapping[str, JsonValue]] = (), +) -> Generator[Rig]: + with ExitStack() as stack: + shared: Final = stack.enter_context(gateway_from_environment()) + database_url: Final = stack.enter_context(scratch_database()) + wires: Final = tuple( + stack.enter_context(wire_server(respond)) for respond in (primary, relay, backup, no_prefill, accept_events) + ) + primary_wire, relay_wire, backup_wire, no_prefill_wire, events_wire = wires + config: Final = write_config( + directory, + ( + *( + deployment(name, primary_wire, PRIMARY_PRICES) + for name in ("primary", "primary-skip", "primary-none", "primary-relay", "primary-shared") + ), + deployment("relay", relay_wire, PRIMARY_PRICES, prefill=True), + deployment("backup", backup_wire, BACKUP_PRICES, prefill=True), + deployment("no-prefill", no_prefill_wire, PRIMARY_PRICES, backend=SEPARATE_BACKEND), + deployment("no-prefill-shared", no_prefill_wire, PRIMARY_PRICES), + ), + fallbacks, + router_settings, + guardrails, + ) + gateway: Final = stack.enter_context( + owned_proxy( + shared, + directory, + {"DATABASE_URL": database_url, "GENERIC_LOGGER_ENDPOINT": events_wire.url}, + config=config, + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=workers, + ) + ) + yield Rig(gateway, database_url, primary_wire, relay_wire, backup_wire, no_prefill_wire, events_wire) + + +@pytest.fixture(scope="module") +def configured(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + fallbacks: Final[tuple[Mapping[str, JsonValue], ...]] = ( + {"primary": ["backup"]}, + {"primary-skip": ["no-prefill", "backup"]}, + {"primary-none": ["no-prefill"]}, + {"primary-relay": ["relay"]}, + {"relay": ["backup"]}, + {"primary-shared": ["no-prefill-shared"]}, + ) + with booted_rig( + tmp_path_factory.mktemp("continuation-configured"), + fallbacks, + {SETTING: True}, + 2, + (registered_guardrail("input-check", "pre_call"),), + ) as rig: + yield rig + + +@pytest.fixture(scope="module") +def output_guarded(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with booted_rig( + tmp_path_factory.mktemp("continuation-output-guarded"), + ({"primary": ["backup"]},), + {SETTING: True}, + 2, + (registered_guardrail("output-check", "post_call"),), + ) as rig: + yield rig + + +@pytest.fixture(scope="module") +def unset(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with booted_rig( + tmp_path_factory.mktemp("continuation-unset"), ({"primary": ["no-prefill", "backup"]},), {}, 1 + ) as rig: + yield rig + + +def new_key(rig: Rig) -> str: + key: Final = rig.gateway.post("/key/generate", {})["key"] + assert isinstance(key, str), key + return key + + +def appended(seen: tuple[ChatCompletionChunk, ...], chunk: ChatCompletionChunk) -> tuple[ChatCompletionChunk, ...]: + return (*seen, chunk) + + +def observed( + response: Iterable[ChatCompletionChunk], releases: Sequence[Release] +) -> Iterator[ChatCompletionChunk | openai.APIError]: + try: + for seen in itertools.accumulate(response, appended, initial=NOTHING_SEEN): + for release in releases: + if release.ready(seen): + release.gate.set() + yield from seen[-1:] + except openai.APIError as error: + yield error + + +def stream( + rig: Rig, + key: str, + model: str, + prompt: str, + releases: Sequence[Release] = (), + extra: Mapping[str, JsonValue] | None = None, +) -> Streamed: + base_url: Final = f"{rig.gateway.client.base_url}".rstrip("/") + "/v1" + try: + with openai.OpenAI(base_url=base_url, api_key=key, max_retries=0, timeout=60) as client: + response: Final = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + stream=True, + stream_options={"include_usage": True}, + max_tokens=MAX_TOKENS, + extra_body=dict(extra or {}), + ) + outcome: Final = tuple(observed(response, releases)) + finally: + for release in releases: + release.gate.set() + errors: Final = tuple(item for item in outcome if isinstance(item, openai.APIError)) + return Streamed( + tuple(item for item in outcome if isinstance(item, ChatCompletionChunk)), errors[0] if errors else None + ) + + +def chat_bodies(wire: Wire, marker: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + JSON_OBJECT.validate_json(request.body) + for request in wire.drain() + if is_chat(request) and marker_of(request) == marker + ) + + +def spend_rows(rig: Rig, key: str) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + 'SELECT request_id, status, spend, prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" ' + "WHERE api_key=%s", + (hashlib.sha256(key.encode()).hexdigest(),), + database_url=rig.database_url, + ), + lambda rows: len(rows) >= 1, + seconds=70, + ) + + +def amount(value: JsonValue) -> float: + assert isinstance(value, (int, float)), value + return float(value) + + +def key_spend(rig: Rig, key: str) -> float: + return amount(object_value(rig.gateway.get("/key/info", {"key": key})["info"])["spend"]) + + +def close(observed_value: float, expected: float) -> bool: + return math.isclose(observed_value, expected, rel_tol=1e-9, abs_tol=1e-12) + + +def cost(tokens: tuple[int, int], prices: tuple[float, float]) -> float: + return tokens[0] * prices[0] + tokens[1] * prices[1] + + +def prefill_turn(text: str) -> dict[str, JsonValue]: + return {"role": "assistant", "content": text, "prefix": True} + + +@dataclass(frozen=True, slots=True) +class Billed: + rows: list[dict[str, JsonValue]] + partial: tuple[int, int] + + +def billed_once(rig: Rig, key: str, received: Streamed) -> Billed: + rows: Final = spend_rows(rig, key) + assert [row["status"] for row in rows] == ["success"], rows + row_tokens: Final = (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) + assert isinstance(row_tokens[0], int) and isinstance(row_tokens[1], int), rows + assert received.usage == row_tokens, (received.usage, row_tokens) + partial: Final = (row_tokens[0] - BACKUP_USAGE[0], row_tokens[1] - BACKUP_USAGE[1]) + assert partial[0] > 0 and partial[1] > 0, (row_tokens, BACKUP_USAGE) + return Billed(rows, partial) + + +def assert_spend_prices_every_token(rig: Rig, key: str, billed: Billed) -> None: + partial_cost: Final = cost(billed.partial, PRIMARY_PRICES) + breakdown: Final = object_value(object_value(billed.rows[0]["metadata"])["cost_breakdown"]) + spend: Final = amount(billed.rows[0]["spend"]) + reported_partial_cost: Final = amount(object_value(breakdown["additional_costs"])["mid_stream_partial_cost"]) + assert close(spend, cost(BACKUP_USAGE, BACKUP_PRICES) + partial_cost) and close( + reported_partial_cost, partial_cost + ), f"partial={billed.partial} spend={spend} breakdown={json.dumps(breakdown)}" + eventually(lambda: key_spend(rig, key), lambda value: close(value, spend), seconds=70) + assert spend_rows(rig, key) == billed.rows + + +def test_text_cut_continues_on_the_prefill_fallback_and_bills_both_calls_once(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + prompt: Final = f"text-cut {marker}" + received: Final = stream(configured, key, "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL),)) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks + assert received.ids == {f"chatcmpl-{marker}-primary"}, received.ids + assert received.role_deltas == 1, received.chunks + assert received.finish_reasons == ("stop",), received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + continued: Final = chat_bodies(configured.backup, marker) + assert len(continued) == 1, continued + assert continued[0]["messages"] == [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)] + assert "_mid_stream_continuation" not in continued[0], continued[0] + billed: Final = billed_once(configured, key, received) + assert continued[0]["max_tokens"] == MAX_TOKENS - billed.partial[1], (continued[0], billed.partial) + assert_spend_prices_every_token(configured, key, billed) + + +def test_spend_row_of_a_continued_stream_carries_the_id_the_caller_received(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + received: Final = stream( + configured, key, "primary", f"text-cut {marker}", (after_text(marker, "primary", JOINED_PARTIAL),) + ) + assert received.error is None, received.error + assert [row["request_id"] for row in spend_rows(configured, key)] == [received.chunks[0].id] + + +def test_tool_call_before_the_cut_keeps_the_original_error(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary", + f"tool-cut {marker}", + (Release(cut_gate(marker, "primary", 0), has_tool_call),), + {"tools": [dict(WEATHER_TOOL)]}, + ) + assert received.error is not None, received.chunks + assert (received.text, has_tool_call(received.chunks)) == (TOOL_PREAMBLE, True), received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + assert chat_bodies(configured.backup, marker) == () + + +def test_continuation_skips_a_fallback_without_prefill_support(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + prompt: Final = f"text-cut {marker}" + received: Final = stream( + configured, new_key(configured), "primary-skip", prompt, (after_text(marker, "primary", JOINED_PARTIAL),) + ) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks + assert chat_bodies(configured.no_prefill, marker) == () + assert [body["messages"] for body in chat_bodies(configured.backup, marker)] == [ + [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)] + ] + + +def test_no_prefill_capable_fallback_keeps_the_original_error(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary-none", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + ) + assert received.error is not None, received.chunks + assert "No deployments available" not in str(received.error), received.error + assert received.text == JOINED_PARTIAL, received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + assert chat_bodies(configured.no_prefill, marker) == () + + +def test_undeclared_fallback_sharing_a_backend_with_a_prefill_deployment_keeps_the_original_error( + configured: Rig, +) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary-shared", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + ) + assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks + assert chat_bodies(configured.no_prefill, marker) == () + + +def test_failure_before_content_still_falls_back_to_a_fallback_without_prefill_support(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + prompt: Final = f"reject {marker}" + received: Final = stream(configured, new_key(configured), "primary-none", prompt) + assert received.error is None, received.error + assert received.text == PLAIN_TEXT, received.chunks + assert [body["messages"] for body in chat_bodies(configured.no_prefill, marker)] == [ + [{"role": "user", "content": prompt}] + ] + + +def stream_with_two_cuts(rig: Rig, key: str, marker: str) -> Streamed: + return stream( + rig, + key, + "primary-relay", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL), after_text(marker, "relay", JOINED_PARTIAL + RELAY_TEXT)), + ) + + +def test_second_cut_continues_with_both_partials_in_one_prefill(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + received: Final = stream_with_two_cuts(configured, key, marker) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + RELAY_TEXT + BACKUP_TEXT, received.chunks + assert received.role_deltas == 1, received.chunks + user_turn: Final = {"role": "user", "content": f"text-cut {marker}"} + relayed: Final = chat_bodies(configured.relay, marker) + continued: Final = chat_bodies(configured.backup, marker) + assert [body["messages"] for body in relayed] == [[user_turn, prefill_turn(JOINED_PARTIAL)]] + assert [body["messages"] for body in continued] == [[user_turn, prefill_turn(JOINED_PARTIAL + RELAY_TEXT)]] + partial: Final = billed_once(configured, key, received).partial + relay_ceiling: Final = relayed[0]["max_tokens"] + assert isinstance(relay_ceiling, int), relayed[0] + assert 0 < MAX_TOKENS - relay_ceiling < partial[1], (relayed[0], partial) + assert continued[0]["max_tokens"] == MAX_TOKENS - partial[1], (continued[0], partial) + + +def test_second_cut_bills_every_call_on_one_row(configured: Rig) -> None: + key: Final = new_key(configured) + received: Final = stream_with_two_cuts(configured, key, uuid.uuid4().hex) + assert received.error is None, received.error + assert_spend_prices_every_token(configured, key, billed_once(configured, key, received)) + + +def test_repeating_a_continued_request_never_returns_only_the_continuation(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + prompt: Final = f"text-cut {marker}" + answers: Final = tuple( + stream(configured, key, "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL, attempt),)) + for attempt in range(2) + ) + assert tuple((answer.error, answer.text) for answer in answers) == ((None, JOINED_PARTIAL + BACKUP_TEXT),) * 2 + + +def test_without_the_setting_a_text_cut_keeps_the_original_error(unset: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + unset, new_key(unset), "primary", f"text-cut {marker}", (after_text(marker, "primary", JOINED_PARTIAL),) + ) + assert received.error is not None, received.chunks + assert received.text == JOINED_PARTIAL, received.chunks + assert len(chat_bodies(unset.primary, marker)) == 1 + assert chat_bodies(unset.no_prefill, marker) == () + assert chat_bodies(unset.backup, marker) == () + + +def router_setting(rig: Rig) -> JsonValue: + return object_value(rig.gateway.get("/router/settings")["current_values"]).get(SETTING) + + +def test_turning_the_setting_on_at_runtime_continues_on_a_prefill_fallback(unset: Rig) -> None: + rig: Final = unset + rig.gateway.post("/config/update", {"router_settings": {SETTING: True}}) + try: + assert router_setting(rig) is True + marker: Final = uuid.uuid4().hex + prompt: Final = f"text-cut {marker}" + received: Final = stream(rig, new_key(rig), "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL),)) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks + assert chat_bodies(rig.no_prefill, marker) == () + assert [body["messages"] for body in chat_bodies(rig.backup, marker)] == [ + [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)] + ] + finally: + rig.gateway.post("/config/update", {"router_settings": {SETTING: False}}) + + +def test_a_registered_output_guardrail_keeps_the_original_error(output_guarded: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + output_guarded, + new_key(output_guarded), + "primary", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + ) + assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks + assert len(chat_bodies(output_guarded.primary, marker)) == 1 + assert chat_bodies(output_guarded.backup, marker) == () + + +def test_a_request_fallback_bringing_its_own_messages_keeps_the_original_error(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + {"fallbacks": [{"model": "backup", "messages": [{"role": "user", "content": f"start over {marker}"}]}]}, + ) + assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + assert chat_bodies(configured.backup, marker) == () + + +def carries_tag(event: Mapping[str, JsonValue], marker: str) -> bool: + tags: Final = event.get("request_tags") + return isinstance(tags, list) and marker in tags + + +def tagged_events(batches: Iterable[Request], marker: str) -> tuple[dict[str, JsonValue], ...]: + events: Final = itertools.chain.from_iterable(EVENT_BATCH.validate_json(batch.body) for batch in batches) + return tuple(event for event in events if carries_tag(event, marker)) + + +def logged_events(sink: Wire, marker: str) -> Iterator[tuple[dict[str, JsonValue], ...]]: + return itertools.accumulate((tagged_events(sink.drain(), marker) for _ in itertools.count()), operator.add) + + +def test_the_cut_call_logs_its_failure_under_its_own_deployment_and_price(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + {"metadata": {"tags": [marker]}}, + ) + assert received.error is None, received.error + polls: Final = logged_events(configured.events, marker) + events: Final = eventually(lambda: next(polls), lambda logged: len(logged) >= 2, seconds=30) + assert sorted(str(event["status"]) for event in events) == ["failure", "success"], events + failure: Final = next(event for event in events if event["status"] == "failure") + assert failure["model_group"] == "primary", failure + assert str(failure["api_base"]).startswith(configured.primary.url), failure + tokens: Final = (failure["prompt_tokens"], failure["completion_tokens"]) + assert isinstance(tokens[0], int) and isinstance(tokens[1], int) and tokens[0] > 0 and tokens[1] > 0, failure + assert close(amount(failure["response_cost"]), cost((tokens[0], tokens[1]), PRIMARY_PRICES)), failure diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index c878f0e149d..bf977325a85 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -642,6 +642,64 @@ async def test_bedrock_converse_replays_reasoning_to_claude_only_as_base64_of_cl assert assistant_content == [*sent, {"text": "hello"}] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "model, final_text", + [("us.anthropic.claude-sonnet-4-6", "Leaves fall"), ("us.amazon.nova-pro-v1:0", "Leaves fall \n")], + ids=["claude", "nova"], +) +async def test_bedrock_converse_strips_trailing_whitespace_from_a_final_assistant_turn_for_claude_only( + use_async: bool, model: str, final_text: str +): + """Claude on Converse 400s on a final assistant turn ending in whitespace, which a cut stream's prefill often has.""" + messages: Final = [ + {"role": "user", "content": "Write a haiku."}, + {"role": "assistant", "content": "Sure. "}, + {"role": "user", "content": "About autumn."}, + {"role": "assistant", "content": "Leaves fall \n"}, + ] + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert [m["content"] for m in result if m["role"] == "assistant"] == [ + [{"text": "Sure. "}], + [{"text": final_text}], + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_bedrock_converse_keeps_whitespace_between_text_blocks_of_a_final_claude_assistant_turn( + use_async: bool, +): + messages: Final = [ + {"role": "user", "content": "Write a haiku."}, + { + "role": "assistant", + "content": [{"type": "text", "text": "Leaves fall "}, {"type": "text", "text": "softly \n"}], + }, + ] + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model="us.anthropic.claude-sonnet-4-6", llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt( + messages=messages, model="us.anthropic.claude-sonnet-4-6", llm_provider="bedrock" + ) + ) + + assert result[-1]["content"] == [{"text": "Leaves fall "}, {"text": "softly"}] + + def test_translate_thinking_blocks_keeps_dropping_redacted_data_when_called_the_old_way(): """Outside callers that only pass the thinking blocks keep the old behavior, with no redactedContent sent.""" assert BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 4347a6639bd..fa649b48a51 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1019,6 +1019,87 @@ async def test_bridged_stream_mid_stream_fallback_error_is_rebuilt_around_the_pr assert excinfo.value.generated_content == "" +@pytest.mark.asyncio +@pytest.mark.parametrize("merge_reasoning", [True, False], ids=["merged_into_content", "kept_as_reasoning"]) +async def test_mid_stream_break_after_reasoning_disqualifies_continuation(logging_obj: Logging, merge_reasoning: bool): + import httpx + + from litellm.exceptions import MidStreamFallbackError + + class _ReasoningThenCut: + def __init__(self) -> None: + self.chunks = iter( + ( + ModelResponseStream(choices=[{"index": 0, "delta": {"reasoning_content": "Let me think"}}]), + ModelResponseStream(choices=[{"index": 0, "delta": {"content": "The answer"}}]), + ) + ) + + def __aiter__(self) -> "_ReasoningThenCut": + return self + + async def __anext__(self) -> ModelResponseStream: + next_chunk: Final = next(self.chunks, None) + if next_chunk is None: + raise httpx.ReadTimeout("connection dropped") + return next_chunk + + response: Final = CustomStreamWrapper( + completion_stream=_ReasoningThenCut(), model="deepseek-reasoner", logging_obj=logging_obj + ) + response.merge_reasoning_content_in_choices = merge_reasoning + + with pytest.raises(MidStreamFallbackError) as excinfo: + async for _ in response: + pass + + assert excinfo.value.is_pre_first_chunk is False + assert excinfo.value.emitted_disqualifying_content is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outer_delta,inner_flag,expected", + [ + ({"tool_calls": [{"index": 0}]}, False, True), + ({"content": "Let me check."}, True, True), + ({"content": "Let me check."}, False, False), + ], + ids=["bridge_wrapper_sent_a_tool_call", "inner_stream_sent_a_tool_call", "plain_text_only"], +) +async def test_bridged_stream_rebuild_keeps_the_continuation_disqualifier( + logging_obj: Logging, outer_delta: dict[str, object], inner_flag: bool, expected: bool +): + from litellm.exceptions import InternalServerError, MidStreamFallbackError + + provider_error: Final = InternalServerError(message="upstream 500", llm_provider="azure", model="gpt-6.1-sol") + inner_error: Final = MidStreamFallbackError( + message=str(provider_error), + model="gpt-6.1-sol", + llm_provider="azure", + original_exception=provider_error, + is_pre_first_chunk=False, + emitted_disqualifying_content=inner_flag, + ) + + async def _raise_inner_error(**kwargs: object) -> NoReturn: + raise inner_error + + response: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-6.1-sol", + logging_obj=logging_obj, + custom_llm_provider="azure", + make_call=_raise_inner_error, + ) + response._accumulate_streamed_delta(outer_delta) + + with pytest.raises(MidStreamFallbackError) as excinfo: + await response.__anext__() + + assert excinfo.value.emitted_disqualifying_content is expected + + @pytest.mark.asyncio @pytest.mark.parametrize("sync_mode", [True, False]) async def test_streaming_rate_limit_midstream_fallback_keeps_provider_headers(logging_obj: Logging, sync_mode: bool): @@ -6230,6 +6311,59 @@ class TestStableStreamingResponseId: assert wrapper.model_response_creator().id == "chatcmpl-from-provider" +class TestContinuationDisqualifiers: + """The mid-stream continuation eligibility hinges on classifying which deltas + carry output a text-only prefill cannot represent.""" + + @pytest.mark.parametrize( + "field", + [ + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", + ], + ) + def test_disqualifying_fields_flagged(self, field): + # reasoning_content included: it reaches the caller as visible reasoning a + # text-only prefill cannot carry, so a reasoning fallback would re-derive it + assert CustomStreamWrapper._delta_disqualifies_continuation({field: [{"x": 1}]}) is True + + @pytest.mark.parametrize("delta", [{"content": "hi"}, {}, {"role": "assistant"}]) + def test_plain_text_not_flagged(self, delta): + assert CustomStreamWrapper._delta_disqualifies_continuation(delta) is False + + def test_non_mapping_delta_is_safe(self): + assert CustomStreamWrapper._delta_disqualifies_continuation(object()) is False + + def test_accumulate_grows_text_and_leaves_flag_clear_for_plain_text(self): + wrapper = object.__new__(CustomStreamWrapper) + wrapper.response_uptil_now = "" + wrapper._emitted_disqualifying_content = False + + wrapper._accumulate_streamed_delta({"content": "Hel"}) + wrapper._accumulate_streamed_delta({"content": "lo"}) + + assert wrapper.response_uptil_now == "Hello" + assert wrapper._emitted_disqualifying_content is False + + def test_accumulate_latches_flag_on_disqualifying_delta(self): + wrapper = object.__new__(CustomStreamWrapper) + wrapper.response_uptil_now = "" + wrapper._emitted_disqualifying_content = False + + wrapper._accumulate_streamed_delta({"content": "Hi"}) + wrapper._accumulate_streamed_delta({"tool_calls": [{"index": 0}]}) + wrapper._accumulate_streamed_delta({"content": "there"}) + + assert wrapper.response_uptil_now == "Hithere" + assert wrapper._emitted_disqualifying_content is True + + @pytest.mark.asyncio async def test_async_stream_without_usage_counts_tokens_off_the_event_loop(): from tests.large_text import text diff --git a/tests/unit/llms/dashscope/test_dashscope_video_transformation.py b/tests/unit/llms/dashscope/test_dashscope_video_transformation.py index 65eefcab1af..15b77aaee3d 100644 --- a/tests/unit/llms/dashscope/test_dashscope_video_transformation.py +++ b/tests/unit/llms/dashscope/test_dashscope_video_transformation.py @@ -194,8 +194,9 @@ class TestDashScopeVideoInputReference: lambda tmp_path: ("first.png", PNG_BYTES, "image/png"), lambda tmp_path: ("first.png", io.BytesIO(PNG_BYTES)), lambda tmp_path: _write_png(tmp_path), + lambda tmp_path: ("first.png", ("inner.png", PNG_BYTES)), ], - ids=["filename-bytes-content-type", "filename-file", "path"], + ids=["filename-bytes-content-type", "filename-file", "path", "nested-tuple"], ) def test_file_tuple_and_path_references_are_encoded(self, make_reference, tmp_path): """Regression: tuples and paths are valid FileTypes but raised before reaching DashScope.""" diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 650517c92ca..1e353a7a82a 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -461,6 +461,21 @@ def test_transform_messages_sanitizes_empty_content(): assert result[1]["content"] == "Hi" +def test_transform_messages_drops_the_prefill_prefix_flag(): + """Databricks rejects unknown message keys, so a continuation prefill reaches Claude without ``prefix``.""" + messages = [ + {"role": "user", "content": "How do transformers work?"}, + {"role": "assistant", "content": "Transformers use", "prefix": True}, + ] + + result = DatabricksConfig().transform_messages( + messages=messages, model="databricks-claude-sonnet-4-6", is_async=False + ) + + assert result[-1] == {"role": "assistant", "content": "Transformers use"} + assert messages[-1]["prefix"] is True + + def test_transform_request_preserves_unity_model_service_name(): config = DatabricksConfig() result = config.transform_request( diff --git a/tests/unit/llms/volcengine/test_volcengine_video_transformation.py b/tests/unit/llms/volcengine/test_volcengine_video_transformation.py new file mode 100644 index 00000000000..1fcf49eeb13 --- /dev/null +++ b/tests/unit/llms/volcengine/test_volcengine_video_transformation.py @@ -0,0 +1,1131 @@ +import base64 +import io +import json +from pathlib import Path +from typing import Final +from unittest.mock import Mock + +import httpx +import pytest +import respx + +import litellm +from litellm import Router +from litellm.exceptions import UnsupportedParamsError +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai.cost_calculation import video_generation_cost +from litellm.llms.volcengine.videos.transformation import VolcEngineVideoConfig, VolcEngineVideoError +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.utils import decode_video_id_with_provider, encode_video_id_with_provider +from litellm.utils import ProviderConfigManager +from litellm.videos.main import avideo_content, avideo_generation + +PNG_BYTES: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URL: Final = f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}" + +API_BASE: Final = "https://ark.cn-beijing.volces.com" +TASKS_URL: Final = f"{API_BASE}/api/v3/contents/generations/tasks" +SEEDANCE_2_5: Final = "doubao-seedance-2-5-260628" +SEEDANCE_2_0: Final = "doubao-seedance-2-0-260128" +SEEDANCE_2_0_FAST: Final = "doubao-seedance-2-0-fast-260128" +TASK_ID: Final = "cgt-2026-abc" + + +def _response(payload: object, status_code: int = 200) -> httpx.Response: + return httpx.Response(status_code, json=payload, request=httpx.Request("GET", TASKS_URL)) + + +def _text_response(text: str, status_code: int = 200) -> httpx.Response: + return httpx.Response(status_code, text=text, request=httpx.Request("GET", TASKS_URL)) + + +def _create_body( + params: dict[str, object], model: str = SEEDANCE_2_5, drop_params: bool | None = None +) -> dict[str, object]: + body, files, url = VolcEngineVideoConfig().transform_video_create_request( + model=model, + prompt="a cat on a roof", + api_base=API_BASE, + video_create_optional_request_params=params, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + assert (files, url) == ((), TASKS_URL) + return body + + +def _created(request_data: dict[str, object], model: str = SEEDANCE_2_5, logging_obj: object = None): + return VolcEngineVideoConfig().transform_video_create_response( + model=model, + raw_response=_response({"id": TASK_ID}), + logging_obj=logging_obj, # pyright: ignore[reportArgumentType] # the transform only reads litellm_params off it + custom_llm_provider="volcengine", + request_data=request_data, + ) + + +def _polled(payload: object, logging_obj: object = None): + return VolcEngineVideoConfig().transform_video_status_retrieve_response( + raw_response=_response(payload), + logging_obj=logging_obj, # pyright: ignore[reportArgumentType] # the transform only reads litellm_params off it + custom_llm_provider="volcengine", + ) + + +def _task(**fields: object) -> dict[str, object]: + return {"id": TASK_ID, "model": SEEDANCE_2_5, "status": "succeeded", **fields} + + +@pytest.mark.parametrize( + "input_reference", + [ + PNG_BYTES, + io.BytesIO(PNG_BYTES), + ("frame.png", PNG_BYTES, "image/png"), + ("frame.png", io.BytesIO(PNG_BYTES)), + ], + ids=["bytes", "file", "tuple", "tuple-with-file"], +) +def test_a_file_input_reference_becomes_a_data_uri_first_frame(input_reference: object): + body = _create_body({"input_reference": input_reference, "duration": 4}) + + assert body == { + "model": SEEDANCE_2_5, + "content": [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": PNG_DATA_URL}, "role": "first_frame"}, + ], + "resolution": "720p", + "duration": 4, + } + + +def test_a_path_input_reference_is_read_from_disk(tmp_path: Path): + frame = tmp_path / "frame.png" + frame.write_bytes(PNG_BYTES) + + body = _create_body({"input_reference": frame, "duration": 4}) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": PNG_DATA_URL}, "role": "first_frame"}, + ] + + +def test_a_url_input_reference_is_passed_as_is_and_never_opened_as_a_path(): + body = _create_body({"input_reference": "https://x/first.png", "duration": 4}) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": "https://x/first.png"}, "role": "first_frame"}, + ] + + +def test_openais_object_input_reference_sends_its_image_url(): + body = _create_body({"input_reference": {"image_url": "https://x/first.png"}, "duration": 4}) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": "https://x/first.png"}, "role": "first_frame"}, + ] + + +def test_a_file_id_input_reference_is_a_bad_request_unless_drop_params(): + with pytest.raises(UnsupportedParamsError, match="file_id is not supported"): + _ = _create_body({"input_reference": {"file_id": "file-abc"}, "duration": 4}) + + body = _create_body({"input_reference": {"file_id": "file-abc"}, "duration": 4}, drop_params=True) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}] + + +def test_a_caller_content_array_wins_over_input_reference_and_the_prompt_leads(): + body = _create_body( + { + "content": [{"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}], + "input_reference": "https://x/ignored.png", + "duration": 4, + } + ) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}, + ] + + +IMAGE_ITEM: Final = {"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"} + + +def test_a_text_item_in_content_is_rejected_because_input_guardrails_only_scan_the_prompt(): + with pytest.raises(UnsupportedParamsError, match="put the text in prompt"): + _ = _create_body({"content": [{"type": "text", "text": "text the guardrail never saw"}], "duration": 4}) + + +def test_a_text_item_in_content_is_dropped_under_drop_params_so_ark_never_sees_unscanned_text(): + body = _create_body( + {"content": [{"type": "text", "text": "text the guardrail never saw"}, IMAGE_ITEM], "duration": 4}, + drop_params=True, + ) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}, IMAGE_ITEM] + + +@pytest.mark.parametrize( + "item", + [ + {"type": "video_url", "video_url": {"url": "https://x/v.mp4"}, "role": "reference_video"}, + {"type": "video_url", "video_url": {"url": "https://x/v.mp4"}}, + {"type": "image_url", "image_url": {"url": "https://x/v.png"}, "role": "reference_video"}, + ], + ids=["reference-video", "untagged-video", "mistagged-role"], +) +def test_a_video_input_is_rejected_because_its_cost_is_not_known_at_create(item: dict[str, object]): + with pytest.raises(UnsupportedParamsError, match="Video input"): + _ = _create_body({"content": [item], "duration": 4}) + + body = _create_body({"content": [item, IMAGE_ITEM], "duration": 4}, drop_params=True) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}, IMAGE_ITEM] + + +@pytest.mark.parametrize( + "content", + ["a string", [{"type": "image_url", "role": 7}], [{"type": 7}]], + ids=["not-an-array", "non-string-role", "non-string-type"], +) +def test_a_content_value_that_is_not_an_array_of_items_is_rejected_unless_drop_params(content: object): + with pytest.raises(UnsupportedParamsError, match="content must be an array"): + _ = _create_body({"content": content, "duration": 4}) + + body = _create_body({"content": content, "duration": 4}, drop_params=True) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}] + + +def test_an_omitted_duration_and_tier_are_sent_as_openais_four_seconds_at_720p_so_ark_cannot_pick_what_is_billed(): + body = _create_body({}) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "720p", + "duration": 4, + } + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_an_endpoint_id_priced_through_base_model_gets_the_base_models_default_tier(): + body, _, _ = VolcEngineVideoConfig().transform_video_create_request( + model="ep-20261015-abc", + prompt="a cat on a roof", + api_base=API_BASE, + video_create_optional_request_params={"duration": 4}, + litellm_params=GenericLiteLLMParams(model_info={"base_model": f"volcengine/{SEEDANCE_2_5}"}), + headers={}, + ) + + assert (body["model"], body["resolution"]) == ("ep-20261015-abc", "720p") + + +@pytest.mark.parametrize("field", ["return_last_frame", "draft", "frame_count"]) +def test_a_field_litellm_does_not_send_to_ark_is_rejected(field: str): + with pytest.raises(UnsupportedParamsError, match=f"does not send {field} to Ark"): + _ = _create_body({"duration": 4, field: True}) + + +def test_a_field_litellm_does_not_send_to_ark_is_dropped_under_drop_params(): + body = _create_body({"duration": 4, "return_last_frame": True, "seed": 7}, drop_params=True) + + assert "return_last_frame" not in body + assert body["seed"] == 7 + + +@pytest.mark.parametrize("duration", [-1, "-1", 0, "abc", 4.5]) +def test_a_duration_ark_would_pick_or_that_is_not_whole_seconds_is_rejected(duration: object): + with pytest.raises(UnsupportedParamsError, match="duration must be a positive whole number"): + _ = _create_body({"duration": duration}) + + assert _create_body({"duration": duration}, drop_params=True)["duration"] == 4 + + +def test_frames_is_rejected_because_it_bypasses_the_billed_duration(): + with pytest.raises(UnsupportedParamsError, match="frames"): + _ = _create_body({"frames": 121, "duration": 6}) + + body = _create_body({"frames": 121, "duration": 6}, drop_params=True) + + assert ("frames" not in body, body["duration"]) == (True, 6) + + +def test_ark_fields_reach_the_body_and_false_flags_are_kept(): + body = _create_body( + { + "duration": 4, + "resolution": "480p", + "ratio": "9:16", + "seed": 7, + "watermark": False, + "generate_audio": False, + "service_tier": "flex", + } + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "480p", + "ratio": "9:16", + "seed": 7, + "watermark": False, + "generate_audio": False, + "service_tier": "flex", + "duration": 4, + } + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_resolution_the_model_has_no_price_for_is_rejected(): + with pytest.raises(UnsupportedParamsError, match="not offered at resolution '1080p', only at 480p, 720p"): + _ = _create_body({"duration": 4, "resolution": "1080p"}, model=SEEDANCE_2_0_FAST) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_an_unpriced_resolution_is_dropped_under_drop_params(): + body = _create_body( + {"duration": 4, "resolution": "1080p", "ratio": "16:9"}, model=SEEDANCE_2_0_FAST, drop_params=True + ) + + assert body == { + "model": SEEDANCE_2_0_FAST, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "720p", + "ratio": "16:9", + "duration": 4, + } + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_model_the_router_registered_without_prices_keeps_its_resolution(): + endpoint_id: Final = "ep-20261015-unpriced" + _ = Router( + model_list=[ + {"model_name": "ark", "litellm_params": {"model": f"volcengine/{endpoint_id}", "api_key": "sk-test"}} + ] + ) + + body = _create_body({"duration": 4, "resolution": "480p"}, model=endpoint_id) + body_without_a_tier = _create_body({"duration": 4}, model=endpoint_id) + + assert body["resolution"] == "480p" + assert "resolution" not in body_without_a_tier + + +def test_a_mistyped_ark_field_is_rejected(): + with pytest.raises(UnsupportedParamsError, match="invalid seed"): + _ = _create_body({"duration": 4, "seed": "abc", "watermark": False}) + + +def test_a_mistyped_ark_field_is_dropped_under_drop_params(): + body = _create_body({"duration": 4, "seed": "abc", "watermark": False}, drop_params=True) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "720p", + "watermark": False, + "duration": 4, + } + + +@pytest.mark.parametrize( + "size,expected", + [ + ("1920x1080", {"ratio": "16:9", "resolution": "1080p"}), + ("854x480", {"ratio": "16:9", "resolution": "480p"}), + ("864x496", {"ratio": "16:9", "resolution": "480p"}), + ("720x1280", {"ratio": "9:16", "resolution": "720p"}), + ("1024x768", {"ratio": "4:3", "resolution": "720p"}), + ("640x640", {"ratio": "1:1", "resolution": "480p"}), + ("960x960", {"ratio": "1:1", "resolution": "720p"}), + ("1440x1440", {"ratio": "1:1", "resolution": "1080p"}), + ("1112x834", {"ratio": "4:3", "resolution": "720p"}), + ("1470x630", {"ratio": "21:9", "resolution": "720p"}), + ("3840x2160", {"ratio": "16:9", "resolution": "4k"}), + ], +) +def test_size_maps_onto_an_ark_ratio_and_tier(size: str, expected: dict[str, str]): + assert VolcEngineVideoConfig().map_openai_params({"size": size}, SEEDANCE_2_5, False) == expected + + +def test_explicit_ratio_and_resolution_beat_the_size_derived_ones(): + mapped = VolcEngineVideoConfig().map_openai_params( + {"size": "1280x720", "seconds": "4", "ratio": "1:1", "resolution": "480p", "parameters": {"seed": 3}}, + SEEDANCE_2_5, + False, + ) + + assert mapped == {"ratio": "1:1", "resolution": "480p", "duration": 4, "seed": 3} + + +@pytest.mark.parametrize( + "params,message", + [ + ({"seconds": "4.5"}, "seconds must be a whole number"), + ({"size": "big"}, "size must be WIDTHxHEIGHT"), + ({"size": "16:9"}, "size must be WIDTHxHEIGHT"), + ({"parameters": "seed=3"}, "parameters must be an object"), + ({"characters": [{"id": "c"}]}, "'characters' is not supported"), + ({"image": "https://x/a.png"}, "'image' is not supported"), + ], +) +def test_a_value_ark_cannot_take_raises_unless_drop_params(params: dict[str, object], message: str): + with pytest.raises(UnsupportedParamsError, match=message): + _ = VolcEngineVideoConfig().map_openai_params(params, SEEDANCE_2_5, False) + + assert VolcEngineVideoConfig().map_openai_params(params, SEEDANCE_2_5, True) == {} + + +def test_request_metadata_is_not_mapped_into_the_body_and_user_becomes_the_safety_identifier(): + mapped = VolcEngineVideoConfig().map_openai_params( + {"user": "u", "extra_headers": {"x": "y"}, "seconds": 4}, SEEDANCE_2_5, False + ) + + assert mapped == {"safety_identifier": "u", "duration": 4} + + +def test_an_explicit_safety_identifier_wins_over_user(): + mapped = VolcEngineVideoConfig().map_openai_params( + {"user": "u", "safety_identifier": "s", "seconds": 4}, SEEDANCE_2_5, False + ) + + assert mapped["safety_identifier"] == "s" + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "model,request_data,seconds,size,usage", + [ + ( + SEEDANCE_2_5, + {"duration": 4, "resolution": "480p", "ratio": "16:9"}, + "4", + "854x480", + {"duration_seconds": 4.0, "video_resolution": "480p"}, + ), + ( + SEEDANCE_2_0, + {"duration": 5, "resolution": "480p", "ratio": "16:9"}, + "5", + "864x496", + {"duration_seconds": 5.0, "video_resolution": "480p"}, + ), + (SEEDANCE_2_5, {"duration": 5}, "5", None, {"duration_seconds": 5.0}), + ], + ids=["2.5-480p", "2.0-480p-renders-its-own-frame", "no-geometry"], +) +def test_the_create_response_reports_what_the_task_bills( + model: str, request_data: dict[str, object], seconds: str, size: str | None, usage: dict[str, object] +): + video = _created(request_data, model=model) + + assert video.model_dump(include={"status", "seconds", "size", "usage", "model"}) == { + "status": "queued", + "seconds": seconds, + "size": size, + "usage": usage, + "model": model, + } + + +@pytest.mark.parametrize( + "usage,expected_cost", + [ + ({"duration_seconds": 4.0, "video_resolution": "480p"}, 4 * 0.1), + ({"duration_seconds": 5.0, "video_resolution": "720p"}, 5 * 0.2), + ({"duration_seconds": 5.0}, 5 * 0.2), + ], +) +def test_the_create_usage_bills_its_duration_at_its_tier_rate(usage: dict[str, object], expected_cost: float): + model_info = { + "output_cost_per_second": 0.2, + "output_cost_per_second_480p": 0.1, + "output_cost_per_second_720p": 0.2, + } + video = _created({"duration": int(usage["duration_seconds"]), "resolution": usage.get("video_resolution")}) + + cost = video_generation_cost( + model=SEEDANCE_2_5, + duration_seconds=video.usage["duration_seconds"], + model_info=model_info, # pyright: ignore[reportArgumentType] # a hand-written rate card stands in for ModelInfo + video_resolution=video.usage.get("video_resolution"), + ) + + assert cost == pytest.approx(expected_cost) + + +def test_the_created_id_carries_the_requested_and_served_model_groups(): + logging_obj = Mock( + litellm_params={"litellm_metadata": {"model_group": "seedance", "deployment_model_name": "seedance-account-b"}} + ) + + video = _created({"duration": 5}, logging_obj=logging_obj) + + assert decode_video_id_with_provider(video.id) == { + "custom_llm_provider": "volcengine", + "model_id": "seedance", + "video_id": TASK_ID, + "served_model": "seedance-account-b", + } + + +def test_a_create_without_router_metadata_carries_the_model(): + video = _created({"duration": 5}, logging_obj=Mock(litellm_params={})) + + assert decode_video_id_with_provider(video.id) == { + "custom_llm_provider": "volcengine", + "model_id": SEEDANCE_2_5, + "video_id": TASK_ID, + } + + +@pytest.mark.parametrize( + "raw_response,status_code,message", + [ + (_response({"error": {"code": "InvalidParameter", "message": "bad ratio"}}), 400, "bad ratio"), + (_response({"error": {"code": "Forbidden", "message": "no access"}}, 403), 403, "no access"), + (_text_response("bad gateway"), 502, "Unexpected Volcengine video response"), + ], + ids=["error-on-200", "error-status-kept", "not-json"], +) +def test_a_create_response_without_a_task_id_raises_instead_of_queueing_an_empty_video( + raw_response: httpx.Response, status_code: int, message: str +): + with pytest.raises(VolcEngineVideoError, match=message) as raised: + _ = VolcEngineVideoConfig().transform_video_create_response( + model=SEEDANCE_2_5, + raw_response=raw_response, + logging_obj=None, # pyright: ignore[reportArgumentType] # the transform only reads litellm_params off it + custom_llm_provider="volcengine", + request_data={"duration": 5}, + ) + + assert raised.value.status_code == status_code + + +@pytest.mark.parametrize( + "ark_status,expected", + [ + ("queued", "queued"), + ("running", "in_progress"), + ("succeeded", "completed"), + ("failed", "failed"), + ("cancelled", "cancelled"), + ("expired", "failed"), + ], +) +def test_the_ark_status_maps_onto_the_openai_vocabulary(ark_status: str, expected: str): + assert _polled(_task(status=ark_status)).status == expected + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_completed_task_reports_its_delivered_geometry_duration_and_tokens(): + video = _polled( + _task( + created_at=1791989938, + updated_at=1791990031, + resolution="480p", + ratio="16:9", + duration=4, + usage={"completion_tokens": 40594, "total_tokens": 40594}, + model=SEEDANCE_2_0, + content={"video_url": "https://tos/v.mp4"}, + ) + ) + + assert video.model_dump(exclude={"id"}) | {"id": decode_video_id_with_provider(video.id)["video_id"]} == { + "id": TASK_ID, + "object": "video", + "status": "completed", + "created_at": 1791989938, + "completed_at": 1791990031, + "expires_at": None, + "error": None, + "progress": None, + "remixed_from_video_id": None, + "seconds": "4", + "size": "864x496", + "model": SEEDANCE_2_0, + "usage": {"duration_seconds": 4.0, "video_resolution": "480p", "total_tokens": 40594}, + } + + +@pytest.mark.parametrize( + "resolution,ratio,expected", + [ + ("720p", "9:16", "720x1280"), + ("1080p", "16:9", "1920x1080"), + ("480p", "adaptive", None), + ], +) +def test_a_frame_the_cost_map_does_not_list_is_scaled_from_the_tier(resolution: str, ratio: str, expected: str | None): + assert _polled(_task(resolution=resolution, ratio=ratio, model="doubao-seedance-unlisted")).size == expected + + +def test_a_failed_task_carries_its_error_block(): + video = _polled(_task(status="failed", error={"code": "OutputVideoSensitiveContentDetected", "message": "flagged"})) + + assert (video.status, video.error, video.completed_at) == ( + "failed", + {"code": "OutputVideoSensitiveContentDetected", "message": "flagged"}, + None, + ) + + +@pytest.mark.parametrize( + "payload,message", + [ + (_task(status="paused"), "Unknown Volcengine task status: paused"), + ({"status": "running"}, "Unexpected Volcengine video response"), + ({"id": TASK_ID}, "Unexpected Volcengine video response"), + ], + ids=["unmapped-status", "no-id", "no-status"], +) +def test_a_poll_litellm_cannot_read_raises_instead_of_returning_a_job_to_poll_forever(payload: object, message: str): + with pytest.raises(VolcEngineVideoError, match=message) as raised: + _ = _polled(payload) + + assert raised.value.status_code == 502 + + +def test_a_poll_keeps_the_ids_requested_and_served_model_when_the_router_served_it(): + video_id = encode_video_id_with_provider(TASK_ID, "volcengine", "seedance", served_model="seedance-account-b") + logging_obj = Mock( + litellm_params={ + "video_id": video_id, + "litellm_metadata": {"model_group": "seedance", "deployment_model_name": "seedance-account-b"}, + } + ) + + video = _polled(_task(status="running"), logging_obj=logging_obj) + + assert video.id == video_id + assert video.model == "seedance" + + +def test_a_poll_drops_a_served_model_the_router_did_not_route_to(): + edited = encode_video_id_with_provider( + TASK_ID, "volcengine", "seedance-account-a", served_model="seedance-account-b" + ) + logging_obj = Mock(litellm_params={"video_id": edited, "litellm_metadata": {"model_group": "seedance-account-a"}}) + + video = _polled(_task(status="running"), logging_obj=logging_obj) + + assert video.id == encode_video_id_with_provider(TASK_ID, "volcengine", "seedance-account-a") + + +async def _download(task_payload: dict[str, object], video_bytes: bytes = b"mp4") -> bytes: + def respond(request: httpx.Request) -> httpx.Response: + if request.url.host == "tos.example": + return httpx.Response(200, content=video_bytes) + return httpx.Response(200, json=task_payload) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + return await avideo_content( + video_id=TASK_ID, + custom_llm_provider="volcengine", + model=f"volcengine/{SEEDANCE_2_5}", + api_key="sk-test", + api_base=API_BASE, + client=handler, + ) + + +@pytest.mark.asyncio +async def test_the_download_fetches_the_signed_url_through_the_callers_client(): + content = await _download(_task(content={"video_url": "https://tos.example/v.mp4"}), video_bytes=b"the-video") + + assert content == b"the-video" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload,message", + [ + (_task(status="queued"), "still processing"), + (_task(status="running"), "still processing"), + (_task(status="failed", error={"message": "flagged"}), "flagged"), + (_task(status="failed"), "ended as failed"), + (_task(status="cancelled"), "ended as cancelled"), + (_task(status="expired"), "Task expired"), + ], +) +async def test_a_download_with_no_video_yet_is_a_bad_request_so_it_is_neither_retried_nor_cools_the_deployment( + payload: dict[str, object], message: str +): + with pytest.raises(litellm.BadRequestError, match=message) as raised: + _ = await _download(payload) + + assert raised.value.status_code == 400 + + +def test_the_list_maps_ark_paging_and_filters(): + url, params = VolcEngineVideoConfig().transform_video_list_request( + api_base=API_BASE, + litellm_params=GenericLiteLLMParams(), + headers={}, + limit=20, + extra_query={"filter.status": "succeeded"}, + ) + + assert (url, params) == (TASKS_URL, {"page_size": "20", "filter.status": "succeeded"}) + + +@pytest.mark.parametrize("paging", [{"after": "video_abc"}, {"order": "asc"}]) +def test_a_list_cursor_or_order_is_rejected_unless_drop_params(paging: dict[str, str]): + config = VolcEngineVideoConfig() + + with pytest.raises(UnsupportedParamsError, match="pages by page_num"): + _ = config.transform_video_list_request( + api_base=API_BASE, litellm_params=GenericLiteLLMParams(), headers={}, **paging + ) + _, params = config.transform_video_list_request( + api_base=API_BASE, litellm_params=GenericLiteLLMParams(drop_params=True), headers={}, limit=5, **paging + ) + + assert params == {"page_size": "5"} + + +def test_the_list_response_is_an_openai_list_body(): + response = VolcEngineVideoConfig().transform_video_list_response( + raw_response=_response({"total": 1, "items": [_task(status="running")]}), + logging_obj=Mock(), # pyright: ignore[reportArgumentType] # unused by the transform + custom_llm_provider="volcengine", + ) + + encoded = encode_video_id_with_provider(TASK_ID, "volcengine", SEEDANCE_2_5) + assert {key: value for key, value in response.items() if key != "data"} == { + "object": "list", + "total": 1, + "has_more": False, + "first_id": encoded, + "last_id": encoded, + } + assert [(item["id"], item["status"]) for item in response["data"]] == [(encoded, "in_progress")] + + +def test_an_empty_list_still_carries_the_object_type(): + response = VolcEngineVideoConfig().transform_video_list_response( + raw_response=_response({"total": 0, "items": []}), + logging_obj=Mock(), # pyright: ignore[reportArgumentType] # unused by the transform + ) + + assert response == {"object": "list", "data": [], "total": 0, "has_more": False} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call", ["avideo_edit", "avideo_extension", "avideo_remix"]) +async def test_edit_extension_and_remix_are_a_bad_request_because_a_video_input_cannot_be_billed_at_create(call: str): + with pytest.raises(litellm.BadRequestError, match="Ark bills a task with a video input") as raised: + _ = await getattr(litellm, call)( + video_id="https://x/source.mp4", + prompt="make it brighter", + seconds="5", + model=f"volcengine/{SEEDANCE_2_5}", + custom_llm_provider="volcengine", + api_key="sk-test", + api_base=API_BASE, + ) + + assert raised.value.status_code == 400 + + +@pytest.mark.parametrize( + "api_base", + [ + None, + API_BASE, + f"{API_BASE}/", + f"{API_BASE}/api/v3", + f"{API_BASE}/api/compatible", + f"{API_BASE}/api/v3/responses", + f"{API_BASE}/v1", + ], +) +def test_any_ark_surface_on_the_shared_credential_normalizes_to_the_host_root( + api_base: str | None, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("VOLCENGINE_API_BASE", raising=False) + monkeypatch.delenv("ARK_API_BASE", raising=False) + + assert VolcEngineVideoConfig().get_complete_url("m", api_base, {}) == API_BASE + + +@pytest.mark.parametrize("env_var", ["VOLCENGINE_API_BASE", "ARK_API_BASE"]) +def test_an_api_base_env_var_routes_a_deployment_without_api_base(env_var: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("VOLCENGINE_API_BASE", raising=False) + monkeypatch.delenv("ARK_API_BASE", raising=False) + monkeypatch.setenv(env_var, "https://ark.ap-southeast.bytepluses.com/api/v3") + + assert VolcEngineVideoConfig().get_complete_url("m", None, {}) == "https://ark.ap-southeast.bytepluses.com" + assert VolcEngineVideoConfig().get_complete_url("m", f"{API_BASE}/api/v3", {}) == API_BASE + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ("https://gw.internal/ark/api/v3", "https://gw.internal/ark"), + ("https://gw.internal/api/ark/api/v3", "https://gw.internal/api/ark"), + ], +) +def test_a_path_prefix_before_the_api_marker_survives(api_base: str, expected: str): + assert VolcEngineVideoConfig().get_complete_url("m", api_base, {}) == expected + + +def test_a_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("ARK_API_KEY", raising=False) + monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False) + + with pytest.raises(ValueError, match="Volcengine API key is required"): + _ = VolcEngineVideoConfig().validate_environment({}, SEEDANCE_2_5) + + +def test_volcengine_resolves_to_the_video_config(): + config = ProviderConfigManager.get_provider_video_config( + model=SEEDANCE_2_5, provider=litellm.LlmProviders.VOLCENGINE + ) + + assert type(config) is VolcEngineVideoConfig + + +ARK_VIDEO_TIERS: Final = { + SEEDANCE_2_5: {"480p": ((854, 480), 70), "720p": ((1280, 720), 70), "1080p": ((1920, 1080), 77)}, + SEEDANCE_2_0: {"480p": ((864, 496), 46), "720p": ((1280, 720), 46), "1080p": ((1920, 1080), 51), "4k": ((3840, 2160), 26)}, + SEEDANCE_2_0_FAST: {"480p": ((864, 496), 37), "720p": ((1280, 720), 37)}, + "doubao-seedance-2-0-mini-260615": {"480p": ((864, 496), 23), "720p": ((1280, 720), 23)}, +} +CNY_PER_USD: Final = 7.0 + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", sorted(ARK_VIDEO_TIERS)) +def test_each_tier_rate_is_its_rendered_pixels_at_arks_price_for_that_tier(model: str): + """Ark prices 1080p and 4k per token above and below the 480p/720p rate, so a tier is not the base + rate scaled by pixels. Prices and frames are Ark's published per-resolution table, read 2026-10-16 + from https://www.volcengine.com/docs/82379/1544106, whose own 2.0 examples agree: 1080p 16:9 5s bills + 2.48 CNY/s and 4k bills 5.05 CNY/s.""" + info = litellm.get_model_info(model=model, custom_llm_provider="volcengine") + frames = info["provider_specific_entry"]["volcengine_video_frames"] + + assert {tier: info[f"output_cost_per_second_{tier}"] for tier in ARK_VIDEO_TIERS[model]} == { + tier: pytest.approx(width * height * 24 / 1024 * cny_per_million / 1e6 / CNY_PER_USD, rel=1e-5) + for tier, ((width, height), cny_per_million) in ARK_VIDEO_TIERS[model].items() + } + + assert { + tier: tuple(int(side) for side in frames[tier]["16:9"].split("x")) + for tier in ARK_VIDEO_TIERS[model] + if tier in frames + } == {tier: frame for tier, (frame, _) in ARK_VIDEO_TIERS[model].items() if tier in frames} + + +async def _wire_body(**kwargs: object) -> dict[str, object]: + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json={"id": TASK_ID}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + _ = await avideo_generation(api_key="sk-test", api_base=API_BASE, client=handler, **kwargs) + + (request,) = sent + assert (str(request.url), request.headers["authorization"]) == (TASKS_URL, "Bearer sk-test") + return json.loads(request.content) + + +@pytest.mark.asyncio +async def test_seconds_size_and_extra_body_content_reach_the_wire(): + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="use these", + seconds="4", + size="854x480", + extra_body={ + "content": [{"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}] + }, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [ + {"type": "text", "text": "use these"}, + {"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}, + ], + "ratio": "16:9", + "resolution": "480p", + "duration": 4, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fields", + [{"parameters": {"seed": 3}}, {"extra_body": {"parameters": {"seed": 3}}}], + ids=["top_level", "extra_body"], +) +async def test_parameters_reach_the_wire_unpacked_whether_sent_top_level_or_in_extra_body(fields: dict[str, object]): + body = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", seconds="4", **fields) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + "seed": 3, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra_body,expected", + [ + ({"size": "1920x1080"}, {"resolution": "1080p"}), + ({"seconds": "8"}, {"duration": 8}), + ({"user": "someone"}, {"safety_identifier": "someone"}), + ], + ids=["size", "seconds", "user"], +) +async def test_an_extra_body_field_litellm_maps_itself_is_honoured_and_its_leftover_copy_ignored( + extra_body: dict[str, object], expected: dict[str, object] +): + """The shared request builder copies extra_body onto the top-level params, where this adapter maps it, and then + merges the raw extra_body back over the mapped params, returning the same key a second time as a field of its + own. The second copy has to be ignored rather than rejected as a field litellm never sends.""" + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="p", + seconds="4", + size="1280x720", + extra_body=extra_body, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "ratio": "16:9", + "resolution": "720p", + "duration": 4, + **expected, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fields,misplaced", + [ + ({"parameters": {"seconds": "8"}}, "seconds"), + ({"extra_body": {"parameters": {"seconds": "8"}}}, "seconds"), + ({"extra_body": {"parameters": {"size": "1920x1080"}}}, "size"), + ({"extra_body": {"parameters": {"user": "someone"}}}, "user"), + ({"extra_body": {"parameters": {"prompt": "a dog"}}}, "prompt"), + ({"extra_body": {"parameters": {"parameters": {"seed": 3}}}}, "parameters"), + ], + ids=["top_level_seconds", "seconds", "size", "user", "prompt", "nested_parameters"], +) +async def test_a_request_field_litellm_maps_itself_is_rejected_inside_parameters( + fields: dict[str, object], misplaced: str +): + with pytest.raises(UnsupportedParamsError, match=f"send {misplaced} as top-level request fields"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", **fields) + + +@pytest.mark.asyncio +async def test_a_request_field_inside_parameters_is_dropped_under_drop_params_and_the_rest_is_sent(): + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="p", + drop_params=True, + extra_body={"parameters": {"seconds": "8", "seed": 3}}, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + "seed": 3, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra_body,field", + [ + ({"prompt": "a dog"}, "prompt"), + ({"model": SEEDANCE_2_0_FAST}, "model"), + ({"extra_headers": {"x-a": "1"}}, "extra_headers"), + ], + ids=["prompt", "model", "extra_headers"], +) +async def test_an_extra_body_copy_of_a_request_field_litellm_does_not_map_is_rejected( + extra_body: dict[str, object], field: str +): + with pytest.raises(UnsupportedParamsError, match=f"does not send {field} to Ark"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", extra_body=extra_body) + + +@pytest.mark.asyncio +async def test_a_user_that_is_not_a_string_is_rejected_rather_than_ignored(): + with pytest.raises(UnsupportedParamsError, match="user must be a string, got int"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", extra_body={"user": 5}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "control", + [{"drop_params": False}, {"organization": "org-1"}], + ids=["drop_params", "organization"], +) +async def test_a_litellm_control_field_is_not_mistaken_for_an_ark_field(control: dict[str, object]): + body = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", seconds="4", **control) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + } + + +@pytest.mark.asyncio +async def test_additional_drop_params_removes_the_named_ark_fields_from_the_wire(): + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="p", + seconds="4", + additional_drop_params=["seed"], + extra_body={"seed": 3, "watermark": False}, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + "watermark": False, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("dropped", "request_fields"), + [ + pytest.param("user", {"user": "u-1"}, id="user"), + pytest.param("seconds", {"seconds": "8"}, id="seconds"), + pytest.param("size", {"size": "1920x1080"}, id="size"), + pytest.param("parameters", {"parameters": {"seed": 3}}, id="parameters"), + pytest.param("parameters", {"extra_body": {"parameters": {"seed": 3}}}, id="extra_body_parameters"), + pytest.param("image", {"image": {"gcsUri": "gs://b/a.png"}}, id="unsupported_image"), + ], +) +async def test_additional_drop_params_naming_a_request_field_sends_what_omitting_it_sends( + dropped: str, request_fields: dict[str, object] +): + request: Final = {"model": f"volcengine/{SEEDANCE_2_5}", "prompt": "p"} + + with_drop = await _wire_body(**request, **request_fields, additional_drop_params=[dropped]) + + assert with_drop == await _wire_body(**request) + + +@pytest.mark.asyncio +async def test_an_extra_body_duration_of_minus_one_is_rejected_before_anything_is_sent(): + with pytest.raises(UnsupportedParamsError, match="Ark-picked length"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", extra_body={"duration": -1}) + + +ACCOUNT_A_HOST: Final = "https://ark-a.example" +ACCOUNT_B_HOST: Final = "https://ark-b.example" + + +def _two_account_router(**router_kwargs: object) -> Router: + return Router( + model_list=[ + { + "model_name": "seedance-a", + "litellm_params": { + "model": f"volcengine/{SEEDANCE_2_5}", + "api_key": "key-a", + "api_base": ACCOUNT_A_HOST, + }, + }, + { + "model_name": "seedance-b", + "litellm_params": { + "model": f"volcengine/{SEEDANCE_2_5}", + "api_key": "key-b", + "api_base": ACCOUNT_B_HOST, + }, + }, + ], + **router_kwargs, # pyright: ignore[reportArgumentType] # forwarded Router options + ) + + +def _proxy_poll_model(router: Router, video_id: str) -> str | None: + decoded: Final = decode_video_id_with_provider(video_id) + requested: Final = router.resolve_model_name_from_model_id( + decoded.get("model_id"), decoded.get("custom_llm_provider") + ) + served: Final = decoded.get("served_model") + return router.resolve_served_model_name(requested, served) if requested and served else requested + + +@pytest.mark.parametrize("caller_metadata", [None, {"tags": ["proxy-request"]}]) +@respx.mock +def test_a_video_is_polled_on_the_ark_account_that_created_it(caller_metadata: dict[str, object] | None): + router = _two_account_router() + _ = respx.post(f"{ACCOUNT_B_HOST}/api/v3/contents/generations/tasks").mock( + return_value=httpx.Response(200, json={"id": TASK_ID}) + ) + task_get = respx.get(f"{ACCOUNT_B_HOST}/api/v3/contents/generations/tasks/{TASK_ID}").mock( + return_value=httpx.Response(200, json=_task(status="running")) + ) + + created = router.video_generation(model="seedance-b", prompt="a cat", seconds="4", metadata=caller_metadata) + _ = router.video_status(model=_proxy_poll_model(router, created.id), video_id=created.id) + + assert [call.request.headers["authorization"] for call in task_get.calls] == ["Bearer key-b"] + + +@respx.mock +def test_a_video_created_through_a_routing_group_is_polled_on_the_member_that_created_it(): + router = _two_account_router( + routing_groups=[ + {"group_name": "seedance", "models": ["seedance-a", "seedance-b"], "routing_strategy": "simple-shuffle"} + ] + ) + for host in (ACCOUNT_A_HOST, ACCOUNT_B_HOST): + _ = respx.post(f"{host}/api/v3/contents/generations/tasks").mock( + return_value=httpx.Response(200, json={"id": TASK_ID}) + ) + _ = respx.get(f"{host}/api/v3/contents/generations/tasks/{TASK_ID}").mock( + return_value=httpx.Response(200, json=_task(status="running")) + ) + + for _ in range(8): + created = router.video_generation(model="seedance", prompt="a cat", seconds="4") + polled = router.video_status(model=_proxy_poll_model(router, created.id), video_id=created.id) + assert polled.id == created.id + + hosts = [call.request.url.host for call in respx.calls] + assert hosts[1::2] == hosts[0::2] diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 7291f49af99..de2676bafaa 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -14,8 +14,8 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils -from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL -from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -243,6 +243,82 @@ def test_batch_session_outranks_the_per_request_trace_id(): assert session_id == "batch-uid-1" +def _chat_session_id( + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, object], + correlation_in_logs: bool = True, + batch_trace_session_id: str | None = None, + metadata: Mapping[str, object] = MappingProxyType({}), +) -> str | None: + monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs) + return _get_session_id_for_spend_log( + kwargs={"litellm_trace_id": "trace-abc", "litellm_params": litellm_params}, + metadata=metadata, + standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING, + omit_when_missing=False, + batch_trace_session_id=batch_trace_session_id, + ) + + +_CALLER_SESSION_PARAMS: Final = MappingProxyType({"litellm_session_id": "sess-1", "metadata": {"trace_id": "otel-1"}}) + + +def test_caller_session_wins_over_the_per_request_trace_id(monkeypatch: pytest.MonkeyPatch): + assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS) == "sess-1" + + +def test_batch_session_outranks_a_caller_session(monkeypatch: pytest.MonkeyPatch): + assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, batch_trace_session_id="batch-uid-1") == "batch-uid-1" + + +def test_caller_session_keeps_the_legacy_trace_id_with_correlation_logging_off(monkeypatch: pytest.MonkeyPatch): + assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, correlation_in_logs=False) == "trace-abc" + + +@pytest.mark.parametrize(("correlation_in_logs", "expected_session_id"), [(True, "convo-1"), (False, "trace-abc")]) +def test_session_displaced_by_a_trace_header_wins_only_with_correlation_logging( + monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, expected_session_id: str +): + header_chain: Final = {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc", "session_id": "trace-abc"}} + session_id: Final = _chat_session_id( + monkeypatch, + header_chain, + correlation_in_logs=correlation_in_logs, + metadata={CALLER_SESSION_ID_METADATA_KEY: "convo-1"}, + ) + assert session_id == expected_session_id + + +@pytest.mark.parametrize( + ("litellm_params", "metadata"), + [ + pytest.param({"litellm_session_id": "convo\x00x"}, {}, id="body-session"), + pytest.param({"litellm_session_id": "trace-abc"}, {CALLER_SESSION_ID_METADATA_KEY: "convo\x00x"}, id="beside-trace-header"), + ], +) +def test_caller_session_postgres_cannot_store_keeps_the_trace_id( + monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object], metadata: Mapping[str, object] +): + assert _chat_session_id(monkeypatch, litellm_params, metadata=metadata) == "trace-abc" + + +@pytest.mark.parametrize( + "litellm_params", + [ + pytest.param({"metadata": {}}, id="no-session"), + pytest.param( + {"litellm_session_id": "minted-1", "metadata": {"session_id": "minted-1", SESSION_ID_GENERATED_METADATA_KEY: True}}, + id="minted-by-generate-policy", + ), + pytest.param({"litellm_session_id": "job-7", "metadata": {"trace_id": "job-7"}}, id="copied-from-trace-id"), + ], +) +def test_session_that_is_not_a_caller_conversation_keeps_the_trace_id( + monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object] +): + assert _chat_session_id(monkeypatch, litellm_params) == "trace-abc" + + def test_omit_policy_still_suppresses_batch_sessions(): session_id: Final = _get_session_id_for_spend_log( kwargs={}, diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 96c11be2650..93202d93566 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -17,6 +17,7 @@ from starlette.datastructures import Headers import litellm from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, SERVER_STREAMING_CLASSIFICATION_KEY, SERVER_STREAMING_CLASSIFICATION_MARKER, @@ -3308,6 +3309,93 @@ def test_add_litellm_metadata_from_request_headers_x_litellm_trace_id_sets_chain assert data["litellm_trace_id"] == "foo" +@pytest.mark.parametrize(("correlation_in_logs", "expected_spend_log_session"), [(True, "convo-1"), (False, None)]) +@pytest.mark.parametrize("session_in_metadata", [False, True], ids=["top-level", "metadata"]) +def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only( + monkeypatch: pytest.MonkeyPatch, + correlation_in_logs: bool, + expected_spend_log_session: str | None, + session_in_metadata: bool, +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs) + metadata: Final[dict[str, str]] = {"session_id": "convo-1"} if session_in_metadata else {} + data: Final[dict[str, object]] = ( + {"metadata": metadata} if session_in_metadata else {"metadata": metadata, "litellm_session_id": "convo-1"} + ) + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers={"x-litellm-trace-id": "req-trace-1"}, data=data, _metadata_variable_name="metadata" + ) + assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_spend_log_session + assert data["litellm_session_id"] == "req-trace-1" + assert metadata["session_id"] == "req-trace-1" + assert data["litellm_trace_id"] == "req-trace-1" + + +@pytest.mark.parametrize( + "session_header", ["x-litellm-session-id", "x-claude-code-session-id", "x-session-id"] +) +@pytest.mark.parametrize("with_trace_header", [False, True], ids=["session-header-only", "beside-trace-header"]) +def test_session_headers_still_beat_the_body_session_with_correlation_logging( + monkeypatch: pytest.MonkeyPatch, session_header: str, with_trace_header: bool +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + metadata: Final[dict[str, str]] = {} + data: Final[dict[str, object]] = {"metadata": metadata, "litellm_session_id": "convo-1"} + trace_header: Final = {"x-litellm-trace-id": "req-trace-1"} if with_trace_header else {} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers={**trace_header, session_header: "hdr-session-1"}, data=data, _metadata_variable_name="metadata" + ) + spend_log_session: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) or data["litellm_session_id"] + assert spend_log_session == "hdr-session-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("forged_in", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize( + "headers", + [{"x-litellm-session-id": "hdr-session-1"}, {}], + ids=["session-header", "no-headers"], +) +async def test_client_metadata_cannot_set_the_spend_log_caller_session( + monkeypatch: pytest.MonkeyPatch, forged_in: str, headers: dict[str, str] +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + updated: Final = await add_litellm_data_to_request( + data={ + "model": "gpt-5.4-mini", + "messages": [{"role": "user", "content": "hi"}], + forged_in: {CALLER_SESSION_ID_METADATA_KEY: "forged-session"}, + }, + request=_make_request_mock("/v1/chat/completions", headers), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"] + + +@pytest.mark.asyncio +async def test_spend_log_caller_session_stays_out_of_requester_metadata(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + updated: Final = await add_litellm_data_to_request( + data={ + "model": "gpt-5.4-mini", + "messages": [{"role": "user", "content": "hi"}], + "litellm_session_id": "convo-1", + "metadata": {"team_label": "kept"}, + }, + request=_make_request_mock("/v1/chat/completions", {"x-litellm-trace-id": "req-trace-1"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + assert updated["metadata"][CALLER_SESSION_ID_METADATA_KEY] == "convo-1" + assert updated["metadata"]["requester_metadata"]["team_label"] == "kept" + assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"]["requester_metadata"] + + def test_add_litellm_metadata_from_request_headers_x_litellm_session_id_sets_chain_id(): """x-litellm-session-id sets both metadata and top-level litellm_session_id/litellm_trace_id for call chaining.""" headers = {"x-litellm-session-id": "bar"} diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 19bbb6481fb..cc5d5c62739 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -17,6 +17,7 @@ from openai.types.responses.response_function_web_search import ( from pydantic import TypeAdapter import litellm +from litellm.constants import CALLER_SESSION_ID_METADATA_KEY from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -6499,3 +6500,69 @@ def test_transform_request_reads_service_tier_from_responses_api_request() -> No ) assert result["service_tier"] == "priority" + + +def _continued_chain_request(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + async def session_lookup(previous_response_id: str) -> ChatCompletionSession: + assert previous_response_id == "resp_turn_1" + return ChatCompletionSession( + messages=[{"role": "user", "content": "turn 1"}], litellm_session_id="chain-session" + ) + + return asyncio.run( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="resp_turn_1", + litellm_completion_request={ + "model": _CLAUDE_ON_BEDROCK, + "custom_llm_provider": "bedrock", + "messages": [{"role": "user", "content": "turn 2"}], + "metadata": request_kwargs.get("metadata"), + }, + instructions=None, + session_lookup=session_lookup, + request_kwargs=request_kwargs, + ) + ) + + +@pytest.mark.parametrize( + ("request_kwargs", "logged_metadata_key"), + [ + ( + { + "metadata": {"tag": "client"}, + "litellm_metadata": {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"}, + }, + "litellm_metadata", + ), + ({"metadata": {"user_api_key": "hashed"}}, "metadata"), + ], +) +def test_a_continued_chain_logs_its_spend_under_the_chain_session_with_request_correlation( + monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object], logged_metadata_key: str +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + sent_kwargs: Final = deepcopy(request_kwargs) + + request: Final = _continued_chain_request(request_kwargs) + + assert request["litellm_trace_id"] == "chain-session" + assert "litellm_session_id" not in request + assert request[logged_metadata_key] == { + **TypeAdapter(dict[str, object]).validate_python(request_kwargs[logged_metadata_key]), + CALLER_SESSION_ID_METADATA_KEY: "chain-session", + } + assert request_kwargs == sent_kwargs + + +def test_a_continued_chain_leaves_the_turns_metadata_alone_without_request_correlation( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", False) + caller_metadata: Final = {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"} + + request: Final = _continued_chain_request({"metadata": caller_metadata, "litellm_metadata": caller_metadata}) + + assert request["litellm_trace_id"] == "chain-session" + assert "litellm_metadata" not in request + assert request["metadata"] == {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"} diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 002a595a5e7..5db563be4b0 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1,4 +1,5 @@ import json +import sqlite3 from typing import Final from unittest.mock import AsyncMock, patch @@ -787,3 +788,80 @@ async def test_message_history_replays_real_key_named_tool_payloads() -> None: tool_message: Final = result["messages"][2] assert json.loads(assistant_message["tool_calls"][0]["function"]["arguments"]) == function_arguments assert json.loads(tool_message["content"]) == function_output + + +class _SqliteBackedPrismaDB: + def __init__(self, rows: list[tuple[str, str, str, str, str | None, str | None]]) -> None: + self._connection = sqlite3.connect(":memory:") + self._connection.row_factory = sqlite3.Row + self._connection.execute( + 'CREATE TABLE "LiteLLM_SpendLogs" ' + '(request_id TEXT, api_key TEXT, session_id TEXT, "endTime" TEXT, "user" TEXT, team_id TEXT)' + ) + self._connection.executemany('INSERT INTO "LiteLLM_SpendLogs" VALUES (?, ?, ?, ?, ?, ?)', rows) + + async def query_raw(self, query: str, *args: object) -> list[dict[str, object]]: + return [dict(row) for row in self._connection.execute(query.replace("$1", "?"), args)] + + +@pytest.mark.asyncio +async def test_session_lookup_does_not_return_another_keys_rows_for_a_shared_session_id(): + """A caller-supplied litellm_session_id can collide with another key's session; history must stay per key.""" + sqlite_db: Final = _SqliteBackedPrismaDB( + [ + ("victim-1", "victim-key", "shared-session", "2026-01-01T00:00:01", "", ""), + ("victim-2", "victim-key", "shared-session", "2026-01-01T00:00:02", "", ""), + ("attacker-1", "attacker-key", "shared-session", "2026-01-01T00:00:03", "", ""), + ("attacker-2", "attacker-key", "other-session", "2026-01-01T00:00:04", "", ""), + ] + ) + fake_prisma_client = _FakePrismaClient(results=[]) + fake_prisma_client.db = sqlite_db + + with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): + spend_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id("attacker-1") + victim_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id("victim-1") + + assert [row["request_id"] for row in spend_logs] == ["attacker-1"] + assert [row["request_id"] for row in victim_logs] == ["victim-1", "victim-2"] + + +async def _history_request_ids(sqlite_db: _SqliteBackedPrismaDB, response_id: str) -> list[str]: + fake_prisma_client: Final = _FakePrismaClient(results=[]) + fake_prisma_client.db = sqlite_db + with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): + spend_logs: Final = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(response_id) + return [row["request_id"] for row in spend_logs] + + +@pytest.mark.asyncio +async def test_session_lookup_keeps_jwt_users_and_teams_apart_when_they_share_an_empty_api_key(): + sqlite_db: Final = _SqliteBackedPrismaDB( + [ + ("alice-1", "", "shared-session", "2026-01-01T00:00:01", "alice", "team-a"), + ("bob-1", "", "shared-session", "2026-01-01T00:00:02", "bob", "team-a"), + ("service-a-1", "", "shared-session", "2026-01-01T00:00:03", "", "team-a"), + ("service-b-1", "", "shared-session", "2026-01-01T00:00:04", "", "team-b"), + ("alice-2", "", "shared-session", "2026-01-01T00:00:05", "alice", "team-a"), + ] + ) + + assert await _history_request_ids(sqlite_db, "alice-1") == ["alice-1", "alice-2"] + assert await _history_request_ids(sqlite_db, "bob-1") == ["bob-1"] + assert await _history_request_ids(sqlite_db, "service-a-1") == ["service-a-1"] + assert await _history_request_ids(sqlite_db, "service-b-1") == ["service-b-1"] + + +@pytest.mark.asyncio +async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_keys_apart(): + sqlite_db: Final = _SqliteBackedPrismaDB( + [ + ("alice-old-key", "hashed-old", "shared-session", "2026-01-01T00:00:01", "alice", "team-a"), + ("ownerless-1", "hashed-other", "shared-session", "2026-01-01T00:00:02", "", ""), + ("alice-new-key", "hashed-new", "shared-session", "2026-01-01T00:00:03", "alice", "team-a"), + ("ownerless-2", "hashed-another", "shared-session", "2026-01-01T00:00:04", "", ""), + ] + ) + + assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"] + assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"] diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py new file mode 100644 index 00000000000..8fa1c799863 --- /dev/null +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -0,0 +1,288 @@ +import pytest + +import litellm +from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ContinuationOutput, + ContinuationPrefillDeploymentCheck, + continuation_output_ceilings, + deployment_supports_prefill, +) + +PREFILL_MODEL = "anthropic/prefill-capable-test-model" +NON_PREFILL_MODEL = "openai/prefill-unknown-test-model" +UNMAPPED_MODEL = "openai/unmapped-test-model" + + +@pytest.fixture(autouse=True) +def _synthetic_cost_map_entries() -> None: + litellm.register_model( + { + PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}, + NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"}, + }, + persist_across_reloads=False, + ) + litellm.get_model_info.cache_clear() + + +def _deployment(model: str, dep_id: str) -> dict[str, object]: + return {"litellm_params": {"model": model}, "model_info": {"id": dep_id}} + + +def test_deployment_supports_prefill_reads_capability(): + assert deployment_supports_prefill(_deployment(PREFILL_MODEL, "a")) is True + assert deployment_supports_prefill(_deployment(NON_PREFILL_MODEL, "b")) is False + assert deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}}) is True + assert deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}, "model_info": "bogus"}) is True + + +def test_deployment_model_info_override_wins_over_cost_map(): + # model_info True opts in a model that is not in the cost map + assert ( + deployment_supports_prefill( + {"litellm_params": {"model": "vendor/custom-model"}, "model_info": {"supports_assistant_prefill": True}} + ) + is True + ) + # model_info False opts out a model the cost map would otherwise allow + assert ( + deployment_supports_prefill( + {"litellm_params": {"model": PREFILL_MODEL}, "model_info": {"supports_assistant_prefill": False}} + ) + is False + ) + # model_info without the key falls through to the cost map + assert deployment_supports_prefill(_deployment(PREFILL_MODEL, "z")) is True + + +@pytest.mark.parametrize( + "deployment,expected", + [ + ({"litellm_params": {"model": "bedrock/arn:custom-import", "base_model": PREFILL_MODEL}}, True), + ({"litellm_params": {"model": "azure/my-deployment"}, "model_info": {"base_model": PREFILL_MODEL}}, True), + ( + { + "litellm_params": {"model": "azure/my-deployment", "base_model": NON_PREFILL_MODEL}, + "model_info": {"base_model": PREFILL_MODEL}, + }, + True, + ), + ( + { + "litellm_params": {"model": PREFILL_MODEL, "base_model": NON_PREFILL_MODEL}, + "model_info": {"base_model": NON_PREFILL_MODEL}, + }, + False, + ), + ], + ids=[ + "litellm_params_base_model", + "model_info_base_model", + "model_info_base_model_wins", + "base_model_wins_over_model", + ], +) +def test_deployment_supports_prefill_reads_the_cost_map_key_of_the_base_model(deployment, expected): + assert deployment_supports_prefill(deployment) is expected + + +def test_deployment_supports_prefill_rejects_malformed_deployments(): + assert deployment_supports_prefill({}) is False + assert deployment_supports_prefill({"litellm_params": {}}) is False + assert deployment_supports_prefill("not-a-dict") is False + + +@pytest.mark.asyncio +async def test_filter_is_noop_without_continuation_marker(): + """A normal (non-continuation) request must be passed through untouched, even + if some deployments cannot prefill.""" + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")] + + for request_kwargs in ({}, None, {MID_STREAM_CONTINUATION_KWARG: False}): + result = await check.async_filter_deployments( + model="group", healthy_deployments=deployments, messages=None, request_kwargs=request_kwargs + ) + assert result == deployments + + +@pytest.mark.asyncio +async def test_filter_ignores_forged_client_flag(): + """A client cannot steer routing: a plain truthy value under the marker key + (which the proxy could forward from the request body) is not the internal + sentinel, so the filter leaves the deployment list untouched.""" + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")] + + for forged in (True, "true", 1, {"any": "json"}): + result = await check.async_filter_deployments( + model="group", + healthy_deployments=deployments, + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: forged}, + ) + assert result == deployments + + +@pytest.mark.asyncio +async def test_filter_keeps_only_prefill_capable_on_continuation(): + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")] + + result = await check.async_filter_deployments( + model="group", + healthy_deployments=deployments, + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + assert [d["model_info"]["id"] for d in result] == ["a"] + + +@pytest.mark.asyncio +async def test_filter_empties_group_when_no_prefill_capable_deployment(): + """No prefill-capable deployment -> empty result, so the router advances the + fallback chain and ultimately surfaces the original error.""" + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(NON_PREFILL_MODEL, "b"), _deployment(UNMAPPED_MODEL, "c")] + + result = await check.async_filter_deployments( + model="group", + healthy_deployments=deployments, + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + assert result == [] + + +@pytest.mark.asyncio +async def test_filter_drops_deployments_whose_own_params_constrain_the_continuation(): + check = ContinuationPrefillDeploymentCheck() + plain = _deployment(PREFILL_MODEL, "plain") + constrained = [ + {"litellm_params": {"model": PREFILL_MODEL, **params}, "model_info": {"id": dep_id}} + for dep_id, params in ( + ("json", {"response_format": {"type": "json_object"}}), + ("two_choices", {"extra_body": {"n": 2}}), + ("forced_tool", {"tool_choice": {"type": "function", "function": {"name": "f"}}}), + ("merged_reasoning", {"merge_reasoning_content_in_choices": True}), + ) + ] + + result = await check.async_filter_deployments( + model="group", + healthy_deployments=[plain, *constrained], + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + assert [d["model_info"]["id"] for d in result] == ["plain"] + + +def _text_chunk(text: str) -> litellm.ModelResponseStream: + return litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}]) + + +@pytest.mark.parametrize( + "prefill_content,fallback_texts,expected_texts", + [ + ("Hello", ("", "Hel", "lo wor", "ld"), ("", "", " wor", "ld")), + ("Hello", ("Hello",), ("",)), + ("Hello", (" world",), (" world",)), + ("Hello", ("Hel", "p me"), ("Hel", "p me")), + ("Hello", ("Hel",), ("Hel",)), + ([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")), + ], + ids=[ + "echo_split_across_chunks_is_dropped", + "exact_echo_then_end_is_dropped", + "continuation_without_echo_passes_through", + "near_echo_is_kept_in_order", + "stream_ending_mid_match_is_kept", + "content_block_prefill_echo_is_dropped", + ], +) +@pytest.mark.asyncio +async def test_continuation_output_drops_only_a_full_echo_of_the_prefill( + prefill_content, fallback_texts, expected_texts +): + output = ContinuationOutput( + "Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", () + ) + + async def fallback_stream(): + for text in fallback_texts: + yield _text_chunk(text) + + released = [chunk async for chunk in output.splice(fallback_stream())] + + assert tuple(chunk.choices[0].delta.content for chunk in released) == expected_texts + + +@pytest.mark.parametrize( + "prefill,fallback_texts,expected_texts", + [ + ("Transformers use ", ("self-attention",), ("self-attention",)), + ("Hello ", ("Hello ", " world"), ("", "world")), + ("Hello \n", ("\nworld",), ("\nworld",)), + ("A.\n\n", ("\n Hi", "\nthere"), (" Hi", "\nthere")), + ("\n\n", ("\nHi",), ("\nHi",)), + ("\nHello\n", ("\n",), ("",)), + ], + ids=[ + "continuation_without_whitespace_passes_through", + "echo_then_re_emitted_space_are_both_dropped", + "different_whitespace_is_kept", + "seam_ends_at_the_first_divergence", + "whitespace_only_prefill_has_no_seam", + "re_emitted_whitespace_at_stream_end_is_dropped", + ], +) +@pytest.mark.asyncio +async def test_continuation_output_drops_only_whitespace_the_client_already_has( + prefill, fallback_texts, expected_texts +): + output = ContinuationOutput(prefill, {"role": "assistant", "content": prefill, "prefix": True}, "group", ()) + + async def fallback_stream(): + for text in fallback_texts: + yield _text_chunk(text) + + released = [chunk async for chunk in output.splice(fallback_stream())] + + assert tuple(chunk.choices[0].delta.content for chunk in released) == expected_texts + + +@pytest.mark.parametrize( + "request_kwargs,expected", + [ + ({"max_tokens": 100}, {"max_tokens": 70}), + ({"max_tokens": 100, "max_completion_tokens": 40}, {"max_tokens": 70, "max_completion_tokens": 10}), + ({}, {}), + ( + {"max_tokens": 100, "extra_body": {"max_tokens": 50, "top_k": 3}}, + {"max_tokens": 70, "extra_body": {"max_tokens": 20, "top_k": 3}}, + ), + ({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}), + ({"extra_body": {"top_k": 3}}, {}), + ], + ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"], +) +def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends( + request_kwargs, expected +): + assert continuation_output_ceilings(request_kwargs, 30) == expected + + +@pytest.mark.parametrize( + "request_kwargs", + [ + {"max_tokens": 30}, + {"max_tokens": 20}, + {"max_tokens": 100, "max_completion_tokens": 30}, + {"max_tokens": 100, "extra_body": {"max_tokens": 30}}, + ], + ids=["max_tokens_met", "max_tokens_exceeded", "one_of_two_met", "extra_body_met"], +) +def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs): + assert continuation_output_ceilings(request_kwargs, 30) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3bd051d71d4..6bc52ab3dfc 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1,4 +1,5 @@ import ast, asyncio, importlib, time +import contextlib import copy import functools import gc @@ -12,7 +13,7 @@ import warnings from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping from datetime import datetime, timedelta, timezone from pathlib import Path -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import TYPE_CHECKING, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -2934,6 +2935,1216 @@ async def test_acompletion_streaming_iterator(): print("\n=== All tests passed! ===") +def _make_midstream_source(error, chunks=None, first_chunk=None): + from unittest.mock import MagicMock + + opening_chunk = ( + first_chunk if first_chunk is not None else MagicMock(choices=[MagicMock(delta=MagicMock(content="Hello"))]) + ) + + class _Source: + def __init__(self): + self.index = 0 + self.chunks = chunks if chunks is not None else [] + self.model = "gpt-4" + self.custom_llm_provider = "openai" + self.logging_obj = MagicMock() + + def __aiter__(self): + return self + + async def __anext__(self): + if self.index == 0: + self.index += 1 + return opening_chunk + raise error + + async def wait_for_failure_logging(self, timeout): + return None + + return _Source() + + +class _FakeFallbackStream: + def __init__(self, item): + self._item = item + self._done = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self._done: + raise StopAsyncIteration + self._done = True + return self._item + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_continues_after_content_when_eligible(): + """Flag on + plain-text break: the router re-enters the fallback chain with an + assistant-prefill continuation and the mid-stream marker, then streams the + fallback's output instead of re-raising.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ) + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + error = MidStreamFallbackError( + message="Connection lost", + model="gpt-4", + llm_provider="openai", + generated_content="Hello", + is_pre_first_chunk=False, + emitted_disqualifying_content=False, + ) + source = _make_midstream_source(error) + fallback_chunk = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": "world"}}]) + initial_kwargs = {"model": "gpt-4", "stream": True} + + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=source, messages=[{"role": "user", "content": "Hi"}], initial_kwargs=initial_kwargs + ) + collected = [chunk async for chunk in result] + + mock_fallback.assert_awaited_once() + passed_kwargs = mock_fallback.await_args.kwargs["kwargs"] + assert passed_kwargs[MID_STREAM_CONTINUATION_KWARG] is MID_STREAM_CONTINUATION_MARKER + assert passed_kwargs["messages"][-1] == {"role": "assistant", "content": "Hello", "prefix": True} + assert fallback_chunk in collected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "continuation_text,joined_output_trips_rule", + [("6789", True), ("6788", False)], + ids=["rule_trips_only_on_the_joined_text", "clean_continuation_streams"], +) +async def test_acompletion_streaming_iterator_continuation_runs_post_call_rules_on_the_joined_text( + monkeypatch, continuation_text, joined_output_trips_rule +): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + monkeypatch.setattr(litellm, "post_call_rules", [lambda output: "123-45-6789" not in output]) + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="123-45-", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_chunk = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": continuation_text}}]) + + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + if joined_output_trips_rule: + with pytest.raises(litellm.APIResponseValidationError): + async for _ in result: + pass + else: + assert fallback_chunk in [chunk async for chunk in result] + + +@pytest.mark.asyncio +async def test_a_second_cut_runs_post_call_rules_only_on_the_text_the_caller_received(monkeypatch): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ) + + checked_texts = [] + monkeypatch.setattr(litellm, "post_call_rules", [lambda output: checked_texts.append(output) is None]) + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + def cut(generated_content): + return MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content=generated_content, + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + + messages = [{"role": "user", "content": "Hi"}] + middle_leg = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source( + cut(" there"), + first_chunk=litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " there"}}]), + ), + messages=[*messages, {"role": "assistant", "content": "Hello", "prefix": True}], + initial_kwargs={"model": "gpt-4", "stream": True, MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + backup_leg = _FakeFallbackStream( + litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " friend"}}]) + ) + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(side_effect=[middle_leg, backup_leg]) + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(cut("Hello")), + messages=messages, + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + _ = [chunk async for chunk in result] + + assert checked_texts == ["Hello there", "Hello there friend"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallback_texts", + [("Hel", "lo", " world"), (" world",)], + ids=["fallback_echoes_the_prefill", "fallback_continues_after_the_prefill"], +) +async def test_acompletion_streaming_iterator_continuation_never_repeats_the_partial_output(fallback_texts): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + + async def fallback_stream(): + for text in fallback_texts: + yield litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}]) + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(return_value=fallback_stream()) + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + + continuation = [chunk for chunk in collected if isinstance(chunk, litellm.ModelResponseStream)] + assert "".join(chunk.choices[0].delta.content for chunk in continuation) == " world" + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_continuation_stays_in_the_primary_stream(): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + primary_chunk = litellm.ModelResponseStream( + id="chatcmpl-primary", created=111, choices=[{"index": 0, "delta": {"role": "assistant", "content": "Hello"}}] + ) + + async def fallback_stream(): + yield litellm.ModelResponseStream( + id="msg-fallback", created=222, choices=[{"index": 0, "delta": {"role": "assistant", "content": " wor"}}] + ) + yield litellm.ModelResponseStream( + id="msg-fallback", created=222, choices=[{"index": 0, "delta": {"content": "ld"}}] + ) + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(return_value=fallback_stream()) + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error, chunks=[primary_chunk]), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + + continuation = [chunk for chunk in collected if isinstance(chunk, litellm.ModelResponseStream)] + assert [(chunk.id, chunk.created, chunk.choices[0].delta.role) for chunk in continuation] == [ + ("chatcmpl-primary", 111, None), + ("chatcmpl-primary", 111, None), + ] + assert "".join(chunk.choices[0].delta.content for chunk in continuation) == " world" + + +def _continuation_router(): + return litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + +def _usage_chunk(prompt_tokens, completion_tokens): + from litellm.types.utils import Usage + + return litellm.ModelResponseStream( + model="gpt-4", + choices=[], + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +async def _drain_with_fallback(router, error, primary_chunks, messages, fallback_items): + from unittest.mock import AsyncMock, patch + + async def fallback_stream(): + for item in fallback_items: + yield item + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(return_value=fallback_stream()) + ) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error, chunks=primary_chunks), + messages=messages, + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + streamed = [chunk for chunk in collected if isinstance(chunk, litellm.ModelResponseStream)] + return mock_fallback.await_args.kwargs["kwargs"], streamed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("primary_reports_usage", [True, False], ids=["usage_reported", "usage_not_reported"]) +async def test_acompletion_streaming_iterator_continuation_usage_sums_both_legs(primary_reports_usage): + from litellm.exceptions import MidStreamFallbackError + + messages = [{"role": "user", "content": "Explain transformers in one paragraph."}] + partial = "Transformers use " + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content=partial, + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + content_chunk = litellm.ModelResponseStream( + model="gpt-4", choices=[{"index": 0, "delta": {"role": "assistant", "content": partial}}] + ) + primary_chunks = [content_chunk, _usage_chunk(21, 3)] if primary_reports_usage else [content_chunk] + primary_usage = litellm.stream_chunk_builder(chunks=list(primary_chunks), messages=messages).usage + + _, streamed = await _drain_with_fallback( + _continuation_router(), error, primary_chunks, messages, [_usage_chunk(91, 5)] + ) + + assert primary_usage.prompt_tokens > 0 + assert streamed[-1].usage.prompt_tokens == primary_usage.prompt_tokens + 91 + assert streamed[-1].usage.completion_tokens == primary_usage.completion_tokens + 5 + + +class _CutSSE(httpx.AsyncByteStream): + def __init__(self, content: str) -> None: + self._content: Final = content + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield f'data: {json.dumps({"id": "cut", "object": "chat.completion.chunk", "created": 1700000000, "model": "m", "choices": [{"index": 0, "delta": {"role": "assistant", "content": self._content}, "finish_reason": None}]})}\n\n'.encode() + raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") + + +_BACKUP_SSE: Final = "".join( + f"data: {json.dumps(chunk)}\n\n" + for chunk in ( + {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]}, + {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}}, + ) +) + "data: [DONE]\n\n" + + +class _SpendRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.successes: Final[list[dict[str, object]]] = [] + self.failures: Final[list[dict[str, object]]] = [] + self.spend_rows: Final[list[dict[str, object]]] = [] + self.succeeded: Final = asyncio.Event() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + self.successes.append(kwargs["standard_logging_object"]) + self.spend_rows.append(get_logging_payload(kwargs, response_obj, start_time, end_time)) + self.succeeded.set() + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.failures.append(kwargs["standard_logging_object"]) + + +class _SlowFailureRecorder(_SpendRecorder): + def __init__(self) -> None: + super().__init__() + self.failed_deployment_ids: Final[list[str]] = [] + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + await asyncio.sleep(0.05) + self.failed_deployment_ids.append(kwargs["litellm_params"]["metadata"]["model_info"]["id"]) + await super().async_log_failure_event(kwargs, response_obj, start_time, end_time) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cut_groups", [("primary",), ("primary", "middle")], ids=["one_break", "two_breaks"]) +async def test_continued_stream_success_log_bills_every_cut_leg( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, cut_groups: tuple[str, ...] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _SpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_CutSSE("Transformers use")) + ) + respx_mock.post(host="api.deepseek.com").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_CutSSE(" learned")) + ) + respx_mock.post(host="api.mistral.ai").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=_BACKUP_SSE) + ) + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "k1", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + }, + { + "model_name": "middle", + "litellm_params": { + "model": "deepseek/deepseek-chat", + "api_key": "k2", + "input_cost_per_token": 0.003, + "output_cost_per_token": 0.004, + }, + }, + {"model_name": "backup", "litellm_params": {"model": "mistral/mistral-large-latest", "api_key": "k3"}}, + ], + fallbacks=[{"primary": ["middle" if "middle" in cut_groups else "backup"]}, {"middle": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + response: Final = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "How do transformers work?"}], + stream=True, + stream_options={"include_usage": True}, + ) + chunks: Final = [chunk async for chunk in response] + await asyncio.wait_for(recorder.succeeded.wait(), timeout=10) + await asyncio.sleep(0.5) + + backup_prompt_cost, backup_completion_cost = litellm.cost_per_token( + model="mistral/mistral-large-latest", prompt_tokens=91, completion_tokens=5 + ) + success: Final = recorder.successes[-1] + assert len(recorder.failures) == len(cut_groups) + assert [failure["response_cost"] for failure in recorder.failures] == pytest.approx( + [ + failure["prompt_tokens"] * input_price + failure["completion_tokens"] * output_price + for failure, (input_price, output_price) in zip(recorder.failures, ((0.001, 0.002), (0.003, 0.004))) + ] + ) + assert success["response_cost"] == pytest.approx( + sum(failure["response_cost"] for failure in recorder.failures) + backup_prompt_cost + backup_completion_cost + ) + assert success["prompt_tokens"] == sum(failure["prompt_tokens"] for failure in recorder.failures) + 91 + assert success["completion_tokens"] == sum(failure["completion_tokens"] for failure in recorder.failures) + 5 + assert success["cost_breakdown"]["total_cost"] == pytest.approx(success["response_cost"]) + spend_row: Final = recorder.spend_rows[-1] + assert (spend_row["spend"], spend_row["prompt_tokens"], spend_row["completion_tokens"]) == ( + success["response_cost"], + success["prompt_tokens"], + success["completion_tokens"], + ) + client_usage: Final = next(chunk.usage for chunk in reversed(chunks) if getattr(chunk, "usage", None)) + assert (client_usage.prompt_tokens, client_usage.completion_tokens) == ( + success["prompt_tokens"], + success["completion_tokens"], + ) + assert {chunk.id for chunk in chunks} == {spend_row["request_id"]} + + +@pytest.mark.asyncio +async def test_continued_stream_sharing_the_proxy_logging_object_bills_the_cut_to_its_own_deployment( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _SlowFailureRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_CutSSE("Transformers use")) + ) + respx_mock.post(host="api.mistral.ai").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=_BACKUP_SSE) + ) + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "k1", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + "model_info": {"id": "primary-deployment"}, + }, + {"model_name": "backup", "litellm_params": {"model": "mistral/mistral-large-latest", "api_key": "k3"}}, + ], + fallbacks=[{"primary": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + messages: Final = [{"role": "user", "content": "How do transformers work?"}] + shared_logging, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + model="primary", + messages=messages, + stream=True, + litellm_call_id="shared-call", + ) + + response: Final = await router.acompletion( + model="primary", + messages=messages, + stream=True, + stream_options={"include_usage": True}, + litellm_call_id="shared-call", + litellm_logging_obj=shared_logging, + ) + _ = [chunk async for chunk in response] + await asyncio.wait_for(recorder.succeeded.wait(), timeout=10) + await asyncio.sleep(0.5) + + backup_prompt_cost, backup_completion_cost = litellm.cost_per_token( + model="mistral/mistral-large-latest", prompt_tokens=91, completion_tokens=5 + ) + (failure,) = recorder.failures + assert (failure["model_group"], failure["model_id"]) == ("primary", "primary-deployment") + assert recorder.failed_deployment_ids == ["primary-deployment"] + assert failure["response_cost"] == pytest.approx( + failure["prompt_tokens"] * 0.001 + failure["completion_tokens"] * 0.002 + ) + assert recorder.successes[-1]["response_cost"] == pytest.approx( + failure["response_cost"] + backup_prompt_cost + backup_completion_cost + ) + + +def test_carry_partial_spend_to_fallback_adds_the_partial_the_cut_stream_inherited(): + from litellm.types.utils import Usage + + cut_logging: Final = LiteLLMLogging( + model="deepseek/deepseek-chat", messages=[], stream=True, call_type="acompletion", + start_time=datetime.now(), litellm_call_id="cut", function_id="cut", + ) + cut_logging.model_call_details["response_cost"] = 0.5 + cut_logging.update_environment_variables( + litellm_params={"custom_llm_provider": "deepseek"}, optional_params={}, model="deepseek/deepseek-chat" + ) + cut_logging.record_continued_partial( + usage=Usage(prompt_tokens=40, completion_tokens=7, total_tokens=47), + response_cost=0.001, + response_id="chatcmpl-primary", + ) + fallback_logging: Final = LiteLLMLogging( + model="mistral-large-latest", messages=[], stream=True, call_type="acompletion", + start_time=datetime.now(), litellm_call_id="fallback", function_id="fallback", + ) + + Router._carry_partial_spend_to_fallback( + SimpleNamespace(logging_obj=fallback_logging), + Router._price_cut_partial( + cast(CustomStreamWrapper, SimpleNamespace(logging_obj=cut_logging, model="deepseek/deepseek-chat")), + litellm.ModelResponse( + id="chatcmpl-relay", usage=Usage(prompt_tokens=60, completion_tokens=3, total_tokens=63) + ), + ), + ) + + relay_prompt_cost, relay_completion_cost = litellm.cost_per_token( + model="deepseek/deepseek-chat", prompt_tokens=60, completion_tokens=3 + ) + carried_usage: Final = fallback_logging.model_call_details["continued_partial_usage"] + assert (carried_usage.prompt_tokens, carried_usage.completion_tokens) == (100, 10) + assert fallback_logging.model_call_details["continued_partial_cost"] == pytest.approx( + relay_prompt_cost + relay_completion_cost + 0.001 + ) + assert fallback_logging.model_call_details["continued_response_id"] == "chatcmpl-primary" + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_plain_fallback_keeps_the_fallback_usage(): + from litellm.exceptions import MidStreamFallbackError + + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="anthropic", generated_content="", + is_pre_first_chunk=True, + ) + role_chunk = litellm.ModelResponseStream( + model="gpt-4", choices=[{"index": 0, "delta": {"role": "assistant", "content": ""}}] + ) + + _, streamed = await _drain_with_fallback( + _continuation_router(), + error, + [role_chunk, _usage_chunk(21, 0)], + [{"role": "user", "content": "Hi"}], + [_usage_chunk(91, 5)], + ) + + assert (streamed[-1].usage.prompt_tokens, streamed[-1].usage.completion_tokens) == (91, 5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "partial,fallback_texts,expected_answer", + [ + ("Transformers use ", (" self", "-attention"), "Transformers use self-attention"), + ("def f():\n ", ("\n", " return 1"), "def f():\n return 1"), + ("First.\n", ("\n\nSecond",), "First.\n\nSecond"), + ], + ids=["space", "indentation", "paragraph_break"], +) +async def test_acompletion_streaming_iterator_continuation_does_not_double_trailing_whitespace( + partial, fallback_texts, expected_answer +): + from litellm.exceptions import MidStreamFallbackError + + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content=partial, + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_items = [ + litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}]) for text in fallback_texts + ] + + passed_kwargs, streamed = await _drain_with_fallback( + _continuation_router(), error, [], [{"role": "user", "content": "Hi"}], fallback_items + ) + + assert passed_kwargs["messages"][-1]["content"] == partial + assert partial + "".join(chunk.choices[0].delta.content for chunk in streamed) == expected_answer + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", [True, "true", " True "]) +async def test_update_settings_turns_on_mid_stream_continuation(monkeypatch, enabled): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ContinuationPrefillDeploymentCheck, + ) + + monkeypatch.setattr(litellm, "callbacks", []) + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + ) + router.update_settings(enable_mid_stream_fallback_continuation=enabled) + + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_chunk = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " world"}}]) + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + + assert mock_fallback.await_args.kwargs["kwargs"][MID_STREAM_CONTINUATION_KWARG] is MID_STREAM_CONTINUATION_MARKER + assert fallback_chunk in collected + assert any(isinstance(callback, ContinuationPrefillDeploymentCheck) for callback in litellm.callbacks) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error_kwargs,request_kwargs", + [ + ({"emitted_disqualifying_content": True}, {}), + ({"emitted_disqualifying_content": False}, {"response_format": {"type": "json_object"}}), + ({"emitted_disqualifying_content": False}, {"tool_choice": "required"}), + ({"emitted_disqualifying_content": False}, {"merge_reasoning_content_in_choices": True}), + ({"emitted_disqualifying_content": False}, {"n": 2}), + ], + ids=["tool_or_thinking_emitted", "json_mode", "forced_tool_choice", "merged_reasoning", "multiple_choices"], +) +async def test_acompletion_streaming_iterator_declines_ineligible_after_content(error_kwargs, request_kwargs): + """Flag on but the break is not continuation-safe: the router re-raises and + never enters the fallback chain, so no duplicated/rejected request is sent.""" + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, **error_kwargs, + ) + source = _make_midstream_source(error) + initial_kwargs = {"model": "gpt-4", "stream": True, **request_kwargs} + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=source, messages=[{"role": "user", "content": "Hi"}], initial_kwargs=initial_kwargs + ) + with pytest.raises(MidStreamFallbackError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_raises_content_filter_instead_of_continuing(): + """A content filter that stops a stream mid-answer reaches the client as ContentPolicyViolationError, + never as a continuation that lets a backup finish the filtered answer.""" + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "azure/gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + content_filter = litellm.ContentPolicyViolationError(message="filtered", model="gpt-4", llm_provider="azure") + error = MidStreamFallbackError( + message="filtered", model="gpt-4", llm_provider="azure", generated_content="Step one is to obtain the", + is_pre_first_chunk=False, emitted_disqualifying_content=False, original_exception=content_filter, + ) + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + with pytest.raises(litellm.ContentPolicyViolationError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +async def _continues_after_content( + initial_kwargs: dict[str, object], + primary_params: Mapping[str, object] = MappingProxyType({}), + default_litellm_params: dict[str, object] | None = None, + cut_model_id: str = "primary-deployment", +) -> bool: + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "k1", **primary_params}, + "model_info": {"id": "primary-deployment"}, + }, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + { + "model_name": "openai-backup", + "litellm_params": {"model": "gpt-4o", "api_key": "k3"}, + "model_info": {"id": "no-prefill-deployment", "supports_assistant_prefill": False}, + }, + { + "model_name": "claude-backup", + "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k4"}, + "model_info": {"id": "prefill-deployment", "supports_assistant_prefill": True}, + }, + { + "model_name": "json-claude-backup", + "litellm_params": { + "model": "anthropic/claude-3-opus-20240229", + "api_key": "k5", + "response_format": {"type": "json_object"}, + }, + "model_info": {"id": "json-prefill-deployment", "supports_assistant_prefill": True}, + }, + ], + fallbacks=[{"gpt-4": ["backup"]}], + default_litellm_params=default_litellm_params, + enable_mid_stream_fallback_continuation=True, + ) + error: Final = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_chunk: Final = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " world"}}]) + cut_stream: Final = _make_midstream_source(error) + cut_stream._hidden_params = {"model_id": cut_model_id} + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ) as mock_fallback: + result: Final = await router._acompletion_streaming_iterator( + model_response=cut_stream, + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True, **initial_kwargs}, + ) + with contextlib.suppress(MidStreamFallbackError): + _ = [chunk async for chunk in result] + return mock_fallback.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "event_hook,continues", + [("post_call", False), (None, False), ("pre_call", True)], + ids=["post_call_guardrail", "every_hook_guardrail", "input_only_guardrail"], +) +async def test_mid_stream_continuation_declines_while_an_output_guardrail_is_registered( + monkeypatch: pytest.MonkeyPatch, event_hook: str | None, continues: bool +) -> None: + from litellm.types.guardrails import GuardrailEventHooks + + guardrail: Final = CustomGuardrail( + guardrail_name="output-check", event_hook=GuardrailEventHooks(event_hook) if event_hook else None + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + assert await _continues_after_content({}) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallbacks,continues", + [ + ([{"model": "backup", "messages": [{"role": "user", "content": "Say hi"}]}], False), + ([{"model": "backup", "stream": False}], False), + ([{"model": "backup", "max_tokens": 300}], False), + ([{"model": "backup"}], True), + ([{"gpt-4": [{"model": "backup", "stream": False}]}], False), + ([{"*": [{"model": "backup", "max_tokens": 300}]}], False), + ([{"gpt-4": [{"model": "backup"}, "backup"]}], True), + ([{"gpt-4": ["backup"]}, {"other-group": [{"model": "backup", "stream": False}]}], True), + ([{}], True), + ([{"gpt-4": 5}], True), + ], + ids=[ + "entry_with_its_own_messages", + "entry_turning_off_streaming", + "entry_with_its_own_max_tokens", + "entry_with_model_only", + "group_target_turning_off_streaming", + "generic_group_target_with_its_own_max_tokens", + "group_targets_with_model_only", + "override_only_in_another_groups_chain", + "empty_entry", + "non_list_chain", + ], +) +async def test_mid_stream_continuation_declines_a_request_fallback_that_overrides_request_params( + fallbacks: list[object], continues: bool +) -> None: + assert await _continues_after_content({"fallbacks": fallbacks}) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallbacks,continues", + [ + ([{"gpt-4": ["no-prefill-deployment"]}], False), + ([{"gpt-4": ["backup", "no-prefill-deployment"]}], False), + ([{"model": "no-prefill-deployment"}], False), + ([{"gpt-4": ["prefill-deployment"]}], True), + ([{"gpt-4": ["openai-backup"]}], True), + ([{"gpt-4": ["json-prefill-deployment"]}], False), + ], + ids=[ + "deployment_id_without_prefill", + "deployment_id_without_prefill_later_in_chain", + "flat_entry_naming_a_deployment_id_without_prefill", + "deployment_id_with_prefill", + "group_without_prefill_left_to_the_deployment_filter", + "deployment_id_with_its_own_response_format", + ], +) +async def test_mid_stream_continuation_declines_a_fallback_naming_a_deployment_that_cannot_continue( + fallbacks: list[object], continues: bool +) -> None: + assert await _continues_after_content({"fallbacks": fallbacks}) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "primary_params,default_litellm_params,continues", + [ + ({"response_format": {"type": "json_object"}}, None, False), + ({"n": 2}, None, False), + ({"tool_choice": "required"}, None, False), + ({"merge_reasoning_content_in_choices": True}, None, False), + ({}, {"response_format": {"type": "json_object"}}, False), + ({"response_format": {"type": "text"}}, None, True), + ({"temperature": 0.2}, None, True), + ({"tool_choice": "required"}, {"tool_choice": "auto"}, False), + ({"response_format": {"type": "json_object"}}, {"response_format": {"type": "text"}}, True), + ({"extra_body": {"n": 2}}, None, False), + ], + ids=[ + "deployment_response_format", + "deployment_n", + "deployment_tool_choice", + "deployment_merged_reasoning", + "router_default_response_format", + "deployment_plain_text_format", + "deployment_unrelated_param", + "deployment_tool_choice_over_router_default", + "router_default_format_over_deployment", + "deployment_extra_body_n", + ], +) +async def test_mid_stream_continuation_reads_constraints_configured_on_the_cut_deployment( + primary_params: Mapping[str, object], default_litellm_params: dict[str, object] | None, continues: bool +) -> None: + assert await _continues_after_content({}, primary_params, default_litellm_params) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra_body,continues", + [({"response_format": {"type": "json_object"}}, False), ({"n": 2}, False), ({"top_k": 5}, True)], + ids=["json_format_in_extra_body", "n_in_extra_body", "unrelated_extra_body_key"], +) +async def test_mid_stream_continuation_reads_constraints_the_request_sends_in_extra_body( + extra_body: dict[str, object], continues: bool +) -> None: + assert await _continues_after_content({"extra_body": extra_body}) is continues + + +@pytest.mark.asyncio +async def test_mid_stream_continuation_declines_when_the_cut_deployment_is_not_in_the_router() -> None: + assert await _continues_after_content({}, cut_model_id="per-request-credential-deployment") is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "routing_strategy", + [ + "simple-shuffle", + "usage-based-routing", + "usage-based-routing-v2", + "least-busy", + "latency-based-routing", + "cost-based-routing", + ], +) +async def test_continuation_routes_only_to_deployments_that_can_continue_under_every_routing_strategy( + routing_strategy: str, +) -> None: + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ) + from litellm.types.router import RouterRateLimitError + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "backup", + "litellm_params": {"model": "gpt-4o", "api_key": "k1"}, + "model_info": {"id": "no-prefill-deployment", "supports_assistant_prefill": False}, + }, + ], + routing_strategy=routing_strategy, + enable_mid_stream_fallback_continuation=True, + ) + messages: Final = [{"role": "user", "content": "Hi"}] + plain_pick: Final = await router.async_get_available_deployment( + model="backup", messages=messages, request_kwargs={} + ) + assert plain_pick["model_info"]["id"] == "no-prefill-deployment" + with pytest.raises(RouterRateLimitError): + await router.async_get_available_deployment( + model="backup", + messages=messages, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "flag_kwargs", [{}, {"enable_mid_stream_fallback_continuation": "false"}], ids=["default", "string_false"] +) +async def test_acompletion_streaming_iterator_flag_off_declines_after_content(flag_kwargs): + """Regression guard for the opt-in default: with the flag off, a post-content + break re-raises and never falls back, exactly as before this feature.""" + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + **flag_kwargs, + ) + error = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + source = _make_midstream_source(error) + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=source, messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + with pytest.raises(MidStreamFallbackError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +def test_build_completion_continuation_input_appends_assistant_prefill(): + messages = [{"role": "user", "content": "hi"}] + built = litellm.Router._build_completion_continuation_input(messages, "partial answer") + assert built[:-1] == messages + assert built[-1] == {"role": "assistant", "content": "partial answer", "prefix": True} + + +@pytest.mark.parametrize( + "prefill_content,second_partial,expected_content", + [ + ("part one ", " part two ", "part one part two "), + ("part one ", "part two ", "part one part two "), + ("First.\n\n", "Second", "First.\n\nSecond"), + ([{"type": "text", "text": "part one "}], " part two ", [{"type": "text", "text": "part one part two "}]), + ], + ids=["separator_sent_again", "separator_not_sent_again", "paragraph_break_not_sent_again", "content_blocks"], +) +def test_build_completion_continuation_input_folds_into_existing_prefill( + prefill_content, second_partial, expected_content +): + """A nested break must not leave two trailing assistant turns: the new partial folds into the prior prefill, + which keeps reading as the text the caller already received.""" + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": prefill_content, "prefix": True}] + built = litellm.Router._build_completion_continuation_input(messages, second_partial) + assert built == [messages[0], {"role": "assistant", "content": expected_content, "prefix": True}] + + +@pytest.mark.parametrize( + "prefill_content,expected_content", + [ + ("Hello", "Hello world"), + ([{"type": "text", "text": "Hello"}], [{"type": "text", "text": "Hello world"}]), + ], + ids=["string_prefill", "content_block_prefill"], +) +def test_build_completion_continuation_input_does_not_double_an_echoed_prefill(prefill_content, expected_content): + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": prefill_content, "prefix": True}] + built = litellm.Router._build_completion_continuation_input(messages, "Hello world") + assert built is not None + assert built[-1] == {"role": "assistant", "content": expected_content, "prefix": True} + + +@pytest.mark.parametrize( + "cut_partial,expected_content", + [("Hel", None), ("Hello", "Hello"), ("Hello wor", "Hello wor")], + ids=["cut_mid_echo", "cut_after_a_full_echo", "cut_after_new_text"], +) +def test_build_completion_continuation_input_declines_a_cut_in_the_middle_of_a_prefill_echo( + cut_partial, expected_content +): + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "Hello", "prefix": True}] + built = litellm.Router._build_completion_continuation_input(messages, cut_partial) + expected = None if expected_content is None else [messages[0], {**messages[1], "content": expected_content}] + assert built == expected + + +@pytest.mark.parametrize( + "assistant_content", ["Earlier answer.", [{"type": "text", "text": "Earlier answer."}]], ids=["string", "blocks"] +) +def test_build_completion_continuation_input_declines_a_trailing_plain_assistant_turn(assistant_content): + """The caller's own assistant turn is history, not a prefill, so the partial is never fused into it.""" + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": assistant_content}] + assert litellm.Router._build_completion_continuation_input(messages, "The capital of France is") is None + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_declines_when_request_ends_in_plain_assistant_turn(): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Earlier answer."}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + with pytest.raises(MidStreamFallbackError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +def test_build_completion_continuation_input_keeps_structured_assistant_content(): + """Content blocks on a trailing assistant turn stay blocks: the partial continues the last text + block in place (a block boundary would let a provider drop the space between "Sure, " and "here"), + lands as a new text block only after a non-text block, and a content shape that cannot be extended declines.""" + from litellm.router import _extend_trailing_text_block + + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "Sure, "}], "prefix": True}, + ] + built = litellm.Router._build_completion_continuation_input(messages, "here it is") + assert built is not None + assert [m["role"] for m in built] == ["user", "assistant"] + assert built[-1] == {"role": "assistant", "content": [{"type": "text", "text": "Sure, here it is"}], "prefix": True} + image_block = {"type": "image_url", "image_url": {"url": "https://example.test/a.png"}} + assert _extend_trailing_text_block([image_block], "here it is") == [image_block, {"type": "text", "text": "here it is"}] + assert _extend_trailing_text_block([], "here it is") == [{"type": "text", "text": "here it is"}] + assert ( + litellm.Router._build_completion_continuation_input([{"role": "assistant", "content": 42, "prefix": True}], "x") + is None + ) + + +def test_mid_stream_continuation_eligible_allows_text_response_format(): + """response_format={"type": "text"} is the unconstrained default and must stay + eligible, unlike json_object / json_schema.""" + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[{"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k"}}], + enable_mid_stream_fallback_continuation=True, + ) + e = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + assert router._mid_stream_continuation_eligible(e=e, request_kwargs={"response_format": {"type": "text"}}) is True + assert router._mid_stream_continuation_eligible(e=e, request_kwargs={"response_format": {"type": "json_object"}}) is False + + +def test_raise_original_mid_stream_error_surfaces_the_provider_exception(): + from litellm.exceptions import MidStreamFallbackError, RateLimitError + + provider_error = RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4") + wrapped = MidStreamFallbackError( + message="rate limited", model="gpt-4", llm_provider="openai", + original_exception=provider_error, generated_content="Hello", + ) + with pytest.raises(RateLimitError) as raised: + litellm.Router._raise_original_mid_stream_error(wrapped) + assert raised.value is provider_error + assert raised.value.__cause__ is wrapped + + bare = MidStreamFallbackError(message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello") + with pytest.raises(MidStreamFallbackError) as bare_raised: + litellm.Router._raise_original_mid_stream_error(bare) + assert bare_raised.value is bare + + @pytest.mark.asyncio async def test_acompletion_streaming_iterator_reraises_original_exception_when_available(): """Async: when the mid-stream MidStreamFallbackError wraps a real provider @@ -8694,6 +9905,11 @@ def test_combine_fallback_usage(): assert chunk.usage.completion_tokens == 5 assert chunk.usage.total_tokens == 15 + Router._combine_fallback_usage(chunk, Usage(prompt_tokens=0, completion_tokens=4, total_tokens=4)) + assert chunk.usage.prompt_tokens == 10 + assert chunk.usage.completion_tokens == 9 + assert chunk.usage.total_tokens == 19 + @pytest.mark.asyncio async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_failure(): diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index d343f464a77..d8625ff9da3 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -544,13 +544,14 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): ) -def test_regex_lookaround_flag_stays_on_the_deployment_that_set_it() -> None: - """A deployment's ``supports_regex_lookaround`` override must not land on the shared +@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"]) +def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: + """A deployment's capability override must not land on the shared ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it.""" backend_model = "bedrock/us.xai.grok-4.6" deploy_id = "grok-deploy-keep-regex" - builtin_flag = litellm.get_model_info(model=backend_model).get("supports_regex_lookaround") + builtin_flag = litellm.get_model_info(model=backend_model).get(flag) model_keys = { deploy_id: litellm.model_cost.get(deploy_id), backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), @@ -561,13 +562,13 @@ def test_regex_lookaround_flag_stays_on_the_deployment_that_set_it() -> None: { "model_name": "grok-keep-regex", "litellm_params": {"model": backend_model}, - "model_info": {"id": deploy_id, "supports_regex_lookaround": not builtin_flag}, + "model_info": {"id": deploy_id, flag: not builtin_flag}, } ], ) - assert litellm.model_cost[deploy_id]["supports_regex_lookaround"] is (not builtin_flag) - assert litellm.get_model_info(model=backend_model).get("supports_regex_lookaround") is builtin_flag + assert litellm.model_cost[deploy_id][flag] is (not builtin_flag) + assert litellm.get_model_info(model=backend_model).get(flag) is builtin_flag finally: _restore_model_cost_entries(model_keys)