Merge commit 'f81e657903' into litellm_fix_object_permission_partial_upsert

litellm/__init__.py

@@ -207,7 +207,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]]
     []
 )  # internal variable - async custom callbacks are routed here.
 pre_call_rules: List[Callable] = []
-post_call_rules: List[Callable] = []
+post_call_rules: List[Callable[..., object]] = []
 turn_off_message_logging: Optional[bool] = False
 standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it

litellm/constants.py

@@ -57,6 +57,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
         "model_group_retry_policy",
         "model_group_alias",
         "enable_weighted_failover",
+        "enable_mid_stream_fallback_continuation",
         "enable_tag_filtering",
         "tag_routing_prefix",
         "optional_pre_call_checks",
@@ -146,6 +147,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
 # Streams exceeding this duration are terminated with a Timeout error.
 # None (default) = no limit.  Set env var to a number of seconds to enable globally.
 LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS")
+MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
 
 # Maximum number of base64 characters to keep in logging payloads.
 # Data URIs exceeding this are replaced with a size placeholder.
@@ -1629,6 +1631,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
 SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
+CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
     "Truncation is a DB storage safeguard. "

litellm/exceptions.py

@@ -1217,6 +1217,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
         num_retries: int | None = None,
         generated_content: str = "",
         is_pre_first_chunk: bool = False,
+        emitted_disqualifying_content: bool = False,
     ):
         original_status: Final = getattr(original_exception, "status_code", None)
         self.status_code = int(original_status) if original_status is not None else 503
@@ -1229,6 +1230,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
         self.num_retries = num_retries
         self.generated_content = generated_content
         self.is_pre_first_chunk = is_pre_first_chunk
+        self.emitted_disqualifying_content = emitted_disqualifying_content
 
         # Create a response if one wasn't provided
         if response is None:

litellm/litellm_core_utils/litellm_logging.py

@@ -2295,6 +2295,36 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
 
+    def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
+        """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
+        self.model_call_details["continued_partial_usage"] = usage
+        self.model_call_details["continued_partial_cost"] = response_cost
+        self.model_call_details["continued_response_id"] = response_id
+
+    def _add_continued_partial_spend(self, response: object) -> None:
+        partial_usage: Final = self.model_call_details.get("continued_partial_usage")
+        partial_cost: Final = self.model_call_details.get("continued_partial_cost")
+        own_usage: Final = getattr(response, "usage", None)
+        own_cost: Final = self.model_call_details.get("response_cost")
+        if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
+            return
+        from litellm.cost_calculator import BaseTokenUsageProcessor
+
+        self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
+            usage_objects=[own_usage, partial_usage]
+        )
+        if not isinstance(own_cost, (int, float)):
+            return
+        self.model_call_details["response_cost"] = own_cost + partial_cost
+        if self.cost_breakdown is not None:
+            self.cost_breakdown = self.cost_breakdown | CostBreakdown(
+                total_cost=own_cost + partial_cost,
+                additional_costs={
+                    **self.cost_breakdown.get("additional_costs", {}),
+                    "mid_stream_partial_cost": partial_cost,
+                },
+            )
+
     def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
         """Bill a fully streamed response on the failure log when a post-call hook rejects it."""
         usage: Final = getattr(assembled, "usage", None)
@@ -2590,6 +2620,15 @@ class Logging(LiteLLMLoggingBaseClass):
             return response
         return response.model_copy(update={"model": model})
 
+    def _with_continued_response_id(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        response_id: Final = self.model_call_details.get("continued_response_id")
+        if not isinstance(response_id, str) or getattr(response, "id", None) == response_id:
+            return response
+        return response.model_copy(update={"id": response_id})
+
     def _success_handler_helper_fn(
         self,
         result: object = None,
@@ -2908,8 +2947,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["response_cost"] = self.response_cost_calculator(
                     result=complete_streaming_response
                 )
+                self._add_continued_partial_spend(complete_streaming_response)
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
-                logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
+                logged_streaming_response: Final = self._with_continued_response_id(
+                    self._with_client_facing_stream_model(complete_streaming_response)
+                )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
                 ## STANDARDIZED LOGGING PAYLOAD
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@@ -3445,12 +3487,15 @@ class Logging(LiteLLMLoggingBaseClass):
                     self.model,
                 )
                 self.model_call_details["response_cost"] = None
+            self._add_continued_partial_spend(complete_streaming_response)
 
             self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
 
             await self._prepare_baseline_cache_estimate(complete_streaming_response)
 
-            logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
+            logged_streaming_response: Final = self._with_continued_response_id(
+                self._with_client_facing_stream_model(complete_streaming_response)
+            )
             self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
 
             ## STANDARDIZED LOGGING PAYLOAD

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1894,6 +1894,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa
     return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS)
 
 
+def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
+    """Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys."""
+    return _message_without_fields(message, frozenset({"prefix"}))
+
+
 def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
     """
     Filters a value from a dictionary

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -4303,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag
     return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]])
 
 
+def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock:
+    return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block
+
+
+def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock:
+    """Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it."""
+    if message["role"] != "assistant":
+        return message
+    blocks: Final = message["content"]
+    last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None)
+    return BedrockMessageBlock(
+        role="assistant",
+        content=[
+            _rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks)
+        ],
+    )
+
+
 def _ensure_document_messages_have_text(
     contents: list[BedrockMessageBlock],
 ) -> list[BedrockMessageBlock]:
@@ -4894,7 +4912,12 @@ class BedrockConverseMessagesProcessor:
                     llm_provider=llm_provider,
                 )
 
-        return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
+        converse_messages: Final = _ensure_document_messages_have_text(
+            _rename_duplicate_bedrock_document_names(contents)
+        )
+        if not (to_claude and converse_messages):
+            return converse_messages
+        return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
 
     @staticmethod
     def translate_thinking_blocks_to_reasoning_content_blocks(
@@ -5303,7 +5326,10 @@ def _bedrock_converse_messages_pt(
                 llm_provider=llm_provider,
             )
 
-    return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
+    converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
+    if not (to_claude and converse_messages):
+        return converse_messages
+    return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
 
 
 def make_valid_bedrock_tool_name(input_tool_name: str) -> str:

litellm/litellm_core_utils/streaming_handler.py

@@ -242,6 +242,7 @@ class CustomStreamWrapper:
         self.completion_stream = completion_stream
         self.sent_first_chunk = False
         self.sent_last_chunk = False
+        self._failure_logging: asyncio.Task[None] | None = None
         self._closed = False
         self._stream_created_time: float = time.time()
 
@@ -269,6 +270,7 @@ class CustomStreamWrapper:
         self.holding_chunk = ""
         self.complete_response = ""
         self.response_uptil_now = ""
+        self._emitted_disqualifying_content = False
         _model_info: Final[dict] = litellm_params.model_info or {}
 
         _api_base: Final = get_api_base(
@@ -1129,6 +1131,7 @@ class CustomStreamWrapper:
         if self.merge_reasoning_content_in_choices is True:
             reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
             if reasoning_content:
+                self._emitted_disqualifying_content = True
                 if self.sent_first_thinking_block is False:
                     # Ensure content is not None before concatenation
                     if model_response.choices[0].delta.content is None:
@@ -1865,9 +1868,7 @@ class CustomStreamWrapper:
                     if response.choices:
                         choice = response.choices[0]
                         if isinstance(choice, StreamingChoices):
-                            self.response_uptil_now += choice.delta.get("content", "") or ""
-                        else:
-                            self.response_uptil_now += ""
+                            self._accumulate_streamed_delta(choice.delta)
                     self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
                     # HANDLE STREAM OPTIONS
                     self.chunks.append(response)
@@ -2084,9 +2085,7 @@ class CustomStreamWrapper:
                     if processed_chunk.choices:
                         choice = processed_chunk.choices[0]
                         if isinstance(choice, StreamingChoices):
-                            self.response_uptil_now += choice.delta.get("content", "") or ""
-                        else:
-                            self.response_uptil_now += ""
+                            self._accumulate_streamed_delta(choice.delta)
                     self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
                     # Add mcp_list_tools to first chunk if present
                     if not self.sent_first_chunk and processed_chunk.choices:
@@ -2156,9 +2155,7 @@ class CustomStreamWrapper:
 
                         choice = processed_chunk.choices[0]
                         if isinstance(choice, StreamingChoices):
-                            self.response_uptil_now += choice.delta.get("content", "") or ""
-                        else:
-                            self.response_uptil_now += ""
+                            self._accumulate_streamed_delta(choice.delta)
                         self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
                         # RETURN RESULT
                         self.chunks.append(processed_chunk)
@@ -2172,7 +2169,7 @@ class CustomStreamWrapper:
             if self.logging_obj is not None:
                 self._record_partial_usage_for_failure()
                 ## LOGGING
-                asyncio.create_task(
+                self._failure_logging = asyncio.create_task(
                     self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
                 )
             await self.aclose()
@@ -2305,11 +2302,17 @@ class CustomStreamWrapper:
         if self.logging_obj is not None:
             self._record_partial_usage_for_failure()
             ## LOGGING
-            asyncio.create_task(
+            self._failure_logging = asyncio.create_task(
                 self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
             )
         self._handle_stream_fallback_error(e)
 
+    async def wait_for_failure_logging(self, timeout: float) -> None:
+        """Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
+        if self._failure_logging is None:
+            return
+        await asyncio.wait((self._failure_logging,), timeout=timeout)
+
     def _record_partial_usage_for_failure(self) -> None:
         """
         A stream that breaks mid-flight still billed the provider for the chunks
@@ -2346,6 +2349,35 @@ class CustomStreamWrapper:
                 recover_error,
             )
 
+    # Delta fields a text-only prefill continuation cannot carry, so a stream
+    # that emitted any of them is not eligible for mid-stream continuation.
+    _CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = (
+        "tool_calls",
+        "function_call",
+        "reasoning_content",
+        "thinking_blocks",
+        "reasoning_items",
+        "audio",
+        "images",
+        "annotations",
+    )
+
+    @classmethod
+    def _delta_disqualifies_continuation(cls, delta: object) -> bool:
+        get: Final = getattr(delta, "get", None)
+        if not callable(get):
+            return False
+        return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
+
+    def _accumulate_streamed_delta(self, delta: object) -> None:
+        # Shared by the sync, async, and non-aiohttp iteration sites so answer
+        # text and the disqualifying-content latch stay in step across all three.
+        get: Final = getattr(delta, "get", None)
+        content: Final = get("content") if callable(get) else None
+        self.response_uptil_now += content if isinstance(content, str) else ""
+        if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
+            self._emitted_disqualifying_content = True
+
     def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
         """
         Common error handling for both __next__ and __anext__.
@@ -2377,6 +2409,7 @@ class CustomStreamWrapper:
                 original_exception=e.original_exception,
                 generated_content=self.response_uptil_now,
                 is_pre_first_chunk=not self.sent_first_chunk,
+                emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content,
             )
 
         # Map to OpenAI exception format. Some providers' mappers (e.g.
@@ -2437,6 +2470,7 @@ class CustomStreamWrapper:
             original_exception=mapped_exception,
             generated_content=self.response_uptil_now,
             is_pre_first_chunk=not self.sent_first_chunk,
+            emitted_disqualifying_content=self._emitted_disqualifying_content,
         )
 
     @staticmethod

litellm/llms/base_llm/videos/async_task_utils.py

@@ -0,0 +1,123 @@
+import base64
+from collections.abc import Mapping
+from io import BufferedReader, BytesIO
+from itertools import accumulate, repeat
+from pathlib import Path, PurePath
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.images.utils import ImageEditRequestUtils
+from litellm.types.videos.utils import decode_video_id_with_provider
+
+_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
+_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+
+
+def _fields(value: object) -> Mapping[str, object] | None:
+    try:
+        return _FIELDS_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
+    return _fields(getattr(logging_obj, "litellm_params", None)) or {}
+
+
+def _id_safe_name(name: object) -> str | None:
+    return name if isinstance(name, str) and name and ";" not in name else None
+
+
+def _stamped_models(metadata: object) -> tuple[str, str | None] | None:
+    fields: Final = _fields(metadata)
+    if fields is None:
+        return None
+    requested: Final = _id_safe_name(fields.get("model_group"))
+    served: Final = _id_safe_name(fields.get("deployment_model_name"))
+    if requested is None:
+        return None
+    return requested, None if served is None or served == requested or "*" in served else served
+
+
+def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None:
+    """
+    A later status or content request checks the key's model access against the model encoded in the id, and
+    routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the
+    group that served the task can send its polls back to the deployment that created it, while a key may be
+    scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one
+    the router was asked for on this hop, and the group that served it when that differs and is not a wildcard.
+    The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
+    """
+    params: Final = _logged_litellm_params(logging_obj)
+    stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
+    return next((models for models in stamped if models), None)
+
+
+def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]:
+    """
+    The served group survives into the returned id only when the router sent this poll to it, so an edited id
+    cannot write any name it likes back.
+    """
+    video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
+    if not isinstance(video_id, str):
+        return None, None
+    decoded: Final = decode_video_id_with_provider(video_id)
+    served: Final = decoded.get("served_model")
+    routed: Final = routed_video_models(logging_obj)
+    return decoded.get("model_id") or None, served if routed is None or served in routed else None
+
+
+def _tuple_payload(file_obj: object) -> object:
+    if not isinstance(file_obj, tuple):
+        return file_obj
+    file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
+    return file_tuple[1] if len(file_tuple) >= 2 else None
+
+
+def _file_content(file_obj: object) -> object:
+    unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj)
+    return next(item for item in unwrapped if not isinstance(item, tuple))
+
+
+def input_reference_bytes(file_obj: object) -> bytes:
+    content: Final = _file_content(file_obj)
+    if isinstance(content, PurePath):
+        return Path(content).read_bytes()
+    if isinstance(content, (BytesIO, BufferedReader)):
+        current_position: Final = content.tell()
+        _ = content.seek(0)
+        data: Final = content.read()
+        _ = content.seek(current_position)
+        return data
+    if isinstance(content, bytes):
+        return content
+    if isinstance(content, bytearray):
+        return bytes(content)
+    read: Final = getattr(content, "read", None)
+    if callable(read):
+        read_data: Final[object] = read()
+        if isinstance(read_data, bytes):
+            return read_data
+    raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
+
+
+def _reference_object_url(reference: Mapping[str, object]) -> str:
+    image_url: Final = reference.get("image_url")
+    if isinstance(image_url, str) and image_url:
+        return image_url
+    raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported")
+
+
+def input_reference_url(image: object) -> str:
+    """A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form
+    carries that URL, or a data URL, in ``image_url``."""
+    if isinstance(image, str):
+        return image
+    reference: Final = _fields(image)
+    if reference is not None:
+        return _reference_object_url(reference)
+    content: Final = input_reference_bytes(image)
+    content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
+    encoded: Final = base64.b64encode(content).decode("utf-8")
+    return f"data:{content_type};base64,{encoded}"

litellm/llms/dashscope/videos/transformation.py

@@ -3,13 +3,10 @@ DashScope (Alibaba Model Studio) async video task API: create, poll and download
 delete or remix endpoint, so those raise.
 """
 
-import base64
 import json
 from collections.abc import Mapping
 from datetime import datetime
-from io import BufferedReader, BytesIO
 from math import log
-from pathlib import Path, PurePath
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Annotated, Final
 
@@ -19,9 +16,13 @@ from pydantic import BaseModel, Field, StrictBool, StrictInt, StrictStr, TypeAda
 
 import litellm
 from litellm.exceptions import UnsupportedParamsError
-from litellm.images.utils import ImageEditRequestUtils
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.videos.async_task_utils import (
+    input_reference_url,
+    polled_video_models,
+    routed_video_models,
+)
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
 from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,  # pyright: ignore[reportPrivateUsage, reportUnknownVariableType]  # house cached-client factory has no public alias and its stub leaves params untyped
@@ -32,7 +33,6 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import ModelInfo
 from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
 from litellm.types.videos.utils import (
-    decode_video_id_with_provider,
     encode_video_id_with_provider,
     extract_original_video_id,
 )
@@ -86,7 +86,6 @@ _EMPTY_PARAMS: Final[dict[str, object]] = {}  # mutable-ok: BaseVideoConfig cont
 
 _TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_TaskResponse)
 _PARAMETERS_ADAPTER: Final = TypeAdapter(dict[str, object])
-_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
 
 _STATUS_MAP: Final = MappingProxyType(
     {
@@ -383,39 +382,6 @@ def _is_priced_tier(resolution: object, priced_tiers: tuple[str, ...]) -> bool:
     return isinstance(resolution, str) and resolution.strip().upper() in priced_tiers
 
 
-def _read_all_bytes(file_obj: object) -> bytes:
-    if isinstance(file_obj, tuple):
-        file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
-        return _read_all_bytes(file_tuple[1] if len(file_tuple) >= 2 else None)
-    if isinstance(file_obj, PurePath):
-        return Path(file_obj).read_bytes()
-    if isinstance(file_obj, (BytesIO, BufferedReader)):
-        current_position: Final = file_obj.tell()
-        file_obj.seek(0)
-        content: Final = file_obj.read()
-        file_obj.seek(current_position)
-        return content
-    if isinstance(file_obj, bytes):
-        return file_obj
-    if isinstance(file_obj, bytearray):
-        return bytes(file_obj)
-    read: Final = getattr(file_obj, "read", None)
-    if callable(read):
-        data: Final = read()
-        if isinstance(data, bytes):
-            return data
-    raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
-
-
-def _image_url(image: object) -> str:
-    if isinstance(image, str):
-        return image
-    content: Final = _read_all_bytes(image)
-    content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
-    encoded: Final = base64.b64encode(content).decode("utf-8")
-    return f"data:{content_type};base64,{encoded}"
-
-
 def _media_reference_type(model: str) -> str | None:
     """Models without the field predate the media array and take the image on the flat ``img_url`` field."""
     info: Final = _dashscope_model_info(model)
@@ -558,58 +524,6 @@ def _video_url_from_task(task: _TaskResponse, raw_response: httpx.Response) -> s
     raise _video_error(raw_response, 502, "Video URL not found in task response. The task may not have succeeded yet.")
 
 
-def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
-    litellm_params: Final[object] = getattr(logging_obj, "litellm_params", None)
-    try:
-        return _PARAMETERS_ADAPTER.validate_python(litellm_params)
-    except ValidationError:
-        return _EMPTY_PARAMS
-
-
-def _polled_models(logging_obj: object) -> tuple[str | None, str | None]:
-    """
-    The proxy routes by the models encoded in the polled id, so they must survive into the returned id. The served
-    group survives only when the router sent this poll to it, so an edited id cannot write any name it likes back.
-    """
-    video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
-    if not isinstance(video_id, str):
-        return None, None
-    decoded: Final = decode_video_id_with_provider(video_id)
-    served: Final = decoded.get("served_model")
-    routed: Final = _poll_models(logging_obj)
-    return decoded.get("model_id") or None, served if routed is None or served in routed else None
-
-
-def _poll_models(logging_obj: object) -> tuple[str, str | None] | None:
-    """
-    A later status or content request checks the key's model access against the model encoded in the id, and
-    routes by it. DashScope serves the same model name from several regional hosts, so only the group that
-    served the task can send its polls back to the region that created it, while a key may be scoped to the
-    alias, routing group or wildcard name it asked for instead. The id carries both names: the one the router
-    was asked for on this hop, and the group that served it when that differs and is not a wildcard. The router
-    stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
-    """
-    params: Final = _logged_litellm_params(logging_obj)
-    stamped: Final = (_stamped_poll_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
-    return next((models for models in stamped if models), None)
-
-
-def _stamped_poll_models(metadata: object) -> tuple[str, str | None] | None:
-    try:
-        fields: Final = _PARAMETERS_ADAPTER.validate_python(metadata)
-    except ValidationError:
-        return None
-    requested: Final = _id_safe_name(fields.get("model_group"))
-    served: Final = _id_safe_name(fields.get("deployment_model_name"))
-    if requested is None:
-        return None
-    return requested, None if served is None or served == requested or "*" in served else served
-
-
-def _id_safe_name(name: object) -> str | None:
-    return name if isinstance(name, str) and name and ";" not in name else None
-
-
 class DashScopeVideoConfig(BaseVideoConfig):
     def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: BaseVideoConfig contract returns list
         return [
@@ -804,7 +718,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
         input_reference: Final = video_create_optional_request_params.get("input_reference")
         if input_reference is None:
             return None, None
-        image_url: Final = _image_url(input_reference)
+        image_url: Final = input_reference_url(input_reference)
         media_type: Final = _media_reference_type(model)
         if media_type is None:
             return "img_url", image_url
@@ -823,7 +737,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
         requested: Final = self._requested_parameters(request_data)
         video_obj: Final = _video_object_from_task(task, raw_response, model=model, requested=requested)
         if custom_llm_provider and video_obj.id:
-            requested_model, served_model = _poll_models(logging_obj) or (model, None)
+            requested_model, served_model = routed_video_models(logging_obj) or (model, None)
             video_obj.id = encode_video_id_with_provider(
                 video_obj.id, custom_llm_provider, requested_model, served_model
             )
@@ -869,7 +783,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
         client: "HTTPHandler | None" = None,
     ) -> VideoObject:
         task: Final = _parse_task_response(raw_response)
-        model, served_model = _polled_models(logging_obj)
+        model, served_model = polled_video_models(logging_obj)
         video_obj: Final = _video_object_from_task(task, raw_response, model=model)
         if custom_llm_provider and video_obj.id:
             video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model, served_model)

litellm/llms/databricks/chat/transformation.py

@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     merge_consecutive_system_messages,
     strip_litellm_internal_message_fields,
     strip_name_from_message,
+    strip_prefill_prefix_field,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.base_utils import (
@@ -487,6 +488,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         Databricks does not support:
         - 'name' in user message.
         - litellm's internal `thinking_blocks` / `reasoning_content` on assistant messages.
+        - the `prefix` prefill flag Mistral and DeepSeek read, which Claude does not need to continue an assistant turn.
         - non-Claude json_object "json" checks read only string content, so a single text block is sent as a string.
         """
         new_messages = []
@@ -496,7 +498,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             else:
                 _message = message
             _message = strip_name_from_message(_message, allowed_name_roles=["user"])
-            _message = strip_litellm_internal_message_fields(_message)
+            _message = strip_prefill_prefix_field(strip_litellm_internal_message_fields(_message))
             # Move message-level cache_control into a content block when content is a string.
             if "cache_control" in _message and isinstance(_message.get("content"), str):
                 _message = self._move_cache_control_into_string_content_block(_message)

litellm/llms/volcengine/videos/transformation.py

@@ -0,0 +1,904 @@
+import json
+from collections.abc import Mapping, Sequence
+from math import log
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, TypeVar
+
+import httpx
+from httpx._types import RequestFiles
+from pydantic import BaseModel, StrictBool, StrictInt, StrictStr, TypeAdapter, ValidationError
+
+import litellm
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.videos.async_task_utils import (
+    input_reference_url,
+    polled_video_models,
+    routed_video_models,
+)
+from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    _get_httpx_client,  # pyright: ignore[reportPrivateUsage, reportUnknownVariableType]  # house cached-client factory has no public alias and its stub leaves params untyped
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # factory stub leaves params untyped
+)
+from litellm.llms.volcengine.common_utils import get_volcengine_base_url
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import ModelInfo
+from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
+from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id
+from litellm.utils import get_model_info
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+
+_ParsedT: Final = TypeVar("_ParsedT")
+
+
+class VolcEngineVideoError(BaseLLMException):
+    pass
+
+
+class _TaskError(BaseModel, frozen=True):
+    code: str | None = None
+    message: str | None = None
+
+
+class _TaskContent(BaseModel, frozen=True):
+    video_url: str | None = None
+
+
+class _TaskUsage(BaseModel, frozen=True):
+    total_tokens: int | None = None
+
+
+class _Task(BaseModel, frozen=True):
+    id: str
+    status: str
+    model: str | None = None
+    error: _TaskError | None = None
+    content: _TaskContent | None = None
+    usage: _TaskUsage | None = None
+    created_at: int | None = None
+    updated_at: int | None = None
+    resolution: str | None = None
+    ratio: str | None = None
+    duration: int | float | None = None
+
+
+class _TaskCreateResponse(BaseModel, frozen=True):
+    id: str | None = None
+    error: _TaskError | None = None
+
+
+class _TaskList(BaseModel, frozen=True):
+    items: Sequence[_Task] = ()
+    total: int | None = None
+
+
+class _ContentItem(BaseModel, frozen=True):
+    type: str | None = None
+    role: str | None = None
+
+
+class _KnownFields(BaseModel, frozen=True):
+    resolution: StrictStr | None = None
+    ratio: StrictStr | None = None
+    seed: StrictInt | None = None
+    camera_fixed: StrictBool | None = None
+    watermark: StrictBool | None = None
+    generate_audio: StrictBool | None = None
+
+
+VOLCENGINE_VIDEO_TASKS_PATH: Final = "/api/v3/contents/generations/tasks"
+VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS: Final = 4
+VOLCENGINE_VIDEO_DEFAULT_RESOLUTION: Final = "720p"
+_VOLCENGINE_API_MARKER: Final = "/api/"
+_VOLCENGINE_COMPAT_SUFFIXES: Final = ("/api", "/v1")
+_FRAMES_FIELD: Final = "volcengine_video_frames"
+
+_EMPTY_PARAMS: Final[dict[str, object]] = {}  # mutable-ok: BaseVideoConfig contract empty params
+
+_TASK_ADAPTER: Final = TypeAdapter(_Task)
+_TASK_CREATE_ADAPTER: Final = TypeAdapter(_TaskCreateResponse)
+_TASK_LIST_ADAPTER: Final = TypeAdapter(_TaskList)
+_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
+_CONTENT_ITEMS_ADAPTER: Final = TypeAdapter(tuple[dict[str, object], ...])
+_FRAMES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]])
+_EMPTY_FRAMES: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType({})
+
+_STATUS_MAP: Final = MappingProxyType(
+    {
+        "queued": "queued",
+        "running": "in_progress",
+        "succeeded": "completed",
+        "failed": "failed",
+        "cancelled": "cancelled",
+        "expired": "failed",
+    }
+)
+_PENDING_STATUSES: Final = frozenset({"queued", "running"})
+_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "cancelled"})
+
+_CREATE_PASSTHROUGH_KEYS: Final = (
+    "resolution",
+    "ratio",
+    "duration",
+    "seed",
+    "camera_fixed",
+    "watermark",
+    "generate_audio",
+    "service_tier",
+    "priority",
+    "output_format",
+    "execution_expires_after",
+    "callback_url",
+    "safety_identifier",
+    "tools",
+    "omni_reference_task_type",
+)
+
+_REQUEST_METADATA: Final = frozenset({"model", "prompt", "user", "extra_headers", "extra_query", "extra_body"})
+_UNSUPPORTED_BY_VOLCENGINE: Final = frozenset({"characters", "image"})
+_MAPPED_ELSEWHERE: Final = frozenset({*_REQUEST_METADATA, *_UNSUPPORTED_BY_VOLCENGINE, "seconds", "size", "parameters"})
+_READ_FROM_THE_REQUEST: Final = _MAPPED_ELSEWHERE - _UNSUPPORTED_BY_VOLCENGINE
+_LITELLM_CONTROLS: Final = frozenset({"drop_params", "additional_drop_params", "organization"})
+_DROP_KEYS_ADAPTER: Final = TypeAdapter(frozenset[str])
+
+# litellm/videos/utils.py merges the caller's raw ``extra_body`` back over the mapped params, which returns the keys
+# this adapter maps itself a second time, as fields of their own. They are already accounted for, so they count as
+# handled rather than unknown.
+_MAPPED_EXTRA_BODY_COPIES: Final = frozenset({"size", "seconds", "user"})
+_CREATE_HANDLED_KEYS: Final = frozenset(
+    {
+        *_CREATE_PASSTHROUGH_KEYS,
+        "content",
+        "input_reference",
+        "frames",
+        *_MAPPED_EXTRA_BODY_COPIES,
+        *_LITELLM_CONTROLS,
+    }
+)
+
+_VIDEO_INPUT_UNSUPPORTED: Final = (
+    "{operation} is not supported through litellm: Ark bills a task with a video input at a different rate per "
+    "token, and litellm bills the video when the task is created, before that cost is known"
+)
+_INPUT_VIDEO_ROLE: Final = "reference_video"
+_INPUT_VIDEO_TYPE: Final = "video_url"
+
+_RATIO_SET: Final = ("16:9", "9:16", "4:3", "3:4", "1:1", "21:9")
+_RESOLUTION_SHORTEST_SIDE: Final = MappingProxyType({"480p": 480, "720p": 720, "1080p": 1080, "4k": 2160})
+_TIER_FRAME_ASPECT: Final = 16 / 9
+
+
+def _video_error(raw_response: httpx.Response, status_code: int, message: str) -> VolcEngineVideoError:
+    """The provider's own error status wins, and a body that failed on a 2xx answers the caller's status instead,
+    since a 2xx error has the proxy retry a call the provider already rejected."""
+    return VolcEngineVideoError(
+        status_code=raw_response.status_code if raw_response.status_code >= 400 else status_code,
+        message=message,
+        headers=raw_response.headers,
+    )
+
+
+def _parse(adapter: TypeAdapter[_ParsedT], raw_response: httpx.Response) -> _ParsedT:
+    try:
+        return adapter.validate_python(raw_response.json())
+    except (json.JSONDecodeError, ValidationError) as e:
+        raise _video_error(raw_response, 502, f"Unexpected Volcengine video response: {e}") from e
+
+
+def _unsupported(model: str, message: str) -> UnsupportedParamsError:
+    return UnsupportedParamsError(message=message, model=model, llm_provider="volcengine")
+
+
+def _droppable(model: str, message: str) -> UnsupportedParamsError:
+    return _unsupported(model, f"{message}, set drop_params=True to ignore it")
+
+
+def _normalized_api_base(api_base: str) -> str:
+    """The Ark credential is shared with the chat, responses and Anthropic surfaces, so its api_base may carry any of
+    their paths, and the video paths carry their own version segment."""
+    trimmed: Final = api_base.rstrip("/")
+    marker_index: Final = trimmed.rfind(_VOLCENGINE_API_MARKER)
+    if marker_index != -1:
+        return trimmed[:marker_index]
+    suffix: Final = next((suffix for suffix in _VOLCENGINE_COMPAT_SUFFIXES if trimmed.endswith(suffix)), None)
+    return trimmed[: -len(suffix)] if suffix is not None else trimmed
+
+
+def _size_dimensions(size: str) -> tuple[int, int] | None:
+    width_str, separator, height_str = size.partition("x")
+    if not separator or not (width_str.isdigit() and height_str.isdigit()):
+        return None
+    if not int(width_str) or not int(height_str):
+        return None
+    return int(width_str), int(height_str)
+
+
+def _ratio_value(ratio: str) -> float:
+    width_str, _, height_str = ratio.partition(":")
+    return int(width_str) / int(height_str)
+
+
+def _ratio_from_size(size: str) -> str | None:
+    """Ark only takes its named ratios, and its own tiers do not reduce onto them (854x480 is 427:240), so the
+    closest ratio wins."""
+    dimensions: Final = _size_dimensions(size)
+    if dimensions is None:
+        return None
+    width, height = dimensions
+    return min(_RATIO_SET, key=lambda candidate: abs(log(width / height / _ratio_value(candidate))))
+
+
+def _resolution_from_size(size: str) -> str | None:
+    """Ark keeps a tier's pixel count across its ratios (960x960 and 1280x720 are both 720p), so the tier whose 16:9
+    frame has the closest pixel count wins."""
+    dimensions: Final = _size_dimensions(size)
+    if dimensions is None:
+        return None
+    width, height = dimensions
+    return min(
+        _RESOLUTION_SHORTEST_SIDE,
+        key=lambda tier: abs(log(width * height / (_RESOLUTION_SHORTEST_SIDE[tier] ** 2 * _TIER_FRAME_ASPECT))),
+    )
+
+
+def _duration_param(seconds: object) -> int | None:
+    if isinstance(seconds, bool):
+        return None
+    if isinstance(seconds, int):
+        return seconds
+    if not isinstance(seconds, (str, float)):
+        return None
+    try:
+        parsed: Final = float(seconds)
+    except ValueError:
+        return None
+    return int(parsed) if parsed.is_integer() else None
+
+
+def _duration_fields(seconds: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if seconds is None:
+        return _EMPTY_PARAMS
+    duration: Final = _duration_param(seconds)
+    if duration is not None:
+        return MappingProxyType({"duration": duration})
+    if drop_params:
+        return _EMPTY_PARAMS
+    raise _droppable(model, f"seconds must be a whole number of seconds, got {seconds!r}")
+
+
+def _size_fields(size: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if size is None:
+        return _EMPTY_PARAMS
+    ratio: Final = _ratio_from_size(size) if isinstance(size, str) else None
+    resolution: Final = _resolution_from_size(size) if isinstance(size, str) else None
+    if ratio is not None and resolution is not None:
+        return MappingProxyType({"ratio": ratio, "resolution": resolution})
+    if drop_params:
+        return _EMPTY_PARAMS
+    raise _droppable(model, f"size must be WIDTHxHEIGHT, got {size!r}")
+
+
+def _parameters_fields(parameters: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if parameters is None:
+        return _EMPTY_PARAMS
+    try:
+        fields: Final = _FIELDS_ADAPTER.validate_python(parameters)
+    except ValidationError:
+        if drop_params:
+            return _EMPTY_PARAMS
+        raise _droppable(
+            model, f"parameters must be an object of Ark request fields, got {type(parameters).__name__}"
+        ) from None
+    misplaced: Final = tuple(sorted(key for key in fields if key in _READ_FROM_THE_REQUEST))
+    if misplaced and not drop_params:
+        raise _droppable(model, f"send {', '.join(misplaced)} as top-level request fields, not inside parameters")
+    return MappingProxyType({key: value for key, value in fields.items() if key not in _READ_FROM_THE_REQUEST})
+
+
+def _user_fields(user: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if isinstance(user, str) and user:
+        return MappingProxyType({"safety_identifier": user})
+    if user is None or isinstance(user, str) or drop_params:
+        return _EMPTY_PARAMS
+    raise _droppable(model, f"user must be a string, got {type(user).__name__}")
+
+
+def _additional_drop_keys(params: Mapping[str, object]) -> frozenset[str]:
+    try:
+        return _DROP_KEYS_ADAPTER.validate_python(params.get("additional_drop_params") or ())
+    except ValidationError:
+        return frozenset()
+
+
+def _without(params: Mapping[str, object], keys: frozenset[str]) -> Mapping[str, object]:
+    return MappingProxyType({key: value for key, value in params.items() if key not in keys})
+
+
+def _malformed_fields(fields: Mapping[str, object]) -> tuple[tuple[str, str], ...]:
+    try:
+        _ = _KnownFields.model_validate(dict(fields))
+    except ValidationError as e:
+        return tuple((str(error["loc"][0]), error["msg"]) for error in e.errors() if error["loc"])
+    return ()
+
+
+def _drops_params(litellm_params: GenericLiteLLMParams) -> bool:
+    return litellm_params.drop_params is True or litellm.drop_params is True
+
+
+def _model_info(model: str) -> ModelInfo | None:
+    try:
+        return get_model_info(model=model, custom_llm_provider="volcengine")
+    except Exception:  # noqa: BLE001  # get_model_info raises bare Exception for unmapped models
+        return None
+
+
+def _pricing_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
+    """A deployment priced through ``model_info.base_model``, like an ``ep-`` endpoint id, bills at that model's
+    tiers."""
+    model_info: Final = _FIELDS_ADAPTER.validate_python(litellm_params["model_info"] or _EMPTY_PARAMS)
+    base_model: Final = model_info.get("base_model")
+    return base_model if isinstance(base_model, str) and base_model else model
+
+
+def _priced_tiers(model: str) -> tuple[str, ...] | None:
+    """A tier the model has no price for is one Ark does not offer it at, and litellm would bill it at the base rate.
+    A model with no tier price listed, missing from the cost map or registered unpriced by the router, has nothing
+    to check against."""
+    info: Final = _model_info(model)
+    if info is None:
+        return None
+    return (
+        tuple(tier for tier in _RESOLUTION_SHORTEST_SIDE if info.get(f"output_cost_per_second_{tier}") is not None)
+        or None
+    )
+
+
+def _delivered_frames(model: str | None) -> Mapping[str, Mapping[str, str]]:
+    info: Final = _model_info(model) if model else None
+    provider_specific: Final[object] = info.get("provider_specific_entry") if info is not None else None
+    frames: Final[object] = provider_specific.get(_FRAMES_FIELD) if isinstance(provider_specific, Mapping) else None
+    try:
+        return _FRAMES_ADAPTER.validate_python(frames) if frames is not None else _EMPTY_FRAMES
+    except ValidationError:
+        return _EMPTY_FRAMES
+
+
+def _size_from_geometry(resolution: str | None, ratio: str | None, model: str | None) -> str | None:
+    """Ark echoes a tier and a ratio rather than pixels, and its rendered frame does not always follow the tier's
+    nominal size, so the frame listed for the model wins over scaling the tier's shortest side up the ratio."""
+    if not resolution or not ratio or ":" not in ratio:
+        return None
+    tier: Final = resolution.strip().lower()
+    normalized_ratio: Final = ratio.strip()
+    listed: Final = _delivered_frames(model).get(tier, MappingProxyType({})).get(normalized_ratio)
+    if listed is not None:
+        return listed
+    shortest_side: Final = _RESOLUTION_SHORTEST_SIDE.get(tier)
+    width_str, _, height_str = normalized_ratio.partition(":")
+    if shortest_side is None or not (width_str.isdigit() and height_str.isdigit()):
+        return None
+    ratio_width: Final = int(width_str)
+    ratio_height: Final = int(height_str)
+    if not ratio_width or not ratio_height:
+        return None
+    shortest_ratio_side: Final = min(ratio_width, ratio_height)
+    return f"{-(-shortest_side * ratio_width // shortest_ratio_side)}x{-(-shortest_side * ratio_height // shortest_ratio_side)}"
+
+
+def _is_input_video(item: _ContentItem) -> bool:
+    return item.type == _INPUT_VIDEO_TYPE or item.role == _INPUT_VIDEO_ROLE
+
+
+def _supplied_items(supplied: object, model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]:
+    try:
+        items: Final = _CONTENT_ITEMS_ADAPTER.validate_python(supplied)
+        entries: Final = tuple(_ContentItem.model_validate(item) for item in items)
+    except ValidationError:
+        if drop_params:
+            return ()
+        raise _droppable(model, "content must be an array of Ark content items") from None
+    if any(entry.type == "text" for entry in entries) and not drop_params:
+        raise _droppable(
+            model,
+            "A text item in content is not supported, put the text in prompt since it is the only text input "
+            "guardrails scan before the request reaches Ark",
+        )
+    if any(_is_input_video(entry) for entry in entries) and not drop_params:
+        raise _droppable(model, _VIDEO_INPUT_UNSUPPORTED.format(operation="Video input (a reference_video item)"))
+    kept: Final = (item for item, entry in zip(items, entries) if entry.type != "text" and not _is_input_video(entry))
+    return tuple(MappingProxyType(item) for item in kept)
+
+
+def _reference_items(params: Mapping[str, object], model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]:
+    """A caller-supplied ``content`` array wins over ``input_reference``, since its roles are the only way to express
+    reference-to-video."""
+    supplied: Final = params.get("content")
+    if supplied is not None:
+        return _supplied_items(supplied, model, drop_params)
+    reference: Final = params.get("input_reference")
+    if reference is None:
+        return ()
+    try:
+        url: Final = input_reference_url(reference)
+    except ValueError as e:
+        if drop_params:
+            return ()
+        raise _droppable(model, str(e)) from None
+    return (MappingProxyType({"type": "image_url", "image_url": {"url": url}, "role": "first_frame"}),)
+
+
+def _sent_content(
+    prompt: str, params: Mapping[str, object], model: str, drop_params: bool
+) -> tuple[Mapping[str, object], ...]:
+    """The prompt always leads, as it is the only text the input guardrails scan."""
+    return (MappingProxyType({"type": "text", "text": prompt}), *_reference_items(params, model, drop_params))
+
+
+def _seconds_label(seconds: object) -> str | None:
+    if isinstance(seconds, float) and seconds.is_integer():
+        return str(int(seconds))
+    return None if seconds is None else str(seconds)
+
+
+def _requested_usage(request_data: Mapping[str, object]) -> Mapping[str, object]:
+    """The create call is the only billed one and Ark's create response carries no usage, so the requested duration
+    and tier are what the video bills at."""
+    duration: Final = _duration_param(request_data.get("duration"))
+    resolution: Final = request_data.get("resolution")
+    return MappingProxyType(
+        {
+            key: value
+            for key, value in (
+                ("duration_seconds", float(duration) if duration is not None and duration > 0 else None),
+                ("video_resolution", resolution.strip().lower() if isinstance(resolution, str) else None),
+            )
+            if value is not None
+        }
+    )
+
+
+def _task_usage(task: _Task) -> Mapping[str, object]:
+    resolution: Final = task.resolution.strip().lower() if task.resolution else None
+    return MappingProxyType(
+        {
+            key: value
+            for key, value in (
+                ("duration_seconds", float(task.duration) if task.duration is not None else None),
+                ("video_resolution", resolution),
+                ("total_tokens", task.usage.total_tokens if task.usage is not None else None),
+            )
+            if value is not None
+        }
+    )
+
+
+def _task_status(task: _Task, raw_response: httpx.Response) -> str:
+    status: Final = _STATUS_MAP.get(task.status.strip().lower())
+    if status is None:  # defaulting it would hand back a job the caller polls forever
+        raise _video_error(raw_response, 502, f"Unknown Volcengine task status: {task.status}")
+    return status
+
+
+def _video_object_from_task(task: _Task, raw_response: httpx.Response, model: str | None) -> VideoObject:
+    status: Final = _task_status(task, raw_response)
+    usage: Final = _task_usage(task)
+    seconds: Final = usage.get("duration_seconds")
+    error: Final[dict[str, object] | None] = (  # mutable-ok: VideoObject.error is a dict field
+        {key: value for key, value in (("code", task.error.code), ("message", task.error.message)) if value is not None}
+        if task.error is not None
+        else None
+    )
+    return VideoObject(
+        id=task.id,
+        object="video",
+        status=status,
+        created_at=task.created_at,
+        completed_at=task.updated_at if status in _TERMINAL_STATUSES else None,
+        error=error,
+        seconds=_seconds_label(seconds),
+        size=_size_from_geometry(task.resolution, task.ratio, task.model),
+        model=model or task.model,
+        usage=dict(usage) if usage else None,
+    )
+
+
+def _video_url_from_task(task: _Task, raw_response: httpx.Response) -> str:
+    """A download before the video is ready is the caller polling too early, not a fault of the deployment, so it
+    answers 400, which litellm neither retries nor counts toward a deployment cooldown. A 404 would cool the
+    deployment down, and a 409 or a 5xx would be retried against a job that is simply not finished."""
+    if task.content is not None and task.content.video_url:
+        return task.content.video_url
+    status: Final = task.status.strip().lower()
+    if status in _PENDING_STATUSES:
+        raise _video_error(
+            raw_response, 400, f"Video is still processing (status: {status}). Please wait and try again."
+        )
+    if task.error is not None and (task.error.message or task.error.code):
+        raise _video_error(raw_response, 400, f"Video generation failed: {task.error.message or task.error.code}")
+    if status == "expired":
+        raise _video_error(raw_response, 400, "Task expired. Ark keeps task records for 7 days after creation.")
+    if status in ("failed", "cancelled"):
+        raise _video_error(raw_response, 400, f"Video generation ended as {status} with no video URL.")
+    raise _video_error(raw_response, 502, "Video URL not found in task response.")
+
+
+class VolcEngineVideoConfig(BaseVideoConfig):
+    """Volcengine Ark (Doubao-Seedance) on Ark's async task API: create, poll, download from the signed
+    ``content.video_url``, and list."""
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: BaseVideoConfig contract returns list
+        return [
+            "model",
+            "prompt",
+            "input_reference",
+            "seconds",
+            "size",
+            "user",
+            "extra_headers",
+            "content",
+            "parameters",
+            *_CREATE_PASSTHROUGH_KEYS,
+        ]
+
+    def map_openai_params(
+        self,
+        video_create_optional_params: VideoCreateOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: BaseVideoConfig contract
+        requested: Final = _without(video_create_optional_params, _additional_drop_keys(video_create_optional_params))
+        unsupported: Final = next(
+            (key for key, value in requested.items() if key in _UNSUPPORTED_BY_VOLCENGINE and value is not None),
+            None,
+        )
+        if unsupported is not None and not drop_params:
+            raise _droppable(model, f"'{unsupported}' is not supported by Volcengine video generation")
+        passthrough: Final = MappingProxyType(
+            {key: value for key, value in requested.items() if value is not None and key not in _MAPPED_ELSEWHERE}
+        )
+        return {
+            **_user_fields(requested.get("user"), model, drop_params),
+            **_size_fields(requested.get("size"), model, drop_params),
+            **_duration_fields(requested.get("seconds"), model, drop_params),
+            **passthrough,
+            **_parameters_fields(requested.get("parameters"), model, drop_params),
+        }
+
+    def _resolve_api_key(self, api_key: str | None) -> str:
+        resolved_api_key: Final = api_key or get_secret_str("ARK_API_KEY") or get_secret_str("VOLCENGINE_API_KEY")
+        if resolved_api_key is None:
+            raise ValueError(
+                "Volcengine API key is required. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass "
+                "api_key parameter."
+            )
+        return resolved_api_key
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract; handler expects a mutable headers dict
+        model: str,
+        api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseVideoConfig contract
+        resolved_api_key: Final = self._resolve_api_key(
+            api_key
+            or (litellm_params.api_key if litellm_params is not None and litellm_params.api_key else None)
+            or litellm.api_key
+        )
+        headers.update({"Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"})
+        return headers
+
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: str | None,
+        litellm_params: dict[str, object],  # mutable-ok: BaseVideoConfig contract
+    ) -> str:
+        return _normalized_api_base(
+            get_volcengine_base_url(api_base or get_secret_str("VOLCENGINE_API_BASE") or get_secret_str("ARK_API_BASE"))
+        )
+
+    def transform_video_create_request(
+        self,
+        model: str,
+        prompt: str,
+        api_base: str,
+        video_create_optional_request_params: dict[str, object],  # mutable-ok: BaseVideoConfig contract
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[dict[str, object], RequestFiles, str]:  # mutable-ok: BaseVideoConfig contract
+        """
+        Ark bills a video when the task finishes, while litellm bills it here at create, so every request has to
+        carry a length and a tier whose cost is known now. Ark-picked length (``duration: -1``) and a video input
+        (edits, extensions and reference videos, billed at a different rate per token) are rejected for that reason,
+        and an omitted length is sent as OpenAI's documented default of 4 seconds, since the 2.5 family's own default
+        is ``-1``.
+        """
+        drop_params: Final = _drops_params(litellm_params)
+        dropped_by_config: Final = _additional_drop_keys(video_create_optional_request_params)
+        kept: Final = _without(video_create_optional_request_params, dropped_by_config)
+        params: Final = _without(
+            {
+                **{key: value for key, value in kept.items() if key != "parameters"},
+                **_parameters_fields(kept.get("parameters"), model, drop_params),
+            },
+            dropped_by_config,
+        )
+        content: Final = _sent_content(prompt, params, model, drop_params)
+        if params.get("frames") is not None and not drop_params:
+            raise _droppable(model, "frames is not supported through litellm, pass seconds instead")
+        requested_duration: Final = params.get("duration")
+        parsed_duration: Final = None if requested_duration is None else _duration_param(requested_duration)
+        if requested_duration is not None and (parsed_duration is None or parsed_duration <= 0) and not drop_params:
+            raise _droppable(
+                model,
+                f"duration must be a positive whole number of seconds, got {requested_duration!r}. Ark-picked length "
+                "(-1) is not supported through litellm, since the video is billed when the task is created, before "
+                "Ark has picked its length",
+            )
+        duration: Final = (
+            parsed_duration
+            if parsed_duration is not None and parsed_duration > 0
+            else VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS
+        )
+
+        unknown_fields: Final = tuple(
+            sorted(key for key, value in params.items() if value is not None and key not in _CREATE_HANDLED_KEYS)
+        )
+        if unknown_fields and not drop_params:
+            raise _droppable(model, f"litellm does not send {', '.join(unknown_fields)} to Ark video generation")
+        requested_fields: Final = MappingProxyType(
+            {key: params[key] for key in _CREATE_PASSTHROUGH_KEYS if params.get(key) is not None}
+        )
+        malformed: Final = _malformed_fields(requested_fields)
+        if malformed and not drop_params:
+            raise _droppable(model, "; ".join(f"invalid {field}: {message}" for field, message in malformed))
+        resolution: Final = requested_fields.get("resolution")
+        priced_tiers: Final = _priced_tiers(_pricing_model(model, litellm_params))
+        drops_resolution: Final = (
+            priced_tiers is not None and isinstance(resolution, str) and resolution.strip().lower() not in priced_tiers
+        )
+        if drops_resolution and not drop_params:
+            raise _droppable(
+                model, f"{model} is not offered at resolution {resolution!r}, only at {', '.join(priced_tiers or ())}"
+            )
+        dropped: Final = frozenset({*(field for field, _ in malformed), *(("resolution",) if drops_resolution else ())})
+        kept_fields: Final = MappingProxyType(
+            {key: value for key, value in requested_fields.items() if key not in dropped}
+        )
+        default_tier: Final = (
+            MappingProxyType({"resolution": VOLCENGINE_VIDEO_DEFAULT_RESOLUTION})
+            if "resolution" not in kept_fields
+            and priced_tiers is not None
+            and VOLCENGINE_VIDEO_DEFAULT_RESOLUTION in priced_tiers
+            else _EMPTY_PARAMS
+        )
+
+        request_data: Final[dict[str, object]] = {  # mutable-ok: request body dict, JSON-serialized by the handler
+            "model": model,
+            "content": [dict(item) for item in content],
+            **default_tier,
+            **kept_fields,
+            "duration": duration,
+        }
+        return request_data, (), f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}"
+
+    def transform_video_create_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+        request_data: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> VideoObject:
+        task: Final = _parse(_TASK_CREATE_ADAPTER, raw_response)
+        if not task.id:
+            message: Final = task.error.message if task.error is not None and task.error.message else "no task id"
+            raise _video_error(raw_response, 400, f"Volcengine did not return a video task id: {message}")
+        requested: Final = request_data or _EMPTY_PARAMS
+        usage: Final = _requested_usage(requested)
+        seconds: Final = usage.get("duration_seconds")
+        requested_model, served_model = routed_video_models(logging_obj) or (model, None)
+        resolution: Final = requested.get("resolution")
+        ratio: Final = requested.get("ratio")
+        return VideoObject(
+            id=encode_video_id_with_provider(task.id, custom_llm_provider, requested_model, served_model)
+            if custom_llm_provider
+            else task.id,
+            object="video",
+            status="queued",
+            model=model,
+            seconds=_seconds_label(seconds),
+            size=_size_from_geometry(
+                resolution if isinstance(resolution, str) else None, ratio if isinstance(ratio, str) else None, model
+            ),
+            usage=dict(usage) if usage else None,
+        )
+
+    @staticmethod
+    def _task_url(video_id: str, api_base: str) -> str:
+        original_task_id: Final = extract_original_video_id(video_id)
+        encoded_task_id: Final = encode_url_path_segment(original_task_id, field_name="video_id")
+        return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}/{encoded_task_id}"
+
+    def transform_video_status_retrieve_request(
+        self,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        return self._task_url(video_id, api_base), _EMPTY_PARAMS
+
+    def transform_video_status_retrieve_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+        client: "HTTPHandler | None" = None,
+    ) -> VideoObject:
+        task: Final = _parse(_TASK_ADAPTER, raw_response)
+        model, served_model = polled_video_models(logging_obj)
+        video_obj: Final = _video_object_from_task(task, raw_response, model)
+        if custom_llm_provider:
+            video_obj.id = encode_video_id_with_provider(task.id, custom_llm_provider, model, served_model)
+        return video_obj
+
+    def transform_video_content_request(
+        self,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        variant: str | None = None,
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        return self._task_url(video_id, api_base), _EMPTY_PARAMS
+
+    def transform_video_content_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        client: "HTTPHandler | None" = None,
+    ) -> bytes:
+        video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response)
+        httpx_client: Final[HTTPHandler] = client if client is not None else _get_httpx_client()
+        video_response: Final = httpx_client.get(video_url)  # pyright: ignore[reportUnknownMemberType]  # HTTPHandler.get stub leaves params/headers untyped
+        video_response.raise_for_status()
+        return video_response.content
+
+    async def async_transform_video_content_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        client: "AsyncHTTPHandler | None" = None,
+    ) -> bytes:
+        video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response)
+        async_httpx_client: Final[AsyncHTTPHandler] = (
+            client if client is not None else get_async_httpx_client(llm_provider=litellm.LlmProviders.VOLCENGINE)
+        )
+        video_response: Final = await async_httpx_client.get(video_url)  # pyright: ignore[reportUnknownMemberType]  # HTTPHandler.get stub leaves params/headers untyped
+        video_response.raise_for_status()
+        return video_response.content
+
+    def transform_video_remix_request(
+        self,
+        video_id: str,
+        prompt: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        extra_body: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video remix"))
+
+    def transform_video_remix_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+    ) -> VideoObject:
+        raise NotImplementedError("Video remix is not supported by Volcengine.")
+
+    def transform_video_edit_request(
+        self,
+        prompt: str,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        video_file: object | None = None,
+        extra_body: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+        prefetched_source_data: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
+        raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video edit"))
+
+    def transform_video_extension_request(
+        self,
+        prompt: str,
+        video_id: str,
+        seconds: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        extra_body: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video extension"))
+
+    def transform_video_list_request(
+        self,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        after: str | None = None,
+        limit: int | None = None,
+        order: str | None = None,
+        extra_query: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        """Ark pages by page_num/page_size with filters and has no cursor or order."""
+        unsupported: Final = tuple(name for name, value in (("after", after), ("order", order)) if value is not None)
+        if unsupported and not _drops_params(litellm_params):
+            raise _droppable(
+                "", f"Ark's task list pages by page_num in extra_query and has no {' or '.join(unsupported)}"
+            )
+        return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}", {
+            **({"page_size": str(limit)} if limit is not None else {}),
+            **(extra_query or {}),
+        }
+
+    def transform_video_list_response(  # pyright: ignore[reportIncompatibleMethodOverride]  # base declares dict[str, str] but the payload is a heterogeneous list body
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+    ) -> dict[str, object]:  # mutable-ok: OpenAI list body served as JSON by the proxy
+        list_payload: Final = _parse(_TASK_LIST_ADAPTER, raw_response)
+        videos: Final = tuple(_video_object_from_task(task, raw_response, None) for task in list_payload.items)
+        data: Final = [
+            video.model_dump()
+            | (
+                {"id": encode_video_id_with_provider(video.id, custom_llm_provider, video.model)}
+                if custom_llm_provider
+                else {}
+            )
+            for video in videos
+        ]
+        return {
+            "object": "list",
+            "data": data,
+            "total": list_payload.total if list_payload.total is not None else len(data),
+            "has_more": False,
+            **({"first_id": data[0]["id"], "last_id": data[-1]["id"]} if data else {}),
+        }
+
+    def transform_video_delete_request(
+        self,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        raise NotImplementedError("Video delete is not supported by Volcengine.")
+
+    def transform_video_delete_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+    ) -> VideoObject:
+        raise NotImplementedError("Video delete is not supported by Volcengine.")
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str] | httpx.Headers,  # mutable-ok: BaseVideoConfig contract
+    ) -> BaseLLMException:
+        return VolcEngineVideoError(status_code=status_code, message=error_message, headers=headers)

litellm/model_prices_and_context_window_backup.json

@@ -2773,7 +2773,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2808,7 +2808,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2843,7 +2843,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2878,7 +2878,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2913,7 +2913,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2948,7 +2948,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -3743,7 +3743,7 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -15494,7 +15494,7 @@
         },
         "supports_adaptive_thinking": true,
         "supports_legacy_thinking": true,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -20832,7 +20832,7 @@
         "output_cost_per_token": 2.5000010000000002e-05,
         "output_dbu_cost_per_token": 0.000357143,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -21047,7 +21047,7 @@
         "output_cost_per_token": 1.5000020000000002e-05,
         "output_dbu_cost_per_token": 0.000214286,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -49176,7 +49176,7 @@
         "max_tokens": 64000,
         "mode": "chat",
         "output_cost_per_token": 2.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
@@ -51006,7 +51006,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -51179,7 +51179,7 @@
         "supported_regions": [
             "global"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -51200,7 +51200,7 @@
         "supported_regions": [
             "us-central1"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -58822,7 +58822,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -60467,6 +60467,61 @@
             }
         ]
     },
+    "volcengine/doubao-seedance-2-5-260628": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.216,
+        "output_cost_per_second_480p": 0.096075,
+        "output_cost_per_second_720p": 0.216,
+        "output_cost_per_second_1080p": 0.5346,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.141942857,
+        "output_cost_per_second_480p": 0.066003429,
+        "output_cost_per_second_720p": 0.141942857,
+        "output_cost_per_second_1080p": 0.354085714,
+        "output_cost_per_second_4k": 0.722057143,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-fast-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.114171429,
+        "output_cost_per_second_480p": 0.053089714,
+        "output_cost_per_second_720p": 0.114171429,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-mini-260615": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.070971429,
+        "output_cost_per_second_480p": 0.033001714,
+        "output_cost_per_second_720p": 0.070971429,
+        "supported_modalities": ["text", "image"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
     "bedrock/us-east-1/zai.glm-5": {
         "input_cost_per_token": 1e-06,
         "output_cost_per_token": 3.2e-06,
@@ -72334,7 +72389,7 @@
         "max_output_tokens": 64000,
         "max_tokens": 64000,
         "mode": "chat",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,

litellm/proxy/litellm_pre_call_utils.py

@@ -27,6 +27,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm._uuid import uuid
 from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
     CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -187,6 +188,30 @@ def _caller_trace_field(data: Mapping[str, object], metadata_variable_name: str,
     return requester_value if isinstance(requester_value, str) and requester_value else None
 
 
+def _caller_body_session_id(data: Mapping[str, object], metadata_variable_name: str) -> str | None:
+    top_level: Final = data.get("litellm_session_id")
+    if isinstance(top_level, str) and top_level:
+        return top_level
+    return _caller_trace_field(data, metadata_variable_name, "session_id")
+
+
+def _caller_session_beside_trace_header(
+    normalized_headers: Mapping[str, str], data: Mapping[str, object], metadata_variable_name: str
+) -> str | None:
+    session_headers: Final = {key: value for key, value in normalized_headers.items() if key != "x-litellm-trace-id"}
+    return get_chain_id_from_headers(session_headers) or _caller_body_session_id(data, metadata_variable_name)
+
+
+def _drop_client_caller_session(request_body: Mapping[str, object]) -> None:
+    for metadata_key in ("metadata", "litellm_metadata"):
+        if isinstance(metadata := request_body.get(metadata_key), dict):
+            metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None)
+
+
+def _requester_metadata_snapshot(metadata: Mapping[str, object]) -> Mapping[str, object]:
+    return copy.deepcopy({key: value for key, value in metadata.items() if key != CALLER_SESSION_ID_METADATA_KEY})
+
+
 def _stampable_key_hash(user_api_key_dict: UserAPIKeyAuth) -> str | None:
     """Only proxy-validated keys are stamped, proven by the unforgeable
     via_virtual_key marker AND a known non-secret shape: the sha256 hex digest
@@ -1623,6 +1648,8 @@ class LiteLLMProxyRequestSetup:
         from litellm.proxy._types import LitellmMetadataFromRequestHeaders
 
         metadata_from_headers: Final = LitellmMetadataFromRequestHeaders()
+        request_body: Final = cast(Mapping[str, object], data)  # cast-ok: request body is a str-keyed JSON object
+        _drop_client_caller_session(request_body)
         spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
         if spend_logs_metadata is not None:
             metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
@@ -1631,6 +1658,9 @@ class LiteLLMProxyRequestSetup:
         # Finally update the requests metadata with the `metadata_from_headers`
         #########################################################################################
 
+        normalized_headers: Final = MappingProxyType(
+            {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)}
+        )
         agent_id_from_header: Final = headers.get("x-litellm-agent-id")
         # Explicit litellm headers take precedence; fall back to any x-*-session-id header.
         chain_id: Final = get_chain_id_from_headers(dict(headers))
@@ -1639,6 +1669,13 @@ class LiteLLMProxyRequestSetup:
             metadata_from_headers["agent_id"] = agent_id_from_header
             verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header)
 
+        spend_log_session_id: Final = (
+            _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name)
+            if litellm.request_correlation_in_logs
+            and chain_id is not None
+            and chain_id == normalized_headers.get("x-litellm-trace-id")
+            else None
+        )
         if chain_id:
             metadata_from_headers["trace_id"] = chain_id
             metadata_from_headers["session_id"] = chain_id
@@ -1663,7 +1700,6 @@ class LiteLLMProxyRequestSetup:
         # - but lets a caller's existing traceparent/baggage headers (from
         # real OTel instrumentation) correlate with litellm's own logs instead
         # of generating an unrelated trace_id.
-        normalized_headers: Final = MappingProxyType({k.lower(): v for k, v in headers.items() if isinstance(k, str)})
         if "litellm_trace_id" not in data:
             traceparent: Final = normalized_headers.get("traceparent")
             if isinstance(traceparent, str):
@@ -1689,11 +1725,7 @@ class LiteLLMProxyRequestSetup:
             baggage: Final = normalized_headers.get("baggage")
             if isinstance(baggage, str):
                 session_id_from_baggage: Final = _session_id_from_baggage(baggage)
-                caller_session_id: Final = _caller_trace_field(
-                    cast(Mapping[str, object], data),  # cast-ok: request body is a str-keyed JSON object
-                    _metadata_variable_name,
-                    "session_id",
-                )
+                caller_session_id: Final = _caller_trace_field(request_body, _metadata_variable_name, "session_id")
                 if session_id_from_baggage and (
                     caller_session_id is None or caller_session_id == session_id_from_baggage
                 ):
@@ -1702,7 +1734,11 @@ class LiteLLMProxyRequestSetup:
                     verbose_proxy_logger.debug("Extracted session_id from W3C baggage header")
 
         if isinstance(data[_metadata_variable_name], dict):
-            data[_metadata_variable_name].update(metadata_from_headers)
+            data[_metadata_variable_name].update(
+                {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id}
+                if spend_log_session_id
+                else metadata_from_headers
+            )
         return data
 
     @staticmethod
@@ -2417,7 +2453,7 @@ async def add_litellm_data_to_request(
     # them — from leaking into requester_metadata where guardrails and audit
     # paths may read from it.
     if "metadata" in data and isinstance(data["metadata"], dict):
-        data[_metadata_variable_name]["requester_metadata"] = copy.deepcopy(data["metadata"])
+        data[_metadata_variable_name]["requester_metadata"] = _requester_metadata_snapshot(data["metadata"])
         if _metadata_variable_name == "litellm_metadata":
             data[_metadata_variable_name].update(
                 _promoted_trace_control_fields(

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
     CLI_SESSION_KEY_PREFIX,
     EMPTY_MAPPING,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -33,6 +34,8 @@ from litellm.constants import (
 from litellm.constants import (
     MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
 )
+from litellm.integrations.otel.model.metadata import caller_session_id
+from litellm.integrations.otel.model.trace_controls import TraceControls
 from litellm.litellm_core_utils.classifier_logging import (
     CLASSIFIER_AUDIT_FIELDS,
     classifier_audit_fields,
@@ -570,11 +573,11 @@ def get_logging_payload(
         elif isinstance(_usage, dict):
             usage = _usage
 
-    # A request that failed mid-stream has no usable response_obj usage, but the
-    # streaming handler may have recovered the usage from the chunks already
-    # delivered. Honor that override so the partial usage lands in spend tracking.
+    # A request that failed mid-stream has no usable response_obj usage, and a stream
+    # continued on a fallback only has the fallback's. combined_usage_object holds the
+    # usage of every call the client's answer took, so it wins in both cases.
     _combined_usage: Final = kwargs.get("combined_usage_object")
-    if not usage and isinstance(_combined_usage, litellm.Usage):
+    if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage):
         usage = _combined_usage.model_dump()
 
     id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata)
@@ -930,6 +933,16 @@ def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) -
     return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX)
 
 
+def _caller_session_for_spend_log(kwargs: Mapping[str, object], metadata: Mapping[str, object] | None) -> str | None:
+    beside_trace_header: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) if metadata else None
+    caller_session: Final = (
+        beside_trace_header
+        if isinstance(beside_trace_header, str) and beside_trace_header
+        else caller_session_id(kwargs, TraceControls())
+    )
+    return caller_session if caller_session and "\x00" not in caller_session else None
+
+
 def _get_session_id_for_spend_log(
     kwargs: Mapping[str, object],
     metadata: Mapping[str, object] | None,
@@ -937,9 +950,12 @@ def _get_session_id_for_spend_log(
     omit_when_missing: bool,
     batch_trace_session_id: str | None = None,
 ) -> str | None:
-    """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may
-    be a copied trace id. Batch call types carry a deterministic session derived from the batch id, which outranks
-    the per-request trace ids because those differ between the create call and the cost poller's row."""
+    """Resolve the session id for the spend log row: `omit` honors only metadata.session_id,
+    batch sessions outrank everything, then the conversation id the caller sent (also one an x-litellm-trace-id header
+    displaced from the request's session), then trace ids.
+
+    A session minted by `missing_session_id: generate` or copied from the request's trace id is not a conversation, so
+    it yields to the trace id, which the Responses bridge points at the previous turn's session to keep a chain whole."""
     if omit_when_missing:
         session_id: Final = metadata.get("session_id") if metadata else None
         return str(session_id) if session_id else None
@@ -948,11 +964,14 @@ def _get_session_id_for_spend_log(
 
     if batch_trace_session_id is not None:
         return batch_trace_session_id
-    if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None:
-        return str(standard_logging_payload.get("trace_id"))
-    if kwargs.get("litellm_trace_id") is not None:
-        return str(kwargs.get("litellm_trace_id"))
-    return str(uuid.uuid4())
+    if litellm.request_correlation_in_logs and (caller_session := _caller_session_for_spend_log(kwargs, metadata)):
+        return caller_session
+    match standard_logging_payload:
+        case {"trace_id": trace_id} if trace_id is not None:
+            return str(trace_id)
+        case _:
+            litellm_trace_id: Final = kwargs.get("litellm_trace_id")
+            return str(litellm_trace_id) if litellm_trace_id is not None else str(uuid.uuid4())
 
 
 def _get_request_duration_ms(start_time: datetime, end_time: datetime) -> int | None:

litellm/responses/litellm_completion_transformation/session_handler.py

@@ -281,7 +281,13 @@ class ResponsesSessionHandler:
 
         SQL query
 
-        SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id
+        SELECT session_id, api_key, "user", team_id FROM spend_logs WHERE request_id = previous_response_id, then
+        SELECT * FROM spend_logs with that session_id and the same owner
+
+        Only rows with the referenced response's owner are returned: the same user and team, and for a key with
+        neither, the same key. The session id can be caller supplied, so matching on it alone would mix owners. A
+        user's or team's regenerated or second key keeps the history; a key with neither does not, and turns written
+        by another owner (an admin continuing the chain) are not replayed.
 
         A just-finished turn gets a short second chance: the worker that served it may
         still be writing its spend log when the follow-up arrives, and an empty result
@@ -303,13 +309,24 @@ class ResponsesSessionHandler:
 
         query: Final = """
             WITH matching_session AS (
-                SELECT session_id
+                SELECT session_id, api_key, "user", team_id
                 FROM "LiteLLM_SpendLogs"
                 WHERE request_id = $1
             )
             SELECT *
-            FROM "LiteLLM_SpendLogs"
-            WHERE session_id IN (SELECT session_id FROM matching_session)
+            FROM "LiteLLM_SpendLogs" AS logs
+            WHERE EXISTS (
+                SELECT 1
+                FROM matching_session
+                WHERE matching_session.session_id = logs.session_id
+                  AND matching_session."user" IS NOT DISTINCT FROM logs."user"
+                  AND matching_session.team_id IS NOT DISTINCT FROM logs.team_id
+                  AND (
+                      COALESCE(matching_session."user", '') <> ''
+                      OR COALESCE(matching_session.team_id, '') <> ''
+                      OR matching_session.api_key = logs.api_key
+                  )
+            )
             ORDER BY "endTime" ASC;
         """
 

litellm/responses/litellm_completion_transformation/transformation.py

@@ -39,7 +39,11 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.caching import InMemoryCache
-from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
+from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
+    REDACTED_BY_LITELLM,
+    REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
+)
 from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.get_supported_openai_params import (
@@ -563,6 +567,14 @@ class ChatCompletionSession(TypedDict, total=False):
 ########### End of Initialize Classes used for Responses API  ###########
 
 
+def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata"
+    metadata: Final = request_kwargs.get(key)
+    if not isinstance(metadata, dict):
+        return MappingProxyType({})
+    return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}}
+
+
 class LiteLLMCompletionResponsesConfig:
     @staticmethod
     def get_supported_openai_params(model: str) -> list:
@@ -1078,8 +1090,6 @@ class LiteLLMCompletionResponsesConfig:
             else:
                 # Both are empty - this likely means function_call_output had empty/invalid call_id
                 # Provide a helpful error message
-                import litellm
-
                 raise litellm.BadRequestError(
                     message=(
                         f"Unable to create messages for completion request. "
@@ -1094,8 +1104,14 @@ class LiteLLMCompletionResponsesConfig:
                 )
 
         litellm_completion_request["messages"] = combined_messages
-        litellm_completion_request["litellm_trace_id"] = chat_completion_session.get("litellm_session_id")
-        return litellm_completion_request
+        chain_session: Final = chat_completion_session.get("litellm_session_id")
+        litellm_completion_request["litellm_trace_id"] = chain_session
+        chain_session_metadata: Final[Mapping[str, object]] = (
+            _chain_session_metadata(chain_session, request_kwargs)
+            if chain_session and litellm.request_correlation_in_logs
+            else MappingProxyType({})
+        )
+        return {**litellm_completion_request, **chain_session_metadata}
 
     @staticmethod
     def _transform_response_input_param_to_chat_completion_message(

litellm/router.py

@@ -38,7 +38,7 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from functools import lru_cache, partial
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast
 
 import anyio
 import httpx
@@ -68,6 +68,7 @@ from litellm.constants import (
     DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
     DEFAULT_MAX_LRU_CACHE_SIZE,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    MID_STREAM_FAILURE_LOG_WAIT_SECONDS,
     OUTPUT_TOKEN_CEILING_PARAMS,
     PROMPT_CACHE_PIN_CACHE_MAX_SIZE,
     ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
@@ -75,7 +76,7 @@ from litellm.constants import (
     RUNTIME_UPDATABLE_ROUTER_SETTINGS,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
-from litellm.integrations.custom_guardrail import is_guardrail_intervention
+from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.otel.routing import routing_decision_attributes
 from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
@@ -194,6 +195,7 @@ from litellm.router_utils.clientside_credential_handler import (
     is_clientside_credential,
 )
 from litellm.router_utils.common_utils import (
+    MID_STREAM_CONTINUATION_KWARG,
     filter_team_based_models,
     filter_web_search_deployments,
     format_fallback_outcome_message,
@@ -233,7 +235,10 @@ from litellm.router_utils.fallback_event_handlers import (
     check_non_standard_fallback_format,
     clear_pre_routing_selection,
     committed_retry_budget_for_request,
+    continuation_fallback_targets,
     fallback_lookup_groups,
+    fallback_target_name,
+    fallback_target_overrides_request_params,
     fallbacks_disabled_for_request,
     get_fallback_model_group_for_lookup_groups,
     get_pre_routing_selection,
@@ -258,6 +263,17 @@ from litellm.router_utils.handle_error import (
     send_llm_exception_alert,
 )
 from litellm.router_utils.health_state_cache import DeploymentHealthCache
+from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
+    MID_STREAM_CONTINUATION_MARKER,
+    ContinuationOutput,
+    ContinuationPrefillDeploymentCheck,
+    ContinuedPartial,
+    constrains_continuation,
+    continuation_deployments,
+    continuation_output_ceilings,
+    continued_text,
+    deployment_can_continue,
+)
 from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
     DeploymentAffinityCheck,
     warn_on_unknown_model_group_affinity_flags,
@@ -303,6 +319,8 @@ from litellm.router_utils.routing_groups import (
 )
 from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
 from litellm.scheduler import FlowItem, Scheduler
+from litellm.secret_managers.main import str_to_bool
+from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.litellm_params import RoutingStrategyName
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -371,6 +389,7 @@ from litellm.utils import (
     EmbeddingResponse,
     ModelResponse,
     Rules,
+    TextCompletionResponse,
     function_setup,
     get_llm_provider,
     get_non_default_completion_params,
@@ -527,6 +546,15 @@ _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeA
 _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
 _DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected"
+_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
+
+
+def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]:
+    """Continue the last text block in place so the prefill reads as the text the caller already received."""
+    last: Final = blocks[-1] if blocks else None
+    if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str):
+        return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}]
+    return [*blocks, {"type": "text", "text": text}]
 
 
 def _is_fallback_hop(request_kwargs: Mapping[str, object]) -> bool:
@@ -1119,6 +1147,7 @@ class Router:
         health_check_ignore_transient_errors: bool = False,
         background_health_check_model_groups: Sequence[str] | None = None,
         enable_weighted_failover: bool = False,
+        enable_mid_stream_fallback_continuation: bool = False,
         fallback_access_check: FallbackAccessCheck | None = None,
         fallback_budget_check: FallbackBudgetCheck | None = None,
         auto_router_capability_limit: AutoRouterCapabilityLimit | None = None,
@@ -1160,6 +1189,7 @@ class Router:
             deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600.
             ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
             enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False.
+            enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False.
             fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
             fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback).
         Returns:
@@ -1355,6 +1385,7 @@ class Router:
         self.fallback_on_unsupported_params = fallback_on_unsupported_params
         self.enable_health_check_routing = enable_health_check_routing
         self.enable_weighted_failover = enable_weighted_failover
+        self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation)
         self.health_check_ignore_transient_errors = health_check_ignore_transient_errors
         self.background_health_check_model_groups: frozenset[str] | None = (
             frozenset(background_health_check_model_groups)
@@ -1549,6 +1580,14 @@ class Router:
         default_pre_call_checks: Final[OptionalPreCallChecks] = []
         self.add_optional_pre_call_checks(default_pre_call_checks)
 
+        if self.enable_mid_stream_fallback_continuation:
+            self._register_continuation_prefill_check()
+
+    @staticmethod
+    def _register_continuation_prefill_check() -> None:
+        # Process-global on purpose: Router.discard() must not drop a filter another router still needs.
+        litellm.logging_callback_manager.add_litellm_callback(ContinuationPrefillDeploymentCheck())
+
     def discard(self):
         """
         Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
@@ -3143,15 +3182,53 @@ class Router:
 
         usage: Final = cast(Usage | None, getattr(fallback_item, "usage", None))
         usage_objects: Final = [usage] if usage is not None else []
-        if (
-            complete_response_object_usage is not None
-            and hasattr(complete_response_object_usage, "usage")
-            and complete_response_object_usage.usage is not None
-        ):
+        if complete_response_object_usage is not None:
             usage_objects.append(complete_response_object_usage)
         combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=usage_objects)
         setattr(fallback_item, "usage", combined_usage)
 
+    @staticmethod
+    def _price_cut_partial(
+        cut_stream: CustomStreamWrapper,
+        partial_response: ModelResponse | TextCompletionResponse | None,
+    ) -> "ContinuedPartial | None":
+        """Price the cut stream's partial, plus any partial it inherited, before a fallback can reuse its logging."""
+        from litellm.cost_calculator import BaseTokenUsageProcessor
+
+        cut_logging: Final = cut_stream.logging_obj
+        partial_usage: Final = getattr(partial_response, "usage", None)
+        if partial_response is None or not isinstance(partial_usage, Usage):
+            return None
+        inherited_usage: Final = cut_logging.model_call_details.get("continued_partial_usage")
+        inherited_cost: Final = cut_logging.model_call_details.get("continued_partial_cost")
+        inherited_id: Final = cut_logging.model_call_details.get("continued_response_id")
+        priced_partial: Final = partial_response.model_copy(
+            update={"model": cut_stream.model} if cut_stream.model else {}
+        )
+        priced_partial._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # no public accessor
+            key: value for key, value in get_hidden_params_dict(partial_response).items() if key != "response_cost"
+        }
+        return ContinuedPartial(
+            usage=BaseTokenUsageProcessor.combine_usage_objects(
+                usage_objects=[partial_usage, inherited_usage]
+                if isinstance(inherited_usage, Usage)
+                else [partial_usage]
+            ),
+            response_cost=(cut_logging.response_cost_calculator(result=priced_partial) or 0.0)
+            + (inherited_cost if isinstance(inherited_cost, float) else 0.0),
+            response_id=inherited_id if isinstance(inherited_id, str) else partial_response.id,
+        )
+
+    @staticmethod
+    def _carry_partial_spend_to_fallback(fallback_response: object, cut_partial: "ContinuedPartial | None") -> None:
+        """Bill what the cut legs consumed on the continuation's success log."""
+        fallback_logging: Final = getattr(fallback_response, "logging_obj", None)
+        if cut_partial is None or not isinstance(fallback_logging, LiteLLMLogging):
+            return
+        fallback_logging.record_continued_partial(
+            usage=cut_partial.usage, response_cost=cut_partial.response_cost, response_id=cut_partial.response_id
+        )
+
     @staticmethod
     def _prepare_fallback_hidden_params(
         fallback_response: object,
@@ -3201,7 +3278,7 @@ class Router:
         self,
         model_response: CustomStreamWrapper,
         messages: list[dict[str, str]],
-        initial_kwargs: dict,
+        initial_kwargs: dict[str, Any],
         deployment_slot: contextlib.AsyncExitStack | None = None,
         first_chunk: _FirstChunk[AsyncIterator[object]] | None = None,
     ) -> CustomStreamWrapper:
@@ -3264,16 +3341,22 @@ class Router:
                 with anyio.CancelScope(shield=True):
                     await close_model_response()
                     await held_slot.aclose()
-                if not e.is_pre_first_chunk and (
-                    e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)
-                ):
-                    if e.original_exception is not None:
-                        raise e.original_exception from e
-                    raise
+                committed: Final = bool(
+                    not e.is_pre_first_chunk
+                    and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks))
+                )
+                cut_hidden_params: Final = get_hidden_params(model_response) or {}
+                continue_after_content: Final = committed and self._mid_stream_continuation_eligible(
+                    e=e, request_kwargs=initial_kwargs, cut_deployment_id=cut_hidden_params.get("model_id")
+                )
+                if committed and not continue_after_content:
+                    self._raise_original_mid_stream_error(e)
 
                 from litellm.main import stream_chunk_builder
 
-                complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks)
+                complete_response_object: Final = stream_chunk_builder(
+                    chunks=model_response.chunks, messages=messages if continue_after_content else None
+                )
                 complete_response_object_usage: Final = cast(
                     Usage | None,
                     getattr(complete_response_object, "usage", None),
@@ -3289,7 +3372,41 @@ class Router:
                         "content_policy_fallbacks", self.content_policy_fallbacks
                     )
                     initial_kwargs["original_function"] = self._acompletion
-                    initial_kwargs["messages"] = messages
+                    continuation_messages: Final = (
+                        self._build_completion_continuation_input(messages, e.generated_content)
+                        if continue_after_content
+                        else None
+                    )
+                    continuation_output: Final = (
+                        ContinuationOutput(
+                            e.generated_content,
+                            continuation_messages[-1],
+                            model_group,
+                            complete_response_object,
+                            checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG)
+                            is not MID_STREAM_CONTINUATION_MARKER,
+                        )
+                        if continuation_messages is not None
+                        else None
+                    )
+                    cut_partial: Final = (
+                        Router._price_cut_partial(model_response, complete_response_object)
+                        if continuation_output is not None
+                        else None
+                    )
+                    if continue_after_content:
+                        emitted_tokens: Final = int(
+                            getattr(complete_response_object_usage, "completion_tokens", 0) or 0
+                        )
+                        reduced_ceilings: Final = continuation_output_ceilings(initial_kwargs, emitted_tokens)
+                        if reduced_ceilings is None or continuation_messages is None:
+                            self._raise_original_mid_stream_error(e)
+                        await model_response.wait_for_failure_logging(timeout=MID_STREAM_FAILURE_LOG_WAIT_SECONDS)
+                        initial_kwargs.update(reduced_ceilings)
+                        initial_kwargs["messages"] = continuation_messages
+                        initial_kwargs[MID_STREAM_CONTINUATION_KWARG] = MID_STREAM_CONTINUATION_MARKER
+                    else:
+                        initial_kwargs["messages"] = messages
                     self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs)
                     fallback_response = await self.async_function_with_fallbacks_common_utils(
                         e=e,
@@ -3305,11 +3422,17 @@ class Router:
 
                     # If fallback returns a streaming response, iterate over it
                     if hasattr(fallback_response, "__aiter__"):
+                        Router._carry_partial_spend_to_fallback(fallback_response, cut_partial)
                         prepared_fallback_hidden_params = Router._adopt_fallback_response_headers(
                             wrapper_ref, fallback_response
                         )
                         fallback_headers_are_settled = False
-                        async for fallback_item in fallback_response:
+                        fallback_items: Final = (
+                            continuation_output.splice(fallback_response)
+                            if continuation_output is not None
+                            else fallback_response
+                        )
+                        async for fallback_item in fallback_items:
                             if not fallback_headers_are_settled:
                                 fallback_headers_are_settled = True
                                 # a fallback that failed over again only repoints itself once it yields
@@ -3322,7 +3445,10 @@ class Router:
                                 and isinstance(fallback_item, ModelResponseStream)
                                 and hasattr(fallback_item, "usage")
                             ):
-                                self._combine_fallback_usage(fallback_item, complete_response_object_usage)
+                                self._combine_fallback_usage(
+                                    fallback_item,
+                                    complete_response_object_usage if continuation_output is not None else None,
+                                )
                             yield fallback_item
                     else:
                         # If fallback returns a non-streaming response, yield None
@@ -3479,6 +3605,97 @@ class Router:
             total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0),
         )
 
+    @staticmethod
+    def _raise_original_mid_stream_error(e: "MidStreamFallbackError") -> NoReturn:
+        """Surface the provider error the stream wrapper carried instead of the internal MidStreamFallbackError."""
+        if e.original_exception is not None:
+            raise e.original_exception from e
+        raise e
+
+    def _params_sent_to_deployment(
+        self, request_kwargs: Mapping[str, object], deployment: Deployment | None
+    ) -> Mapping[str, object]:
+        """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
+        deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
+        defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None}
+        deployment_tool_choice: Final = (
+            {"tool_choice": deployment_params["tool_choice"]}
+            if deployment_params.get("tool_choice") is not None
+            else {}
+        )
+        return MappingProxyType({**deployment_params, **defaults, **deployment_tool_choice, **request_kwargs})
+
+    def _names_deployment_that_cannot_continue(self, target: object) -> bool:
+        name: Final = fallback_target_name(target)
+        if name is None or name in self.model_names or not self.has_model_id(name):
+            return False
+        deployment: Final = self.get_deployment(model_id=name)
+        return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True))
+
+    def _mid_stream_continuation_eligible(
+        self,
+        e: "MidStreamFallbackError",
+        request_kwargs: Mapping[str, object],
+        cut_deployment_id: object = None,
+    ) -> bool:
+        """Whether a stream that broke after plain assistant text may continue via prefill."""
+        if not self.enable_mid_stream_fallback_continuation:
+            return False
+        if not e.generated_content or e.emitted_disqualifying_content:
+            return False
+        if isinstance(e.original_exception, litellm.ContentPolicyViolationError):
+            return False
+        cut_deployment: Final = (
+            self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None
+        )
+        if isinstance(cut_deployment_id, str) and cut_deployment is None:
+            return False
+        if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
+            return False
+        requested_group: Final = request_kwargs.get("model")
+        lookup_groups: Final = fallback_lookup_groups(
+            request_kwargs, requested_group if isinstance(requested_group, str) else None
+        )
+        targets: Final = continuation_fallback_targets(request_kwargs.get("fallbacks", self.fallbacks), lookup_groups)
+        if any(fallback_target_overrides_request_params(target) for target in targets):
+            return False
+        if any(self._names_deployment_that_cannot_continue(target) for target in targets):
+            return False
+        return not any(
+            isinstance(callback, CustomGuardrail)
+            and callback._event_hook_is_event_type(  # pyright: ignore[reportPrivateUsage]  # the only check of a guardrail's hooks
+                GuardrailEventHooks.post_call
+            )
+            for callback in litellm.callbacks
+        )
+
+    @staticmethod
+    def _build_completion_continuation_input(
+        messages: Sequence[Mapping[str, object]],
+        generated_content: str,
+    ) -> Sequence[Mapping[str, object]] | None:
+        """Append the partial output as an assistant prefill, or extend a trailing `prefix: True` assistant turn in
+        place so the request never ends in two assistant messages. None when the request ends in a plain assistant
+        turn, in a prefill turn whose content has a shape this cannot extend, or when the cut came mid-echo of the
+        prefill."""
+        last: Final = messages[-1] if messages else None
+        if last is None or last.get("role") != "assistant":
+            return [*messages, {"role": "assistant", "content": generated_content, "prefix": True}]
+        if last.get("prefix") is not True:
+            return None
+        new_text: Final = continued_text(last, generated_content)
+        if new_text is None:
+            return None
+        content: Final = last.get("content")
+        if content is None or isinstance(content, str):
+            return [*messages[:-1], {**last, "content": f"{content or ''}{new_text}", "prefix": True}]
+        try:
+            blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content)
+        except ValidationError:
+            return None
+        extended: Final = _extend_trailing_text_block(blocks, new_text)
+        return [*messages[:-1], {**last, "content": extended, "prefix": True}]
+
     @staticmethod
     def _build_responses_continuation_input(
         input_val: Union[str, "ResponseInputParam"] | None,
@@ -3880,13 +4097,6 @@ class Router:
                         raise e.original_exception from e
                     raise
 
-                from litellm.main import stream_chunk_builder
-
-                complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks)
-                complete_response_object_usage: Final = cast(
-                    Usage | None,
-                    getattr(complete_response_object, "usage", None),
-                )
                 try:
                     model_group: Final = cast(str, initial_kwargs.get("model"))
                     fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", router_self.fallbacks)
@@ -3932,7 +4142,7 @@ class Router:
                                 and isinstance(fallback_item, ModelResponseStream)
                                 and hasattr(fallback_item, "usage")
                             ):
-                                router_self._combine_fallback_usage(fallback_item, complete_response_object_usage)
+                                router_self._combine_fallback_usage(fallback_item, None)
                             yield fallback_item
                     else:
                         yield None
@@ -13004,6 +13214,7 @@ class Router:
             "retry_policy",
             "model_group_alias",
             "enable_weighted_failover",
+            "enable_mid_stream_fallback_continuation",
             "enable_tag_filtering",
             "tag_routing_prefix",
             "fallback_on_unsupported_params",
@@ -13060,12 +13271,10 @@ class Router:
                     rebuild_routing_groups = True
                 elif var == "optional_pre_call_checks":
                     self.set_optional_pre_call_checks(kwargs[var])
+                elif var == "enable_mid_stream_fallback_continuation":
+                    self._set_mid_stream_fallback_continuation(kwargs[var])
                 elif var == "retry_policy":
-                    value = kwargs[var]
-                    if isinstance(value, dict):
-                        value = RetryPolicy(**value)
-                    if value is None or isinstance(value, RetryPolicy):
-                        setattr(self, var, value)
+                    self._set_retry_policy(kwargs[var])
                 else:
                     value = kwargs[var]
                     # only run routing strategy init if it has changed
@@ -13092,6 +13301,20 @@ class Router:
             self._routing_groups_input = routing_groups_input
         verbose_router_logger.debug("Updated Router settings: %s", self.get_settings())
 
+    @staticmethod
+    def _is_flag_enabled(value: object) -> bool:
+        return value is True or (isinstance(value, str) and str_to_bool(value) is True)
+
+    def _set_mid_stream_fallback_continuation(self, enabled: object) -> None:
+        self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enabled)
+        if self.enable_mid_stream_fallback_continuation:
+            self._register_continuation_prefill_check()
+
+    def _set_retry_policy(self, value: object) -> None:
+        policy: Final = RetryPolicy.model_validate(value) if isinstance(value, dict) else value
+        if policy is None or isinstance(policy, RetryPolicy):
+            self.retry_policy = policy
+
     def _get_client(self, deployment, kwargs, client_type=None):
         """
         Returns the appropriate client based on the given deployment, kwargs, and client_type.
@@ -15119,9 +15342,8 @@ class Router:
             request_kwargs=request_kwargs,
         )
         strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
-        pick_attributes: Final = _deployment_pick_attributes(
-            model, _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs)
-        )
+        routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs)
+        pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs)
 
         if isinstance(healthy_deployments, dict):
             if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@@ -15159,7 +15381,9 @@ class Router:
             )
             healthy_deployments = _pre_cooldown_deployments
 
-        healthy_deployments = self._filter_blocked_deployments(healthy_deployments)
+        healthy_deployments = continuation_deployments(
+            self._filter_blocked_deployments(healthy_deployments), routing_kwargs
+        )
 
         # filter pre-call checks
         if self.enable_pre_call_checks and (messages is not None or input is not None):

litellm/router_utils/common_utils.py

@@ -27,7 +27,8 @@ from litellm.types.utils import LlmProviders, ProviderSpecificModelInfo
 
 _V = TypeVar("_V")
 
-ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors"})
+MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation"
+ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG})
 _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
 
 

litellm/router_utils/fallback_event_handlers.py

@@ -615,6 +615,32 @@ def get_fallback_model_group(
     return fallback_model_group, generic_fallback_idx
 
 
+def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]:
+    group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple(  # mutable-ok: mirrors the fallbacks config shape
+        entry for entry in entries if isinstance(entry, Mapping) and entry
+    )
+    chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups)
+    return chain if isinstance(chain, list) else ()
+
+
+def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]:
+    entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else ()
+    return (
+        entries
+        if check_non_standard_fallback_format(fallbacks=entries)
+        else _resolved_fallback_chain(entries, lookup_groups)
+    )
+
+
+def fallback_target_overrides_request_params(target: object) -> bool:
+    return isinstance(target, Mapping) and "model" in target and len(target) > 1
+
+
+def fallback_target_name(target: object) -> str | None:
+    name: Final = target.get("model") if isinstance(target, Mapping) else target
+    return name if isinstance(name, str) else None
+
+
 PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id")
 PROVIDER_SCOPED_RESOURCE_FUNCTION_NAMES: Final = frozenset(
     {

litellm/router_utils/pre_call_checks/continuation_prefill_check.py

@@ -0,0 +1,264 @@
+"""Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave
+the output unconstrained."""
+
+from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence
+from dataclasses import dataclass
+from itertools import accumulate
+from os.path import commonprefix
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger, Span
+from litellm.litellm_core_utils.rules import Rules
+from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
+from litellm.utils import supports_assistant_prefill
+
+
+class _ContinuationMarker:
+    """Only the router constructs one, so a JSON request body cannot forge the continuation flag."""
+
+
+MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
+
+_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
+_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
+_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens")
+
+
+def _str_keyed(value: object) -> Mapping[str, object]:
+    try:
+        return _STR_KEYED_DICT_ADAPTER.validate_python(value)
+    except ValidationError:
+        return _NO_FIELDS
+
+
+def deployment_supports_prefill(deployment: object) -> bool:
+    deployment_map: Final = _str_keyed(deployment)
+    model_info: Final = _str_keyed(deployment_map.get("model_info"))
+    declared: Final = model_info.get("supports_assistant_prefill")
+    if isinstance(declared, bool):
+        return declared
+    litellm_params: Final = _str_keyed(deployment_map.get("litellm_params"))
+    candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model"))
+    model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
+    return model is not None and supports_assistant_prefill(model=model)
+
+
+def constrains_continuation(params: Mapping[str, object]) -> bool:
+    sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
+    choice_count: Final = sent.get("n")
+    response_format: Final = sent.get("response_format")
+    tool_choice: Final = sent.get("tool_choice")
+    return (
+        (isinstance(choice_count, int) and choice_count > 1)
+        or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
+        or tool_choice == "required"
+        or isinstance(tool_choice, Mapping)
+        or sent.get("merge_reasoning_content_in_choices") is True
+    )
+
+
+def deployment_can_continue(deployment: object) -> bool:
+    litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params"))
+    return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params)
+
+
+def continuation_deployments(
+    healthy_deployments: list[dict[str, object]],  # mutable-ok: router deployment-list contract
+    request_kwargs: Mapping[str, object] | None,
+) -> list[dict[str, object]]:  # mutable-ok: returns a mutable deployment list
+    marker: Final = (request_kwargs or {}).get(MID_STREAM_CONTINUATION_KWARG)
+    if not isinstance(marker, _ContinuationMarker):
+        return healthy_deployments
+    return [deployment for deployment in healthy_deployments if deployment_can_continue(deployment)]
+
+
+class ContinuationPrefillDeploymentCheck(CustomLogger):
+    async def async_filter_deployments(
+        self,
+        model: str,
+        healthy_deployments: list[dict[str, object]],  # mutable-ok: CustomLogger deployment-list contract
+        messages: Sequence[AllMessageValues] | None,
+        request_kwargs: Mapping[str, object] | None = None,
+        parent_otel_span: Span | None = None,
+    ) -> list[dict[str, object]]:  # mutable-ok: returns a mutable deployment list
+        return continuation_deployments(healthy_deployments, request_kwargs)
+
+
+def _delta_text(chunk: ModelResponseStream) -> str:
+    if not chunk.choices:
+        return ""
+    content: Final = chunk.choices[0].delta.content
+    return content if isinstance(content, str) else ""
+
+
+_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
+
+
+def _prefill_text(prefill_message: Mapping[str, object]) -> str:
+    content: Final = prefill_message.get("content")
+    if isinstance(content, str):
+        return content
+    try:
+        blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content)
+    except ValidationError:
+        return ""
+    return "".join(
+        text for block in blocks if block.get("type") == "text" and isinstance(text := block.get("text"), str)
+    )
+
+
+def _resendable_whitespace(prefill: str) -> str:
+    return prefill[len(prefill.rstrip()) :] if prefill.strip() else ""
+
+
+def continued_text(prefill_message: Mapping[str, object], generated_content: str) -> str | None:
+    prefill: Final = _prefill_text(prefill_message)
+    if generated_content and len(generated_content) < len(prefill) and prefill.startswith(generated_content):
+        return None
+    echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0
+    after_echo: Final = generated_content[echo:]
+    return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :]
+
+
+def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]:
+    return MappingProxyType(
+        {key: value - emitted_tokens for key in _OUTPUT_CEILING_KEYS if isinstance(value := params.get(key), int)}
+    )
+
+
+def continuation_output_ceilings(
+    request_kwargs: Mapping[str, object], emitted_tokens: int
+) -> Mapping[str, object] | None:
+    """The caller's output ceilings, top-level and in extra_body, minus the tokens already emitted, or None once
+    nothing is left."""
+    extra_body: Final = _str_keyed(request_kwargs.get("extra_body"))
+    top_level: Final = _reduced_ceilings(request_kwargs, emitted_tokens)
+    in_extra_body: Final = _reduced_ceilings(extra_body, emitted_tokens)
+    if any(ceiling <= 0 for ceiling in (*top_level.values(), *in_extra_body.values())):
+        return None
+    if not in_extra_body:
+        return top_level
+    return MappingProxyType({**top_level, "extra_body": {**extra_body, **in_extra_body}})
+
+
+def _with_first_delta(chunk: ModelResponseStream, update: Mapping[str, object]) -> ModelResponseStream:
+    choice: Final = chunk.choices[0]
+    delta: Final = choice.delta.model_copy(update=update)
+    return chunk.model_copy(update={"choices": [choice.model_copy(update={"delta": delta}), *chunk.choices[1:]]})
+
+
+def _without_leading_text(chunk: ModelResponseStream, count: int) -> ModelResponseStream:
+    if count <= 0:
+        return chunk
+    return _with_first_delta(chunk, {"content": _delta_text(chunk)[count:]})
+
+
+def _without_role(chunk: ModelResponseStream) -> ModelResponseStream:
+    if not chunk.choices or chunk.choices[0].delta.role is None:
+        return chunk
+    return _with_first_delta(chunk, {"role": None})
+
+
+def _stream_identity(primary_response: object) -> Mapping[str, object]:
+    if not isinstance(primary_response, ModelResponse):
+        return _NO_FIELDS
+    return MappingProxyType({"id": primary_response.id, "created": primary_response.created})
+
+
+def _drop_leading_text(chunks: tuple[ModelResponseStream, ...], count: int) -> tuple[ModelResponseStream, ...]:
+    lengths: Final = tuple(len(_delta_text(chunk)) for chunk in chunks)
+    offsets: Final = tuple(accumulate(lengths, initial=0))[:-1]
+    return tuple(
+        _without_leading_text(chunk, min(length, count - offset))
+        for chunk, offset, length in zip(chunks, offsets, lengths, strict=True)
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class ContinuedPartial:
+    usage: Usage
+    response_cost: float
+    response_id: str
+
+
+class ContinuationOutput:
+    """Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the
+    prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing
+    whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and
+    runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A
+    continuation nested in another leaves the rules to the outermost one, the only one holding the whole text."""
+
+    def __init__(
+        self,
+        emitted_text: str,
+        prefill_message: Mapping[str, object],
+        model: str,
+        primary_response: object,
+        checks_post_call_rules: bool = True,
+    ) -> None:
+        self._text = emitted_text
+        self._checks_post_call_rules: Final = checks_post_call_rules
+        self._prefill_text: Final = _prefill_text(prefill_message)
+        self._model: Final = model
+        self._identity: Final = _stream_identity(primary_response)
+        self._rules: Final = Rules()
+        self._held: tuple[ModelResponseStream, ...] = ()
+        self._echo_settled = not self._prefill_text
+        self._seam = _resendable_whitespace(self._prefill_text)
+
+    async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]:
+        async for item in stream:
+            if not isinstance(item, ModelResponseStream):
+                yield item
+                continue
+            for released in self._observed(self._past_seam(self._released(item))):
+                yield self._in_primary_stream(released)
+        for released in self._observed(self._past_seam(self._flushed())):
+            yield self._in_primary_stream(released)
+
+    def _in_primary_stream(self, chunk: ModelResponseStream) -> ModelResponseStream:
+        return _without_role(chunk.model_copy(update=self._identity) if self._identity else chunk)
+
+    def _flushed(self) -> tuple[ModelResponseStream, ...]:
+        held: Final = self._held
+        self._held = ()
+        self._echo_settled = True
+        return held
+
+    def _released(self, chunk: ModelResponseStream) -> tuple[ModelResponseStream, ...]:
+        if self._echo_settled:
+            return (chunk,)
+        held: Final = (*self._held, chunk)
+        text: Final = "".join(_delta_text(held_chunk) for held_chunk in held)
+        if len(text) < len(self._prefill_text) and self._prefill_text.startswith(text):
+            self._held = held
+            return ()
+        self._held = ()
+        self._echo_settled = True
+        if not text.startswith(self._prefill_text):
+            return held
+        return _drop_leading_text(held, len(self._prefill_text))
+
+    def _past_seam(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]:
+        return tuple(self._without_seam(chunk) for chunk in chunks)
+
+    def _without_seam(self, chunk: ModelResponseStream) -> ModelResponseStream:
+        text: Final = _delta_text(chunk)
+        matched: Final = len(commonprefix((text, self._seam)))
+        self._seam = self._seam[matched:] if matched == len(text) else ""
+        return _without_leading_text(chunk, matched)
+
+    def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]:
+        if not self._checks_post_call_rules or not litellm.post_call_rules:
+            return chunks
+        for chunk in chunks:
+            self._text += _delta_text(chunk)
+            self._rules.post_call_rules(input=self._text, model=self._model)
+        return chunks

litellm/types/utils.py

@@ -4045,7 +4045,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
 
 DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
 
-DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_regex_lookaround"})
+DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
+    {"supports_regex_lookaround", "supports_assistant_prefill"}
+)
 
 SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
     frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)

litellm/utils.py

@@ -3060,6 +3060,20 @@ def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None
     )
 
 
+def supports_assistant_prefill(model: str, custom_llm_provider: str | None = None) -> bool:
+    """
+    Whether the model can continue a prefilled assistant message (Anthropic's
+    ``prefix: True`` trick and equivalents). Missing metadata reads as False,
+    so a mid-stream fallback continuation is only routed to a model known to
+    support it.
+    """
+    return _supports_factory(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        key="supports_assistant_prefill",
+    )
+
+
 def supports_thinking_cache_preservation(model: str, custom_llm_provider: str | None = None) -> bool:
     return supports_factory(
         model=model,
@@ -9933,6 +9947,10 @@ class ProviderConfigManager:
             return MinimaxVideoConfig()
         elif LlmProviders.EDENAI == provider:
             return litellm.EdenAIVideoConfig()
+        elif LlmProviders.VOLCENGINE == provider:
+            from litellm.llms.volcengine.videos.transformation import VolcEngineVideoConfig
+
+            return VolcEngineVideoConfig()
         return None
 
     @staticmethod

model_prices_and_context_window.json

@@ -2773,7 +2773,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2808,7 +2808,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2843,7 +2843,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2878,7 +2878,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2913,7 +2913,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2948,7 +2948,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -3743,7 +3743,7 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -15494,7 +15494,7 @@
         },
         "supports_adaptive_thinking": true,
         "supports_legacy_thinking": true,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -20832,7 +20832,7 @@
         "output_cost_per_token": 2.5000010000000002e-05,
         "output_dbu_cost_per_token": 0.000357143,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -21047,7 +21047,7 @@
         "output_cost_per_token": 1.5000020000000002e-05,
         "output_dbu_cost_per_token": 0.000214286,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -49176,7 +49176,7 @@
         "max_tokens": 64000,
         "mode": "chat",
         "output_cost_per_token": 2.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
@@ -51006,7 +51006,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -51179,7 +51179,7 @@
         "supported_regions": [
             "global"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -51200,7 +51200,7 @@
         "supported_regions": [
             "us-central1"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -58822,7 +58822,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -60467,6 +60467,61 @@
             }
         ]
     },
+    "volcengine/doubao-seedance-2-5-260628": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.216,
+        "output_cost_per_second_480p": 0.096075,
+        "output_cost_per_second_720p": 0.216,
+        "output_cost_per_second_1080p": 0.5346,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.141942857,
+        "output_cost_per_second_480p": 0.066003429,
+        "output_cost_per_second_720p": 0.141942857,
+        "output_cost_per_second_1080p": 0.354085714,
+        "output_cost_per_second_4k": 0.722057143,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-fast-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.114171429,
+        "output_cost_per_second_480p": 0.053089714,
+        "output_cost_per_second_720p": 0.114171429,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-mini-260615": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.070971429,
+        "output_cost_per_second_480p": 0.033001714,
+        "output_cost_per_second_720p": 0.070971429,
+        "supported_modalities": ["text", "image"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
     "bedrock/us-east-1/zai.glm-5": {
         "input_cost_per_token": 1e-06,
         "output_cost_per_token": 3.2e-06,
@@ -72334,7 +72389,7 @@
         "max_output_tokens": 64000,
         "max_tokens": 64000,
         "mode": "chat",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,

provider_endpoints_support.json

@@ -2766,7 +2766,8 @@
         "batches": false,
         "rerank": false,
         "a2a": true,
-        "interactions": true
+        "interactions": true,
+        "video_generations": true
       }
     },
     "voyage": {

tests/code_coverage_tests/router_code_coverage.py

@@ -114,6 +114,12 @@ ignored_function_names = [
     "_deployment_that_just_failed",  # Tested through the same-boundary hop test in test_router_order_fallback.py
     "_collect_ptu_share_teams",  # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
     "_resolve_team_ptu_ceiling",  # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
+    "_register_continuation_prefill_check",  # Tested through the continuation tests in test_router.py
+    "_set_mid_stream_fallback_continuation",  # Tested through the update_settings continuation test in test_router.py
+    "_is_flag_enabled",  # Tested through the string-flag continuation tests in test_router.py
+    "_params_sent_to_deployment",  # Tested through the configured-constraint continuation tests in test_router.py
+    "_names_deployment_that_cannot_continue",  # Tested through the deployment-id continuation tests in test_router.py
+    "_set_retry_policy",  # Tested through update_settings in test_router_retry_policy_update.py
     "_resolve_pre_call_check_input_tokens",  # Tested through the token-count failure pre-call check tests in test_router.py
     "_params_neither_dropped_nor_allowed",  # Tested through the allowed-or-dropped param pre-call check tests in test_router.py
     "_deployment_rejects_params",  # Tested through the unsupported-param pre-call check tests in test_router.py

tests/integration/spend/test_spend_log_caller_session.py

@@ -0,0 +1,783 @@
+import itertools
+import json
+import re
+import threading
+import uuid
+from urllib.parse import urlsplit
+from collections import deque
+from collections.abc import Iterator, Mapping, Sequence
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import ExitStack
+from dataclasses import dataclass, replace
+from datetime import datetime, timedelta, timezone
+from hashlib import sha256
+from pathlib import Path
+from queue import SimpleQueue
+from types import MappingProxyType
+from typing import Final
+
+import anthropic
+import httpx
+import openai
+import psutil
+import pytest
+import yaml
+from integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    Scenario,
+    eventually,
+    gateway_from_environment,
+    list_value,
+    object_value,
+    string_value,
+)
+from integration._support.budget import readiness_seconds
+from integration._support.database import read_rows
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue
+
+pytestmark: Final = pytest.mark.timeout(2 * readiness_seconds(2) + 2 * graceful_stop_seconds() + 240)
+
+MARKER: Final = re.compile(rb"callersess-[0-9a-f]{32}")
+CALLER_SESSION_KEY: Final = "litellm_caller_session_id"
+PATHS: Final = MappingProxyType(
+    {"chat": "/v1/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"}
+)
+ENDPOINT_STREAMS: Final = tuple(itertools.product(PATHS, (False, True)))
+Headers = Mapping[str, str] | Sequence[tuple[str, str]]
+
+
+def _marker() -> str:
+    return "callersess-" + uuid.uuid4().hex
+
+
+def _token(label: str) -> str:
+    return f"{label}-{uuid.uuid4().hex}"
+
+
+def _frame(payload: Mapping[str, JsonValue]) -> bytes:
+    return b"data: " + json.dumps(payload).encode() + b"\n\n"
+
+
+def _chat_reply(text: str, stream: bool) -> Reply:
+    identity: Final = "chatcmpl-" + uuid.uuid4().hex
+    usage: Final[dict[str, JsonValue]] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}
+    if not stream:
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": identity,
+                    "object": "chat.completion",
+                    "created": 1,
+                    "model": "gpt-4o-mini",
+                    "choices": [
+                        {"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}
+                    ],
+                    "usage": usage,
+                }
+            ).encode()
+        )
+    chunk: Final[dict[str, JsonValue]] = {
+        "id": identity,
+        "object": "chat.completion.chunk",
+        "created": 1,
+        "model": "gpt-4o-mini",
+    }
+    return Reply(
+        content_type="text/event-stream",
+        chunks=(
+            _frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}}]}),
+            _frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": usage}),
+            b"data: [DONE]\n\n",
+        ),
+    )
+
+
+def _upstream(request: Request) -> Reply:
+    markers: Final = tuple(found.group(0).decode() for found in MARKER.finditer(request.body))
+    if request.method != "POST" or not request.target.endswith("/chat/completions") or not markers:
+        return Reply(status=404, body=b'{"error":"unexpected request"}')
+    return _chat_reply(f"answer {markers[-1]}", JSON_OBJECT.validate_json(request.body).get("stream") is True)
+
+
+def _user_markers(body: Mapping[str, JsonValue]) -> tuple[str, ...]:
+    user_contents: Final = tuple(
+        json.dumps(object_value(message)["content"])
+        for message in list_value(body.get("messages", []))
+        if object_value(message)["role"] == "user"
+    )
+    return tuple(found.group(0).decode() for content in user_contents if (found := MARKER.search(content.encode())))
+
+
+def _spans(batch: Request) -> Iterator[dict[str, JsonValue]]:
+    for resource in list_value(JSON_OBJECT.validate_json(batch.body)["resourceSpans"]):
+        for scope in list_value(object_value(resource)["scopeSpans"]):
+            yield from (object_value(span) for span in list_value(object_value(scope)["spans"]))
+
+
+def _attributes(span: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
+    return {
+        string_value(object_value(attribute)["key"]): object_value(attribute)["value"]
+        for attribute in list_value(span.get("attributes", []))
+    }
+
+
+def _batch_attributes(batches: Sequence[Request]) -> Iterator[dict[str, JsonValue]]:
+    return (_attributes(span) for span in itertools.chain.from_iterable(_spans(batch) for batch in batches))
+
+
+@dataclass(frozen=True, slots=True)
+class Collector:
+    wire: Wire
+    accepted: deque[Request]
+    guard: threading.Lock
+
+    def span_attributes(self, response_id: str) -> tuple[dict[str, JsonValue], ...]:
+        with self.guard:
+            batches: Final = tuple(self.accepted)
+        attributes: Final = tuple(_batch_attributes(batches))
+        return tuple(
+            found
+            for found in attributes
+            if object_value(found.get("gen_ai.response.id", {})).get("stringValue") == response_id
+        )
+
+    def conversation_ids(self, conversations: frozenset[str]) -> tuple[str, ...]:
+        with self.guard:
+            batches: Final = tuple(self.accepted)
+        responded: Final = (
+            object_value(found.get("gen_ai.conversation.id", {})).get("stringValue")
+            for found in _batch_attributes(batches)
+            if "gen_ai.response.id" in found
+        )
+        return tuple(
+            conversation
+            for conversation in responded
+            if isinstance(conversation, str) and conversation in conversations
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    proxy: Gateway
+    scenario: Scenario
+    model: str
+    upstream: Wire
+    process: OwnedProxy
+
+    def body(self, endpoint: str, marker: str, extra: Mapping[str, JsonValue]) -> dict[str, JsonValue]:
+        match endpoint:
+            case "chat":
+                return {"model": self.model, "messages": [{"role": "user", "content": marker}], **extra}
+            case "messages":
+                return {
+                    "model": self.model,
+                    "max_tokens": 16,
+                    "messages": [{"role": "user", "content": marker}],
+                    **extra,
+                }
+            case _:
+                return {"model": self.model, "input": marker, **extra}
+
+    def post(
+        self,
+        endpoint: str,
+        marker: str,
+        fields: Mapping[str, JsonValue] = MappingProxyType({}),
+        *,
+        stream: bool = False,
+        headers: Headers = (),
+        key: str | None = None,
+    ) -> httpx.Response:
+        payload: Final = self.body(endpoint, marker, fields) | ({"stream": True} if stream else {})
+        request_headers: Final = (
+            ("Authorization", f"Bearer {self.proxy.key if key is None else key}"),
+            *(headers.items() if isinstance(headers, Mapping) else headers),
+        )
+        with self.proxy.client.stream("POST", PATHS[endpoint], json=payload, headers=request_headers) as response:
+            response.read()
+        return response
+
+    def send(
+        self,
+        endpoint: str,
+        marker: str,
+        fields: Mapping[str, JsonValue] = MappingProxyType({}),
+        *,
+        stream: bool = False,
+        headers: Headers = (),
+        key: str | None = None,
+    ) -> httpx.Response:
+        response: Final = self.post(endpoint, marker, fields, stream=stream, headers=headers, key=key)
+        assert response.status_code == 200, response.text
+        return response
+
+    def upstream_bodies(self, marker: str) -> tuple[dict[str, JsonValue], ...]:
+        bodies: Final = tuple(JSON_OBJECT.validate_json(request.body) for request in self.upstream.drain())
+        return tuple(body for body in bodies if _user_markers(body)[-1:] == (marker,))
+
+
+@dataclass(frozen=True, slots=True)
+class Rigs:
+    flag_off: Rig
+    flag_on: Rig
+    sink: Collector
+
+
+def _spend_rows(call_id: str) -> list[dict[str, JsonValue]]:
+    return read_rows(
+        'SELECT session_id, request_id, metadata::text AS metadata_text FROM "LiteLLM_SpendLogs" '
+        "WHERE litellm_call_id=%s",
+        (call_id,),
+    )
+
+
+def _spend_row(response: httpx.Response) -> dict[str, JsonValue]:
+    call_id: Final = response.headers["x-litellm-call-id"]
+    rows: Final = eventually(lambda: _spend_rows(call_id), lambda values: len(values) >= 1, seconds=70)
+    assert len(rows) == 1, rows
+    return rows[0]
+
+
+def _session(response: httpx.Response) -> JsonValue:
+    return _spend_row(response)["session_id"]
+
+
+def _model(scenario: Scenario, upstream: Wire) -> str:
+    return scenario.model(
+        model="hosted_vllm/gpt-4o-mini",
+        api_base=f"{upstream.url}/v1",
+        use_chat_completions_api=True,
+        input_cost_per_token=0.001,
+        output_cost_per_token=0.002,
+    )
+
+
+def _rig_config(
+    directory: Path, name: str, litellm_settings: Mapping[str, JsonValue], extra: Mapping[str, JsonValue]
+) -> Path:
+    shared: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final = {
+        **shared,
+        "general_settings": {**object_value(shared["general_settings"]), "store_prompts_in_spend_logs": True},
+        "litellm_settings": {**object_value(shared["litellm_settings"]), **litellm_settings},
+        **extra,
+    }
+    path: Final = directory / f"{name}.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@pytest.fixture(scope="module")
+def rigs(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rigs]:
+    accepted: Final[deque[Request]] = deque()  # mutable-ok: sink thread appends each accepted OTLP batch
+    guard: Final = threading.Lock()
+
+    def sink(request: Request) -> Reply:
+        with guard:
+            accepted.append(request)
+        return Reply()
+
+    directory: Final = tmp_path_factory.mktemp("caller-session")
+    with ExitStack() as stack:
+        upstream: Final = stack.enter_context(wire_server(_upstream))
+        collector: Final = Collector(stack.enter_context(wire_server(sink)), accepted, guard)
+        shared: Final = stack.enter_context(gateway_from_environment())
+        flag_off: Final = stack.enter_context(
+            owned_proxy_process(shared, directory, {}, config=_rig_config(directory, "flag-off", {}, {}), workers=2)
+        )
+        flag_on: Final = stack.enter_context(
+            owned_proxy_process(
+                shared,
+                directory,
+                {"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"},
+                config=_rig_config(
+                    directory,
+                    "flag-on",
+                    {"request_correlation_in_logs": True, "callbacks": ["otel"]},
+                    {
+                        "callback_settings": {
+                            "otel": {"exporter": "http/json", "endpoint": collector.wire.url, "mapper_names": ["genai"]}
+                        }
+                    },
+                ),
+                workers=2,
+            )
+        )
+        off_scenario: Final = stack.enter_context(flag_off.gateway.scenario())
+        on_scenario: Final = stack.enter_context(flag_on.gateway.scenario())
+        yield Rigs(
+            Rig(flag_off.gateway, off_scenario, _model(off_scenario, upstream), upstream, flag_off),
+            Rig(flag_on.gateway, on_scenario, _model(on_scenario, upstream), upstream, flag_on),
+            collector,
+        )
+
+
+@pytest.mark.parametrize(("endpoint", "stream"), ENDPOINT_STREAMS)
+def test_spend_log_session_is_the_body_session_beside_a_trace_header(rigs: Rigs, endpoint: str, stream: bool) -> None:
+    marker: Final = _marker()
+    session: Final = _token("convo")
+    response: Final = rigs.flag_on.send(
+        endpoint,
+        marker,
+        stream=stream,
+        headers={"x-litellm-trace-id": _token("trace")},
+        fields={"litellm_session_id": session},
+    )
+
+    row: Final = _spend_row(response)
+    upstream: Final = rigs.flag_on.upstream_bodies(marker)
+
+    assert row["session_id"] == session, row
+    assert CALLER_SESSION_KEY not in string_value(row["metadata_text"]), row
+    assert len(upstream) == 1, upstream
+    assert CALLER_SESSION_KEY not in json.dumps(upstream[0]), upstream
+
+
+def test_sdk_clients_record_the_body_session_beside_a_trace_header(rigs: Rigs) -> None:
+    rig: Final = rigs.flag_on
+    base_url: Final = str(rig.proxy.client.base_url)
+    traces: Final = tuple(_token("trace") for _ in range(3))
+    sessions: Final = tuple(_token("convo") for _ in range(3))
+    markers: Final = tuple(_marker() for _ in range(3))
+    chat_client: Final = openai.OpenAI(base_url=base_url + "/v1", api_key=rig.proxy.key, max_retries=0)
+    messages_client: Final = anthropic.Anthropic(base_url=base_url, api_key=rig.proxy.key, max_retries=0)
+
+    with chat_client.chat.completions.with_streaming_response.create(
+        model=rig.model,
+        messages=[{"role": "user", "content": markers[0]}],
+        stream=True,
+        extra_headers={"x-litellm-trace-id": traces[0]},
+        extra_body={"litellm_session_id": sessions[0]},
+    ) as chat_stream:
+        chat_call: Final = chat_stream.headers["x-litellm-call-id"]
+        chat_text: Final = "".join(
+            chunk.choices[0].delta.content or "" for chunk in chat_stream.parse() if chunk.choices
+        )
+    message: Final = messages_client.messages.with_raw_response.create(
+        model=rig.model,
+        max_tokens=16,
+        messages=[{"role": "user", "content": markers[1]}],
+        extra_headers={"x-litellm-trace-id": traces[1]},
+        extra_body={"litellm_session_id": sessions[1]},
+    )
+    with chat_client.responses.with_streaming_response.create(
+        model=rig.model,
+        input=markers[2],
+        stream=True,
+        extra_headers={"x-litellm-trace-id": traces[2]},
+        extra_body={"litellm_session_id": sessions[2]},
+    ) as responses_stream:
+        responses_call: Final = responses_stream.headers["x-litellm-call-id"]
+        completed: Final = tuple(event for event in responses_stream.parse() if event.type == "response.completed")
+
+    first_block: Final = message.parse().content[0]
+    assert chat_text == f"answer {markers[0]}", chat_text
+    assert isinstance(first_block, anthropic.types.TextBlock) and first_block.text == f"answer {markers[1]}", message
+    assert len(completed) == 1, completed
+    calls: Final = (chat_call, message.headers["x-litellm-call-id"], responses_call)
+    landed: Final = tuple(
+        eventually(lambda call=call: _spend_rows(call), lambda values: len(values) == 1, seconds=70)[0]["session_id"]
+        for call in calls
+    )
+    assert landed == sessions, (landed, calls)
+
+
+def test_conversation_span_keeps_the_trace_header_and_no_internal_key(rigs: Rigs) -> None:
+    trace: Final = _token("trace")
+    response: Final = rigs.flag_on.send(
+        "chat", _marker(), headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": _token("convo")}
+    )
+    response_id: Final = string_value(JSON_OBJECT.validate_json(response.content)["id"])
+
+    spans: Final = eventually(lambda: rigs.sink.span_attributes(response_id), lambda found: len(found) == 1, seconds=30)
+
+    assert spans[0]["gen_ai.conversation.id"] == {"stringValue": trace}, spans
+    assert CALLER_SESSION_KEY not in json.dumps(spans[0]), spans
+
+
+def test_a_continued_chains_conversation_spans_keep_each_turns_trace_header(rigs: Rigs) -> None:
+    traces: Final = (_token("trace"), _token("trace"))
+    first: Final = _turn(rigs.flag_on, _marker(), headers={"x-litellm-trace-id": traces[0]})
+    _turn(
+        rigs.flag_on,
+        _marker(),
+        first,
+        headers={"x-litellm-trace-id": traces[1]},
+        fields={"litellm_session_id": _token("convo")},
+    )
+
+    conversations: Final = eventually(
+        lambda: rigs.sink.conversation_ids(frozenset(traces)), lambda found: len(found) == 2, seconds=30
+    )
+
+    assert sorted(conversations) == sorted(traces), conversations
+
+
+@pytest.mark.parametrize("endpoint", list(PATHS))
+def test_flag_off_keeps_the_trace_id_beside_a_body_session(rigs: Rigs, endpoint: str) -> None:
+    trace: Final = _token("trace")
+
+    response: Final = rigs.flag_off.send(
+        endpoint, _marker(), headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": _token("convo")}
+    )
+
+    assert _session(response) == trace
+
+
+@pytest.mark.parametrize("endpoint", list(PATHS))
+def test_response_cache_hit_records_the_body_session_beside_a_trace_header(rigs: Rigs, endpoint: str) -> None:
+    marker: Final = _marker()
+    sessions: Final = (_token("convo"), _token("convo"))
+    responses: Final = tuple(
+        rigs.flag_on.send(
+            endpoint, marker, headers={"x-litellm-trace-id": _token("trace")}, fields={"litellm_session_id": session}
+        )
+        for session in sessions
+    )
+
+    assert tuple(_session(response) for response in responses) == sessions
+    assert len(rigs.flag_on.upstream_bodies(marker)) == 1
+
+
+def test_metadata_session_beside_a_new_trace_per_turn_groups_both_turns(rigs: Rigs) -> None:
+    session: Final = _token("convo")
+    responses: Final = tuple(
+        rigs.flag_on.send(
+            "chat",
+            _marker(),
+            headers={"x-litellm-trace-id": _token("trace")},
+            fields={"metadata": {"session_id": session}},
+        )
+        for _ in range(2)
+    )
+
+    assert tuple(_session(response) for response in responses) == (session, session)
+    now: Final = datetime.now(timezone.utc)
+    page: Final = rigs.flag_on.proxy.get(
+        "/spend/logs/ui",
+        params={
+            "session_id": session,
+            "start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"),
+            "end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"),
+        },
+    )
+    listed: Final = sorted(string_value(object_value(row)["request_id"]) for row in list_value(page["data"]))
+    assert listed == sorted(
+        string_value(JSON_OBJECT.validate_json(response.content)["id"]) for response in responses
+    ), page
+
+
+def _shape(shape: str, trace: str, session: str, header: str) -> tuple[Headers, dict[str, JsonValue], str]:
+    traced: Final = {"x-litellm-trace-id": trace}
+    match shape:
+        case "session-header-wins":
+            return {"x-litellm-session-id": header}, {"litellm_session_id": session}, header
+        case "trace-header-only":
+            return traced, {}, trace
+        case "body-session-only":
+            return {}, {"litellm_session_id": session}, session
+        case "int-session":
+            return traced, {"litellm_session_id": 7}, trace
+        case "list-session":
+            return traced, {"litellm_session_id": [session]}, trace
+        case "empty-session":
+            return traced, {"litellm_session_id": ""}, trace
+        case "null-session":
+            return traced, {"litellm_session_id": None}, trace
+        case "nul-session":
+            return traced, {"litellm_session_id": session + "\x00x"}, trace
+        case "two-trace-headers":
+            return (
+                (("x-litellm-trace-id", trace), ("x-litellm-trace-id", header)),
+                {"litellm_session_id": session},
+                session,
+            )
+        case _:
+            return traced, {"litellm_session_id": session + "s" * 5120}, session + "s" * 5120
+
+
+@pytest.mark.parametrize(
+    "shape",
+    [
+        "session-header-wins",
+        "trace-header-only",
+        "body-session-only",
+        "int-session",
+        "list-session",
+        "empty-session",
+        "null-session",
+        "nul-session",
+        "two-trace-headers",
+        "5kb-session",
+    ],
+)
+def test_spend_log_session_for_each_session_shape(rigs: Rigs, shape: str) -> None:
+    headers, fields, expected = _shape(shape, _token("trace"), _token("convo"), _token("header"))
+    neighbor_trace: Final = _token("neighbor")
+
+    response: Final = rigs.flag_on.send("chat", _marker(), fields, headers=headers)
+    neighbor: Final = rigs.flag_on.send("chat", _marker(), headers={"x-litellm-trace-id": neighbor_trace})
+
+    assert _session(response) == expected
+    assert _session(neighbor) == neighbor_trace
+
+
+@pytest.mark.parametrize("endpoint", ["chat", "messages"])
+def test_client_sent_internal_session_key_is_ignored(rigs: Rigs, endpoint: str) -> None:
+    forged: Final = _token("forged")
+    trace: Final = _token("trace")
+    forged_metadata: Final[dict[str, JsonValue]] = {
+        "metadata" if endpoint == "chat" else "litellm_metadata": {CALLER_SESSION_KEY: forged}
+    }
+
+    untraced: Final = rigs.flag_on.send(endpoint, _marker(), forged_metadata)
+    traced: Final = rigs.flag_on.send(endpoint, _marker(), forged_metadata, headers={"x-litellm-trace-id": trace})
+
+    assert _session(untraced) not in (forged, None)
+    assert _session(traced) == trace
+
+
+def test_identical_requests_each_log_once_with_the_body_session(rigs: Rigs) -> None:
+    session: Final = _token("convo")
+    trace: Final = _token("trace")
+    marker: Final = _marker()
+
+    responses: Final = tuple(
+        rigs.flag_on.send("chat", marker, headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": session})
+        for _ in range(3)
+    )
+
+    assert len({response.headers["x-litellm-call-id"] for response in responses}) == 3
+    assert tuple(_session(response) for response in responses) == (session, session, session)
+
+
+def _pass_through_route(rig: Rig, target: str) -> str:
+    path: Final = f"/caller-session-{uuid.uuid4().hex}"
+    created: Final = rig.proxy.post("/config/pass_through_endpoint", {"path": path, "target": target, "auth": True})
+    endpoint_id: Final = string_value(object_value(list_value(created["endpoints"])[0])["id"])
+    rig.scenario.cleanups.callback(
+        lambda: rig.proxy.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": endpoint_id})
+    )
+    return path
+
+
+@pytest.mark.parametrize("field", ["litellm_session_id", "metadata"])
+def test_pass_through_spend_log_session_is_the_body_session_beside_a_trace_header(rigs: Rigs, field: str) -> None:
+    marker: Final = _marker()
+    session: Final = _token("convo")
+    rig: Final = rigs.flag_on
+    path: Final = _pass_through_route(rig, f"{rig.upstream.url}/v1/chat/completions")
+    key: Final = rig.scenario.key(allowed_passthrough_routes=[path])
+    fields: Final[dict[str, JsonValue]] = (
+        {"litellm_session_id": session} if field == "litellm_session_id" else {"metadata": {"session_id": session}}
+    )
+
+    response: Final = rig.proxy.request(
+        "POST",
+        path,
+        rig.body("chat", marker, fields),
+        key=key,
+        headers={"x-litellm-trace-id": _token("trace")},
+    )
+
+    rows: Final = eventually(
+        lambda: read_rows(
+            'SELECT session_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (sha256(key.encode()).hexdigest(),)
+        ),
+        lambda found: len(found) >= 1,
+        seconds=70,
+    )
+    assert response.status_code == 200, response.text
+    assert f"answer {marker}" in response.text
+    assert len(rig.upstream_bodies(marker)) == 1
+    assert rows == [{"session_id": session}]
+
+
+def _turn(
+    rig: Rig,
+    marker: str,
+    previous: str | None = None,
+    *,
+    key: str | None = None,
+    headers: Headers = (),
+    fields: Mapping[str, JsonValue] = MappingProxyType({}),
+) -> str:
+    chained: Final[dict[str, JsonValue]] = {"previous_response_id": previous} if previous else {}
+    response: Final = rig.send("responses", marker, {**chained, **fields}, headers=headers, key=key)
+    _spend_row(response)
+    return string_value(JSON_OBJECT.validate_json(response.content)["id"])
+
+
+def _turn_fields(shape: str, session: str) -> tuple[dict[str, JsonValue], dict[str, JsonValue]]:
+    match shape:
+        case "session-every-turn":
+            return {"litellm_session_id": session}, {"litellm_session_id": session}
+        case "session-from-turn-2":
+            return {}, {"litellm_session_id": session}
+        case _:
+            return {"litellm_session_id": _token("convo")}, {"litellm_session_id": _token("convo")}
+
+
+@pytest.mark.parametrize(
+    ("correlated", "shape", "replays_turn_1"),
+    [
+        (True, "session-every-turn", True),
+        (True, "session-from-turn-2", True),
+        (True, "new-session-per-turn", True),
+        (False, "session-from-turn-2", False),
+    ],
+)
+def test_bridged_chain_history(rigs: Rigs, correlated: bool, shape: str, replays_turn_1: bool) -> None:
+    rig: Final = rigs.flag_on if correlated else rigs.flag_off
+    first_fields, second_fields = _turn_fields(shape, _token("convo"))
+    markers: Final = (_marker(), _marker(), _marker())
+    first: Final = _turn(rig, markers[0], headers={"x-litellm-trace-id": _token("trace")}, fields=first_fields)
+    second: Final = _turn(rig, markers[1], first, headers={"x-litellm-trace-id": _token("trace")}, fields=second_fields)
+
+    _turn(rig, markers[2], second)
+
+    replayed: Final = rig.upstream_bodies(markers[2])
+    assert len(replayed) == 1, replayed
+    assert _user_markers(replayed[0]) == (markers if replays_turn_1 else markers[1:])
+
+
+def test_bridged_chains_sharing_a_metadata_session_replay_each_others_turns(rigs: Rigs) -> None:
+    shared: Final[dict[str, JsonValue]] = {"metadata": {"session_id": _token("convo")}}
+    markers: Final = (_marker(), _marker(), _marker())
+    first: Final = _turn(rigs.flag_on, markers[0], fields=shared)
+    _turn(rigs.flag_on, markers[1], fields=shared)
+
+    _turn(rigs.flag_on, markers[2], first)
+
+    replayed: Final = rigs.flag_on.upstream_bodies(markers[2])
+    assert len(replayed) == 1, replayed
+    assert _user_markers(replayed[0]) == markers
+
+
+@dataclass(frozen=True, slots=True)
+class Owners:
+    first: str
+    second: str
+
+
+def _owners(scenario: Scenario, model: str, shape: str) -> Owners:
+    user: Final = _token("user")
+    match shape:
+        case "other-user":
+            return Owners(
+                scenario.key(user_id=user, models=[model]), scenario.key(user_id=_token("user"), models=[model])
+            )
+        case "same-user":
+            return Owners(scenario.key(user_id=user, models=[model]), scenario.key(user_id=user, models=[model]))
+        case "same-team":
+            team: Final = scenario.team()
+            return Owners(scenario.key(team_id=team, models=[model]), scenario.key(team_id=team, models=[model]))
+        case _:
+            return Owners(scenario.key(models=[model]), scenario.key(models=[model]))
+
+
+@pytest.mark.parametrize(
+    ("shape", "replays_first_owner"),
+    [("other-user", False), ("same-user", True), ("same-team", True), ("ownerless-keys", False)],
+)
+def test_bridged_history_follows_the_previous_responses_owner(
+    rigs: Rigs, shape: str, replays_first_owner: bool
+) -> None:
+    rig: Final = rigs.flag_off
+    owners: Final = _owners(rig.scenario, rig.model, shape)
+    session: Final = _token("convo")
+    markers: Final = (_marker(), _marker(), _marker())
+    _turn(rig, markers[0], key=owners.first, fields={"litellm_session_id": session})
+    second: Final = _turn(rig, markers[1], key=owners.second, fields={"litellm_session_id": session})
+
+    _turn(rig, markers[2], second, key=owners.second)
+
+    replayed: Final = rig.upstream_bodies(markers[2])
+    assert len(replayed) == 1, replayed
+    assert _user_markers(replayed[0]) == (markers if replays_first_owner else markers[1:])
+
+
+BURST: Final = ENDPOINT_STREAMS * 4
+
+
+def test_concurrent_burst_across_endpoints_logs_each_session_once(rigs: Rigs) -> None:
+    sessions: Final = tuple(_token("convo") for _ in BURST)
+
+    def one(index: int) -> httpx.Response:
+        endpoint, stream = BURST[index]
+        return rigs.flag_on.send(
+            endpoint,
+            _marker(),
+            stream=stream,
+            headers={"x-litellm-trace-id": _token("trace")},
+            fields={"litellm_session_id": sessions[index]},
+        )
+
+    with ThreadPoolExecutor(max_workers=12) as pool:
+        responses: Final = tuple(pool.map(one, range(len(BURST))))
+
+    assert tuple(_session(response) for response in responses) == sessions
+
+
+def _open_upstream_connections(pid: int, upstream: Wire) -> int:
+    port: Final = urlsplit(upstream.url).port
+    return sum(
+        1
+        for connection in psutil.Process(pid).net_connections(kind="tcp")
+        if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
+    )
+
+
+def _post_or_none(rig: Rig, session: str) -> httpx.Response | None:
+    try:
+        return rig.post(
+            "chat", _marker(), headers={"x-litellm-trace-id": _token("trace")}, fields={"litellm_session_id": session}
+        )
+    except httpx.HTTPError:
+        return None
+
+
+def test_killing_one_worker_mid_burst_keeps_logging_the_body_session(rigs: Rigs) -> None:
+    release: Final = threading.Event()
+    arrived: Final[SimpleQueue[bytes]] = SimpleQueue()
+
+    def held(request: Request) -> Reply:
+        arrived.put(request.body)
+        assert release.wait(timeout=60), "the burst was never released"
+        return _upstream(request)
+
+    process: Final = rigs.flag_on.process
+    sessions: Final = tuple(_token("convo") for _ in range(20))
+    after_session: Final = _token("convo")
+    with wire_server(held) as wire:
+        rig: Final = replace(rigs.flag_on, model=_model(rigs.flag_on.scenario, wire), upstream=wire)
+        workers: Final = eventually(process.worker_pids, lambda found: len(found) == 2, seconds=30)
+        with ThreadPoolExecutor(max_workers=len(sessions)) as pool:
+            pending: Final = tuple(pool.submit(_post_or_none, rig, session) for session in sessions)
+            eventually(arrived.qsize, lambda size: size == len(sessions), seconds=60)
+            held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire) for pid in workers})
+            victim, survivor = sorted(workers, key=held_by.__getitem__)
+            with process.replacing_worker(victim) as kill:
+                kill.freeze()
+                kill()
+                release.set()
+                results: Final = tuple(future.result() for future in pending)
+                after: Final = rig.send(
+                    "chat",
+                    _marker(),
+                    headers={"x-litellm-trace-id": _token("trace")},
+                    fields={"litellm_session_id": after_session},
+                )
+
+    served: Final = tuple(
+        (response, session)
+        for response, session in zip(results, sessions, strict=True)
+        if response is not None and response.status_code == 200
+    )
+    assert sum(held_by.values()) == len(sessions), held_by
+    assert len(served) == held_by[survivor] >= len(sessions) // 2, (held_by, len(served))
+    assert _session(after) == after_session
+    assert tuple(_session(response) for response, _ in served) == tuple(session for _, session in served)

tests/integration/streaming/test_mid_stream_fallback_continuation.py

@@ -0,0 +1,749 @@
+from __future__ import annotations
+
+import functools
+import hashlib
+import itertools
+import json
+import math
+import operator
+import threading
+import uuid
+from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence
+from contextlib import ExitStack, contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+import openai
+import pytest
+import yaml
+from integration._support.client import JSON_OBJECT, Gateway, eventually, gateway_from_environment, object_value
+from integration._support.database import read_rows, scratch_database
+from integration._support.process import graceful_stop_seconds, owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+from openai.types.chat import ChatCompletionChunk
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(4 * graceful_stop_seconds() + 120)
+
+SETTING: Final = "enable_mid_stream_fallback_continuation"
+PARTIAL: Final = ("Alpha beta", " gamma")
+JOINED_PARTIAL: Final = "".join(PARTIAL)
+RELAY_TEXT: Final = " delta"
+BACKUP_TEXT: Final = " omega."
+PLAIN_TEXT: Final = "plain answer"
+TOOL_PREAMBLE: Final = "Let me check"
+BACKUP_USAGE: Final = (40, 6)
+MAX_TOKENS: Final = 200
+PRIMARY_PRICES: Final = (0.001, 0.002)
+BACKUP_PRICES: Final = (0.003, 0.005)
+SHARED_BACKEND: Final = "openai/gpt-4o-mini"
+SEPARATE_BACKEND: Final = "openai/gpt-4o"
+WEATHER_TOOL: Final[Mapping[str, JsonValue]] = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
+    },
+}
+TOOL_CALL_DELTA: Final[Mapping[str, JsonValue]] = {
+    "index": 0,
+    "id": "call_1",
+    "type": "function",
+    "function": {"name": "get_weather", "arguments": ""},
+}
+NOTHING_SEEN: Final[tuple[ChatCompletionChunk, ...]] = ()
+ALLOW_EVERYTHING: Final = "def apply_guardrail(inputs, request_data, input_type):\n    return allow()\n"
+EVENT_BATCH: Final = TypeAdapter(list[dict[str, JsonValue]])
+
+
+@functools.cache
+def cut_gate(marker: str, hop: str, attempt: int) -> threading.Event:
+    return threading.Event()
+
+
+@functools.cache
+def attempts(marker: str, hop: str) -> Iterator[int]:
+    return itertools.count()
+
+
+def frame(identity: str, delta: Mapping[str, JsonValue], finish: str | None = None) -> bytes:
+    body: Final = {
+        "id": identity,
+        "object": "chat.completion.chunk",
+        "created": 1700000000,
+        "model": "gpt-4o-mini",
+        "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish}],
+    }
+    return b"data: " + json.dumps(body).encode() + b"\n\n"
+
+
+def complete_stream(identity: str, text: str, usage: tuple[int, int]) -> Reply:
+    usage_frame: Final[dict[str, JsonValue]] = {
+        "id": identity,
+        "object": "chat.completion.chunk",
+        "created": 1700000000,
+        "model": "gpt-4o-mini",
+        "choices": [],
+        "usage": {"prompt_tokens": usage[0], "completion_tokens": usage[1], "total_tokens": sum(usage)},
+    }
+    return Reply(
+        content_type="text/event-stream",
+        chunks=(
+            frame(identity, {"role": "assistant", "content": text}),
+            frame(identity, {}, finish="stop"),
+            b"data: " + json.dumps(usage_frame).encode() + b"\n\n",
+            b"data: [DONE]\n\n",
+        ),
+    )
+
+
+def cut_after_caller_sees(marker: str, hop: str, *frames: bytes) -> Reply:
+    return Reply(
+        content_type="text/event-stream",
+        chunks=(b"".join(frames), b"data: never sent\n\n"),
+        abort_after=1,
+        gate_after_first=cut_gate(marker, hop, next(attempts(marker, hop))),
+    )
+
+
+def first_user_text(request: Request) -> str:
+    messages: Final = JSON_OBJECT.validate_json(request.body)["messages"]
+    assert isinstance(messages, list), messages
+    first: Final = object_value(messages[0])["content"]
+    assert isinstance(first, str), first
+    return first
+
+
+def marker_of(request: Request) -> str:
+    return first_user_text(request).split()[-1]
+
+
+def is_chat(request: Request) -> bool:
+    return request.target.endswith("/chat/completions")
+
+
+def model_list() -> Reply:
+    return Reply(body=b'{"object": "list", "data": []}')
+
+
+def primary(request: Request) -> Reply:
+    if not is_chat(request):
+        return model_list()
+    scenario: Final = first_user_text(request).split()[0]
+    marker: Final = marker_of(request)
+    identity: Final = f"chatcmpl-{marker}-primary"
+    if scenario == "reject":
+        return Reply(status=500, body=b'{"error": {"message": "primary is down", "type": "server_error"}}')
+    if scenario == "tool-cut":
+        return cut_after_caller_sees(
+            marker,
+            "primary",
+            frame(identity, {"role": "assistant", "content": TOOL_PREAMBLE}),
+            frame(identity, {"tool_calls": [dict(TOOL_CALL_DELTA)]}),
+        )
+    return cut_after_caller_sees(
+        marker,
+        "primary",
+        frame(identity, {"role": "assistant", "content": PARTIAL[0]}),
+        frame(identity, {"content": PARTIAL[1]}),
+    )
+
+
+def relay(request: Request) -> Reply:
+    if not is_chat(request):
+        return model_list()
+    marker: Final = marker_of(request)
+    return cut_after_caller_sees(
+        marker, "relay", frame(f"chatcmpl-{marker}-relay", {"role": "assistant", "content": RELAY_TEXT})
+    )
+
+
+def backup(request: Request) -> Reply:
+    if not is_chat(request):
+        return model_list()
+    return complete_stream(f"chatcmpl-{marker_of(request)}-backup", BACKUP_TEXT, BACKUP_USAGE)
+
+
+def no_prefill(request: Request) -> Reply:
+    if not is_chat(request):
+        return model_list()
+    return complete_stream(f"chatcmpl-{marker_of(request)}-plain", PLAIN_TEXT, (9, 2))
+
+
+def accept_events(request: Request) -> Reply:
+    return Reply()
+
+
+def registered_guardrail(name: str, mode: str) -> dict[str, JsonValue]:
+    return {
+        "guardrail_name": name,
+        "litellm_params": {
+            "guardrail": "custom_code",
+            "mode": mode,
+            "default_on": False,
+            "custom_code": ALLOW_EVERYTHING,
+        },
+    }
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    gateway: Gateway
+    database_url: str
+    primary: Wire
+    relay: Wire
+    backup: Wire
+    no_prefill: Wire
+    events: Wire
+
+
+def joined_text(chunks: Iterable[ChatCompletionChunk]) -> str:
+    return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
+
+
+def has_tool_call(chunks: Iterable[ChatCompletionChunk]) -> bool:
+    return any(choice.delta.tool_calls for chunk in chunks for choice in chunk.choices)
+
+
+@dataclass(frozen=True, slots=True)
+class Release:
+    gate: threading.Event
+    ready: Callable[[tuple[ChatCompletionChunk, ...]], bool]
+
+
+def after_text(marker: str, hop: str, text: str, attempt: int = 0) -> Release:
+    return Release(cut_gate(marker, hop, attempt), lambda seen: joined_text(seen) == text)
+
+
+@dataclass(frozen=True, slots=True)
+class Streamed:
+    chunks: tuple[ChatCompletionChunk, ...]
+    error: openai.APIError | None
+
+    @property
+    def text(self) -> str:
+        return joined_text(self.chunks)
+
+    @property
+    def ids(self) -> frozenset[str]:
+        return frozenset(chunk.id for chunk in self.chunks)
+
+    @property
+    def role_deltas(self) -> int:
+        return sum(1 for chunk in self.chunks for choice in chunk.choices if choice.delta.role is not None)
+
+    @property
+    def finish_reasons(self) -> tuple[str, ...]:
+        return tuple(choice.finish_reason for chunk in self.chunks for choice in chunk.choices if choice.finish_reason)
+
+    @property
+    def usage(self) -> tuple[int, int]:
+        reported: Final = tuple(chunk.usage for chunk in self.chunks if chunk.usage is not None)
+        assert reported, self.chunks
+        return (reported[-1].prompt_tokens, reported[-1].completion_tokens)
+
+
+def deployment(
+    name: str,
+    wire: Wire,
+    prices: tuple[float, float],
+    *,
+    backend: str = SHARED_BACKEND,
+    prefill: bool | None = None,
+) -> dict[str, JsonValue]:
+    return {
+        "model_name": name,
+        "litellm_params": {
+            "model": backend,
+            "api_base": wire.url + "/v1",
+            "api_key": "synthetic-continuation-key",
+            "input_cost_per_token": prices[0],
+            "output_cost_per_token": prices[1],
+        },
+        **({"model_info": {"supports_assistant_prefill": prefill}} if prefill is not None else {}),
+    }
+
+
+def write_config(
+    directory: Path,
+    model_entries: Sequence[Mapping[str, JsonValue]],
+    fallbacks: Sequence[Mapping[str, JsonValue]],
+    router_settings: Mapping[str, JsonValue],
+    guardrails: Sequence[Mapping[str, JsonValue]],
+) -> Path:
+    base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final = {
+        **base,
+        "model_list": [dict(entry) for entry in model_entries],
+        "litellm_settings": {
+            **object_value(base["litellm_settings"]),
+            "callbacks": ["generic_api"],
+            "DEFAULT_FLUSH_INTERVAL_SECONDS": 1,
+        },
+        "guardrails": [dict(entry) for entry in guardrails],
+        "router_settings": {
+            **object_value(base["router_settings"]),
+            "num_retries": 0,
+            "fallbacks": [dict(entry) for entry in fallbacks],
+            **router_settings,
+        },
+    }
+    path: Final = directory / f"continuation-{uuid.uuid4().hex[:8]}.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@contextmanager
+def booted_rig(
+    directory: Path,
+    fallbacks: Sequence[Mapping[str, JsonValue]],
+    router_settings: Mapping[str, JsonValue],
+    workers: int,
+    guardrails: Sequence[Mapping[str, JsonValue]] = (),
+) -> Generator[Rig]:
+    with ExitStack() as stack:
+        shared: Final = stack.enter_context(gateway_from_environment())
+        database_url: Final = stack.enter_context(scratch_database())
+        wires: Final = tuple(
+            stack.enter_context(wire_server(respond)) for respond in (primary, relay, backup, no_prefill, accept_events)
+        )
+        primary_wire, relay_wire, backup_wire, no_prefill_wire, events_wire = wires
+        config: Final = write_config(
+            directory,
+            (
+                *(
+                    deployment(name, primary_wire, PRIMARY_PRICES)
+                    for name in ("primary", "primary-skip", "primary-none", "primary-relay", "primary-shared")
+                ),
+                deployment("relay", relay_wire, PRIMARY_PRICES, prefill=True),
+                deployment("backup", backup_wire, BACKUP_PRICES, prefill=True),
+                deployment("no-prefill", no_prefill_wire, PRIMARY_PRICES, backend=SEPARATE_BACKEND),
+                deployment("no-prefill-shared", no_prefill_wire, PRIMARY_PRICES),
+            ),
+            fallbacks,
+            router_settings,
+            guardrails,
+        )
+        gateway: Final = stack.enter_context(
+            owned_proxy(
+                shared,
+                directory,
+                {"DATABASE_URL": database_url, "GENERIC_LOGGER_ENDPOINT": events_wire.url},
+                config=config,
+                remove_environment=("DATABASE_URL_READ_REPLICA",),
+                workers=workers,
+            )
+        )
+        yield Rig(gateway, database_url, primary_wire, relay_wire, backup_wire, no_prefill_wire, events_wire)
+
+
+@pytest.fixture(scope="module")
+def configured(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    fallbacks: Final[tuple[Mapping[str, JsonValue], ...]] = (
+        {"primary": ["backup"]},
+        {"primary-skip": ["no-prefill", "backup"]},
+        {"primary-none": ["no-prefill"]},
+        {"primary-relay": ["relay"]},
+        {"relay": ["backup"]},
+        {"primary-shared": ["no-prefill-shared"]},
+    )
+    with booted_rig(
+        tmp_path_factory.mktemp("continuation-configured"),
+        fallbacks,
+        {SETTING: True},
+        2,
+        (registered_guardrail("input-check", "pre_call"),),
+    ) as rig:
+        yield rig
+
+
+@pytest.fixture(scope="module")
+def output_guarded(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    with booted_rig(
+        tmp_path_factory.mktemp("continuation-output-guarded"),
+        ({"primary": ["backup"]},),
+        {SETTING: True},
+        2,
+        (registered_guardrail("output-check", "post_call"),),
+    ) as rig:
+        yield rig
+
+
+@pytest.fixture(scope="module")
+def unset(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    with booted_rig(
+        tmp_path_factory.mktemp("continuation-unset"), ({"primary": ["no-prefill", "backup"]},), {}, 1
+    ) as rig:
+        yield rig
+
+
+def new_key(rig: Rig) -> str:
+    key: Final = rig.gateway.post("/key/generate", {})["key"]
+    assert isinstance(key, str), key
+    return key
+
+
+def appended(seen: tuple[ChatCompletionChunk, ...], chunk: ChatCompletionChunk) -> tuple[ChatCompletionChunk, ...]:
+    return (*seen, chunk)
+
+
+def observed(
+    response: Iterable[ChatCompletionChunk], releases: Sequence[Release]
+) -> Iterator[ChatCompletionChunk | openai.APIError]:
+    try:
+        for seen in itertools.accumulate(response, appended, initial=NOTHING_SEEN):
+            for release in releases:
+                if release.ready(seen):
+                    release.gate.set()
+            yield from seen[-1:]
+    except openai.APIError as error:
+        yield error
+
+
+def stream(
+    rig: Rig,
+    key: str,
+    model: str,
+    prompt: str,
+    releases: Sequence[Release] = (),
+    extra: Mapping[str, JsonValue] | None = None,
+) -> Streamed:
+    base_url: Final = f"{rig.gateway.client.base_url}".rstrip("/") + "/v1"
+    try:
+        with openai.OpenAI(base_url=base_url, api_key=key, max_retries=0, timeout=60) as client:
+            response: Final = client.chat.completions.create(
+                model=model,
+                messages=[{"role": "user", "content": prompt}],
+                stream=True,
+                stream_options={"include_usage": True},
+                max_tokens=MAX_TOKENS,
+                extra_body=dict(extra or {}),
+            )
+            outcome: Final = tuple(observed(response, releases))
+    finally:
+        for release in releases:
+            release.gate.set()
+    errors: Final = tuple(item for item in outcome if isinstance(item, openai.APIError))
+    return Streamed(
+        tuple(item for item in outcome if isinstance(item, ChatCompletionChunk)), errors[0] if errors else None
+    )
+
+
+def chat_bodies(wire: Wire, marker: str) -> tuple[dict[str, JsonValue], ...]:
+    return tuple(
+        JSON_OBJECT.validate_json(request.body)
+        for request in wire.drain()
+        if is_chat(request) and marker_of(request) == marker
+    )
+
+
+def spend_rows(rig: Rig, key: str) -> list[dict[str, JsonValue]]:
+    return eventually(
+        lambda: read_rows(
+            'SELECT request_id, status, spend, prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" '
+            "WHERE api_key=%s",
+            (hashlib.sha256(key.encode()).hexdigest(),),
+            database_url=rig.database_url,
+        ),
+        lambda rows: len(rows) >= 1,
+        seconds=70,
+    )
+
+
+def amount(value: JsonValue) -> float:
+    assert isinstance(value, (int, float)), value
+    return float(value)
+
+
+def key_spend(rig: Rig, key: str) -> float:
+    return amount(object_value(rig.gateway.get("/key/info", {"key": key})["info"])["spend"])
+
+
+def close(observed_value: float, expected: float) -> bool:
+    return math.isclose(observed_value, expected, rel_tol=1e-9, abs_tol=1e-12)
+
+
+def cost(tokens: tuple[int, int], prices: tuple[float, float]) -> float:
+    return tokens[0] * prices[0] + tokens[1] * prices[1]
+
+
+def prefill_turn(text: str) -> dict[str, JsonValue]:
+    return {"role": "assistant", "content": text, "prefix": True}
+
+
+@dataclass(frozen=True, slots=True)
+class Billed:
+    rows: list[dict[str, JsonValue]]
+    partial: tuple[int, int]
+
+
+def billed_once(rig: Rig, key: str, received: Streamed) -> Billed:
+    rows: Final = spend_rows(rig, key)
+    assert [row["status"] for row in rows] == ["success"], rows
+    row_tokens: Final = (rows[0]["prompt_tokens"], rows[0]["completion_tokens"])
+    assert isinstance(row_tokens[0], int) and isinstance(row_tokens[1], int), rows
+    assert received.usage == row_tokens, (received.usage, row_tokens)
+    partial: Final = (row_tokens[0] - BACKUP_USAGE[0], row_tokens[1] - BACKUP_USAGE[1])
+    assert partial[0] > 0 and partial[1] > 0, (row_tokens, BACKUP_USAGE)
+    return Billed(rows, partial)
+
+
+def assert_spend_prices_every_token(rig: Rig, key: str, billed: Billed) -> None:
+    partial_cost: Final = cost(billed.partial, PRIMARY_PRICES)
+    breakdown: Final = object_value(object_value(billed.rows[0]["metadata"])["cost_breakdown"])
+    spend: Final = amount(billed.rows[0]["spend"])
+    reported_partial_cost: Final = amount(object_value(breakdown["additional_costs"])["mid_stream_partial_cost"])
+    assert close(spend, cost(BACKUP_USAGE, BACKUP_PRICES) + partial_cost) and close(
+        reported_partial_cost, partial_cost
+    ), f"partial={billed.partial} spend={spend} breakdown={json.dumps(breakdown)}"
+    eventually(lambda: key_spend(rig, key), lambda value: close(value, spend), seconds=70)
+    assert spend_rows(rig, key) == billed.rows
+
+
+def test_text_cut_continues_on_the_prefill_fallback_and_bills_both_calls_once(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    key: Final = new_key(configured)
+    prompt: Final = f"text-cut {marker}"
+    received: Final = stream(configured, key, "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL),))
+    assert received.error is None, received.error
+    assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks
+    assert received.ids == {f"chatcmpl-{marker}-primary"}, received.ids
+    assert received.role_deltas == 1, received.chunks
+    assert received.finish_reasons == ("stop",), received.chunks
+    assert len(chat_bodies(configured.primary, marker)) == 1
+    continued: Final = chat_bodies(configured.backup, marker)
+    assert len(continued) == 1, continued
+    assert continued[0]["messages"] == [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)]
+    assert "_mid_stream_continuation" not in continued[0], continued[0]
+    billed: Final = billed_once(configured, key, received)
+    assert continued[0]["max_tokens"] == MAX_TOKENS - billed.partial[1], (continued[0], billed.partial)
+    assert_spend_prices_every_token(configured, key, billed)
+
+
+def test_spend_row_of_a_continued_stream_carries_the_id_the_caller_received(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    key: Final = new_key(configured)
+    received: Final = stream(
+        configured, key, "primary", f"text-cut {marker}", (after_text(marker, "primary", JOINED_PARTIAL),)
+    )
+    assert received.error is None, received.error
+    assert [row["request_id"] for row in spend_rows(configured, key)] == [received.chunks[0].id]
+
+
+def test_tool_call_before_the_cut_keeps_the_original_error(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        configured,
+        new_key(configured),
+        "primary",
+        f"tool-cut {marker}",
+        (Release(cut_gate(marker, "primary", 0), has_tool_call),),
+        {"tools": [dict(WEATHER_TOOL)]},
+    )
+    assert received.error is not None, received.chunks
+    assert (received.text, has_tool_call(received.chunks)) == (TOOL_PREAMBLE, True), received.chunks
+    assert len(chat_bodies(configured.primary, marker)) == 1
+    assert chat_bodies(configured.backup, marker) == ()
+
+
+def test_continuation_skips_a_fallback_without_prefill_support(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    prompt: Final = f"text-cut {marker}"
+    received: Final = stream(
+        configured, new_key(configured), "primary-skip", prompt, (after_text(marker, "primary", JOINED_PARTIAL),)
+    )
+    assert received.error is None, received.error
+    assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks
+    assert chat_bodies(configured.no_prefill, marker) == ()
+    assert [body["messages"] for body in chat_bodies(configured.backup, marker)] == [
+        [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)]
+    ]
+
+
+def test_no_prefill_capable_fallback_keeps_the_original_error(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        configured,
+        new_key(configured),
+        "primary-none",
+        f"text-cut {marker}",
+        (after_text(marker, "primary", JOINED_PARTIAL),),
+    )
+    assert received.error is not None, received.chunks
+    assert "No deployments available" not in str(received.error), received.error
+    assert received.text == JOINED_PARTIAL, received.chunks
+    assert len(chat_bodies(configured.primary, marker)) == 1
+    assert chat_bodies(configured.no_prefill, marker) == ()
+
+
+def test_undeclared_fallback_sharing_a_backend_with_a_prefill_deployment_keeps_the_original_error(
+    configured: Rig,
+) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        configured,
+        new_key(configured),
+        "primary-shared",
+        f"text-cut {marker}",
+        (after_text(marker, "primary", JOINED_PARTIAL),),
+    )
+    assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks
+    assert chat_bodies(configured.no_prefill, marker) == ()
+
+
+def test_failure_before_content_still_falls_back_to_a_fallback_without_prefill_support(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    prompt: Final = f"reject {marker}"
+    received: Final = stream(configured, new_key(configured), "primary-none", prompt)
+    assert received.error is None, received.error
+    assert received.text == PLAIN_TEXT, received.chunks
+    assert [body["messages"] for body in chat_bodies(configured.no_prefill, marker)] == [
+        [{"role": "user", "content": prompt}]
+    ]
+
+
+def stream_with_two_cuts(rig: Rig, key: str, marker: str) -> Streamed:
+    return stream(
+        rig,
+        key,
+        "primary-relay",
+        f"text-cut {marker}",
+        (after_text(marker, "primary", JOINED_PARTIAL), after_text(marker, "relay", JOINED_PARTIAL + RELAY_TEXT)),
+    )
+
+
+def test_second_cut_continues_with_both_partials_in_one_prefill(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    key: Final = new_key(configured)
+    received: Final = stream_with_two_cuts(configured, key, marker)
+    assert received.error is None, received.error
+    assert received.text == JOINED_PARTIAL + RELAY_TEXT + BACKUP_TEXT, received.chunks
+    assert received.role_deltas == 1, received.chunks
+    user_turn: Final = {"role": "user", "content": f"text-cut {marker}"}
+    relayed: Final = chat_bodies(configured.relay, marker)
+    continued: Final = chat_bodies(configured.backup, marker)
+    assert [body["messages"] for body in relayed] == [[user_turn, prefill_turn(JOINED_PARTIAL)]]
+    assert [body["messages"] for body in continued] == [[user_turn, prefill_turn(JOINED_PARTIAL + RELAY_TEXT)]]
+    partial: Final = billed_once(configured, key, received).partial
+    relay_ceiling: Final = relayed[0]["max_tokens"]
+    assert isinstance(relay_ceiling, int), relayed[0]
+    assert 0 < MAX_TOKENS - relay_ceiling < partial[1], (relayed[0], partial)
+    assert continued[0]["max_tokens"] == MAX_TOKENS - partial[1], (continued[0], partial)
+
+
+def test_second_cut_bills_every_call_on_one_row(configured: Rig) -> None:
+    key: Final = new_key(configured)
+    received: Final = stream_with_two_cuts(configured, key, uuid.uuid4().hex)
+    assert received.error is None, received.error
+    assert_spend_prices_every_token(configured, key, billed_once(configured, key, received))
+
+
+def test_repeating_a_continued_request_never_returns_only_the_continuation(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    key: Final = new_key(configured)
+    prompt: Final = f"text-cut {marker}"
+    answers: Final = tuple(
+        stream(configured, key, "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL, attempt),))
+        for attempt in range(2)
+    )
+    assert tuple((answer.error, answer.text) for answer in answers) == ((None, JOINED_PARTIAL + BACKUP_TEXT),) * 2
+
+
+def test_without_the_setting_a_text_cut_keeps_the_original_error(unset: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        unset, new_key(unset), "primary", f"text-cut {marker}", (after_text(marker, "primary", JOINED_PARTIAL),)
+    )
+    assert received.error is not None, received.chunks
+    assert received.text == JOINED_PARTIAL, received.chunks
+    assert len(chat_bodies(unset.primary, marker)) == 1
+    assert chat_bodies(unset.no_prefill, marker) == ()
+    assert chat_bodies(unset.backup, marker) == ()
+
+
+def router_setting(rig: Rig) -> JsonValue:
+    return object_value(rig.gateway.get("/router/settings")["current_values"]).get(SETTING)
+
+
+def test_turning_the_setting_on_at_runtime_continues_on_a_prefill_fallback(unset: Rig) -> None:
+    rig: Final = unset
+    rig.gateway.post("/config/update", {"router_settings": {SETTING: True}})
+    try:
+        assert router_setting(rig) is True
+        marker: Final = uuid.uuid4().hex
+        prompt: Final = f"text-cut {marker}"
+        received: Final = stream(rig, new_key(rig), "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL),))
+        assert received.error is None, received.error
+        assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks
+        assert chat_bodies(rig.no_prefill, marker) == ()
+        assert [body["messages"] for body in chat_bodies(rig.backup, marker)] == [
+            [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)]
+        ]
+    finally:
+        rig.gateway.post("/config/update", {"router_settings": {SETTING: False}})
+
+
+def test_a_registered_output_guardrail_keeps_the_original_error(output_guarded: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        output_guarded,
+        new_key(output_guarded),
+        "primary",
+        f"text-cut {marker}",
+        (after_text(marker, "primary", JOINED_PARTIAL),),
+    )
+    assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks
+    assert len(chat_bodies(output_guarded.primary, marker)) == 1
+    assert chat_bodies(output_guarded.backup, marker) == ()
+
+
+def test_a_request_fallback_bringing_its_own_messages_keeps_the_original_error(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        configured,
+        new_key(configured),
+        "primary",
+        f"text-cut {marker}",
+        (after_text(marker, "primary", JOINED_PARTIAL),),
+        {"fallbacks": [{"model": "backup", "messages": [{"role": "user", "content": f"start over {marker}"}]}]},
+    )
+    assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks
+    assert len(chat_bodies(configured.primary, marker)) == 1
+    assert chat_bodies(configured.backup, marker) == ()
+
+
+def carries_tag(event: Mapping[str, JsonValue], marker: str) -> bool:
+    tags: Final = event.get("request_tags")
+    return isinstance(tags, list) and marker in tags
+
+
+def tagged_events(batches: Iterable[Request], marker: str) -> tuple[dict[str, JsonValue], ...]:
+    events: Final = itertools.chain.from_iterable(EVENT_BATCH.validate_json(batch.body) for batch in batches)
+    return tuple(event for event in events if carries_tag(event, marker))
+
+
+def logged_events(sink: Wire, marker: str) -> Iterator[tuple[dict[str, JsonValue], ...]]:
+    return itertools.accumulate((tagged_events(sink.drain(), marker) for _ in itertools.count()), operator.add)
+
+
+def test_the_cut_call_logs_its_failure_under_its_own_deployment_and_price(configured: Rig) -> None:
+    marker: Final = uuid.uuid4().hex
+    received: Final = stream(
+        configured,
+        new_key(configured),
+        "primary",
+        f"text-cut {marker}",
+        (after_text(marker, "primary", JOINED_PARTIAL),),
+        {"metadata": {"tags": [marker]}},
+    )
+    assert received.error is None, received.error
+    polls: Final = logged_events(configured.events, marker)
+    events: Final = eventually(lambda: next(polls), lambda logged: len(logged) >= 2, seconds=30)
+    assert sorted(str(event["status"]) for event in events) == ["failure", "success"], events
+    failure: Final = next(event for event in events if event["status"] == "failure")
+    assert failure["model_group"] == "primary", failure
+    assert str(failure["api_base"]).startswith(configured.primary.url), failure
+    tokens: Final = (failure["prompt_tokens"], failure["completion_tokens"])
+    assert isinstance(tokens[0], int) and isinstance(tokens[1], int) and tokens[0] > 0 and tokens[1] > 0, failure
+    assert close(amount(failure["response_cost"]), cost((tokens[0], tokens[1]), PRIMARY_PRICES)), failure

tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -642,6 +642,64 @@ async def test_bedrock_converse_replays_reasoning_to_claude_only_as_base64_of_cl
     assert assistant_content == [*sent, {"text": "hello"}]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize(
+    "model, final_text",
+    [("us.anthropic.claude-sonnet-4-6", "Leaves fall"), ("us.amazon.nova-pro-v1:0", "Leaves fall \n")],
+    ids=["claude", "nova"],
+)
+async def test_bedrock_converse_strips_trailing_whitespace_from_a_final_assistant_turn_for_claude_only(
+    use_async: bool, model: str, final_text: str
+):
+    """Claude on Converse 400s on a final assistant turn ending in whitespace, which a cut stream's prefill often has."""
+    messages: Final = [
+        {"role": "user", "content": "Write a haiku."},
+        {"role": "assistant", "content": "Sure.  "},
+        {"role": "user", "content": "About autumn."},
+        {"role": "assistant", "content": "Leaves fall \n"},
+    ]
+
+    result: Final = (
+        await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
+            messages=messages, model=model, llm_provider="bedrock"
+        )
+        if use_async
+        else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock")
+    )
+
+    assert [m["content"] for m in result if m["role"] == "assistant"] == [
+        [{"text": "Sure.  "}],
+        [{"text": final_text}],
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+async def test_bedrock_converse_keeps_whitespace_between_text_blocks_of_a_final_claude_assistant_turn(
+    use_async: bool,
+):
+    messages: Final = [
+        {"role": "user", "content": "Write a haiku."},
+        {
+            "role": "assistant",
+            "content": [{"type": "text", "text": "Leaves fall "}, {"type": "text", "text": "softly \n"}],
+        },
+    ]
+
+    result: Final = (
+        await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
+            messages=messages, model="us.anthropic.claude-sonnet-4-6", llm_provider="bedrock"
+        )
+        if use_async
+        else _bedrock_converse_messages_pt(
+            messages=messages, model="us.anthropic.claude-sonnet-4-6", llm_provider="bedrock"
+        )
+    )
+
+    assert result[-1]["content"] == [{"text": "Leaves fall "}, {"text": "softly"}]
+
+
 def test_translate_thinking_blocks_keeps_dropping_redacted_data_when_called_the_old_way():
     """Outside callers that only pass the thinking blocks keep the old behavior, with no redactedContent sent."""
     assert BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks(

tests/unit/litellm_core_utils/test_streaming_handler.py

@@ -1019,6 +1019,87 @@ async def test_bridged_stream_mid_stream_fallback_error_is_rebuilt_around_the_pr
     assert excinfo.value.generated_content == ""
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("merge_reasoning", [True, False], ids=["merged_into_content", "kept_as_reasoning"])
+async def test_mid_stream_break_after_reasoning_disqualifies_continuation(logging_obj: Logging, merge_reasoning: bool):
+    import httpx
+
+    from litellm.exceptions import MidStreamFallbackError
+
+    class _ReasoningThenCut:
+        def __init__(self) -> None:
+            self.chunks = iter(
+                (
+                    ModelResponseStream(choices=[{"index": 0, "delta": {"reasoning_content": "Let me think"}}]),
+                    ModelResponseStream(choices=[{"index": 0, "delta": {"content": "The answer"}}]),
+                )
+            )
+
+        def __aiter__(self) -> "_ReasoningThenCut":
+            return self
+
+        async def __anext__(self) -> ModelResponseStream:
+            next_chunk: Final = next(self.chunks, None)
+            if next_chunk is None:
+                raise httpx.ReadTimeout("connection dropped")
+            return next_chunk
+
+    response: Final = CustomStreamWrapper(
+        completion_stream=_ReasoningThenCut(), model="deepseek-reasoner", logging_obj=logging_obj
+    )
+    response.merge_reasoning_content_in_choices = merge_reasoning
+
+    with pytest.raises(MidStreamFallbackError) as excinfo:
+        async for _ in response:
+            pass
+
+    assert excinfo.value.is_pre_first_chunk is False
+    assert excinfo.value.emitted_disqualifying_content is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "outer_delta,inner_flag,expected",
+    [
+        ({"tool_calls": [{"index": 0}]}, False, True),
+        ({"content": "Let me check."}, True, True),
+        ({"content": "Let me check."}, False, False),
+    ],
+    ids=["bridge_wrapper_sent_a_tool_call", "inner_stream_sent_a_tool_call", "plain_text_only"],
+)
+async def test_bridged_stream_rebuild_keeps_the_continuation_disqualifier(
+    logging_obj: Logging, outer_delta: dict[str, object], inner_flag: bool, expected: bool
+):
+    from litellm.exceptions import InternalServerError, MidStreamFallbackError
+
+    provider_error: Final = InternalServerError(message="upstream 500", llm_provider="azure", model="gpt-6.1-sol")
+    inner_error: Final = MidStreamFallbackError(
+        message=str(provider_error),
+        model="gpt-6.1-sol",
+        llm_provider="azure",
+        original_exception=provider_error,
+        is_pre_first_chunk=False,
+        emitted_disqualifying_content=inner_flag,
+    )
+
+    async def _raise_inner_error(**kwargs: object) -> NoReturn:
+        raise inner_error
+
+    response: Final = CustomStreamWrapper(
+        completion_stream=None,
+        model="gpt-6.1-sol",
+        logging_obj=logging_obj,
+        custom_llm_provider="azure",
+        make_call=_raise_inner_error,
+    )
+    response._accumulate_streamed_delta(outer_delta)
+
+    with pytest.raises(MidStreamFallbackError) as excinfo:
+        await response.__anext__()
+
+    assert excinfo.value.emitted_disqualifying_content is expected
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("sync_mode", [True, False])
 async def test_streaming_rate_limit_midstream_fallback_keeps_provider_headers(logging_obj: Logging, sync_mode: bool):
@@ -6230,6 +6311,59 @@ class TestStableStreamingResponseId:
         assert wrapper.model_response_creator().id == "chatcmpl-from-provider"
 
 
+class TestContinuationDisqualifiers:
+    """The mid-stream continuation eligibility hinges on classifying which deltas
+    carry output a text-only prefill cannot represent."""
+
+    @pytest.mark.parametrize(
+        "field",
+        [
+            "tool_calls",
+            "function_call",
+            "reasoning_content",
+            "thinking_blocks",
+            "reasoning_items",
+            "audio",
+            "images",
+            "annotations",
+        ],
+    )
+    def test_disqualifying_fields_flagged(self, field):
+        # reasoning_content included: it reaches the caller as visible reasoning a
+        # text-only prefill cannot carry, so a reasoning fallback would re-derive it
+        assert CustomStreamWrapper._delta_disqualifies_continuation({field: [{"x": 1}]}) is True
+
+    @pytest.mark.parametrize("delta", [{"content": "hi"}, {}, {"role": "assistant"}])
+    def test_plain_text_not_flagged(self, delta):
+        assert CustomStreamWrapper._delta_disqualifies_continuation(delta) is False
+
+    def test_non_mapping_delta_is_safe(self):
+        assert CustomStreamWrapper._delta_disqualifies_continuation(object()) is False
+
+    def test_accumulate_grows_text_and_leaves_flag_clear_for_plain_text(self):
+        wrapper = object.__new__(CustomStreamWrapper)
+        wrapper.response_uptil_now = ""
+        wrapper._emitted_disqualifying_content = False
+
+        wrapper._accumulate_streamed_delta({"content": "Hel"})
+        wrapper._accumulate_streamed_delta({"content": "lo"})
+
+        assert wrapper.response_uptil_now == "Hello"
+        assert wrapper._emitted_disqualifying_content is False
+
+    def test_accumulate_latches_flag_on_disqualifying_delta(self):
+        wrapper = object.__new__(CustomStreamWrapper)
+        wrapper.response_uptil_now = ""
+        wrapper._emitted_disqualifying_content = False
+
+        wrapper._accumulate_streamed_delta({"content": "Hi"})
+        wrapper._accumulate_streamed_delta({"tool_calls": [{"index": 0}]})
+        wrapper._accumulate_streamed_delta({"content": "there"})
+
+        assert wrapper.response_uptil_now == "Hithere"
+        assert wrapper._emitted_disqualifying_content is True
+
+
 @pytest.mark.asyncio
 async def test_async_stream_without_usage_counts_tokens_off_the_event_loop():
     from tests.large_text import text

tests/unit/llms/dashscope/test_dashscope_video_transformation.py

@@ -194,8 +194,9 @@ class TestDashScopeVideoInputReference:
             lambda tmp_path: ("first.png", PNG_BYTES, "image/png"),
             lambda tmp_path: ("first.png", io.BytesIO(PNG_BYTES)),
             lambda tmp_path: _write_png(tmp_path),
+            lambda tmp_path: ("first.png", ("inner.png", PNG_BYTES)),
         ],
-        ids=["filename-bytes-content-type", "filename-file", "path"],
+        ids=["filename-bytes-content-type", "filename-file", "path", "nested-tuple"],
     )
     def test_file_tuple_and_path_references_are_encoded(self, make_reference, tmp_path):
         """Regression: tuples and paths are valid FileTypes but raised before reaching DashScope."""

tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py

@@ -461,6 +461,21 @@ def test_transform_messages_sanitizes_empty_content():
     assert result[1]["content"] == "Hi"
 
 
+def test_transform_messages_drops_the_prefill_prefix_flag():
+    """Databricks rejects unknown message keys, so a continuation prefill reaches Claude without ``prefix``."""
+    messages = [
+        {"role": "user", "content": "How do transformers work?"},
+        {"role": "assistant", "content": "Transformers use", "prefix": True},
+    ]
+
+    result = DatabricksConfig().transform_messages(
+        messages=messages, model="databricks-claude-sonnet-4-6", is_async=False
+    )
+
+    assert result[-1] == {"role": "assistant", "content": "Transformers use"}
+    assert messages[-1]["prefix"] is True
+
+
 def test_transform_request_preserves_unity_model_service_name():
     config = DatabricksConfig()
     result = config.transform_request(

tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py

@@ -14,8 +14,8 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 import litellm
 import litellm.constants as litellm_constants
 import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils
-from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL
-from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY
+from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL
+from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY
 from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
 from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -243,6 +243,82 @@ def test_batch_session_outranks_the_per_request_trace_id():
     assert session_id == "batch-uid-1"
 
 
+def _chat_session_id(
+    monkeypatch: pytest.MonkeyPatch,
+    litellm_params: Mapping[str, object],
+    correlation_in_logs: bool = True,
+    batch_trace_session_id: str | None = None,
+    metadata: Mapping[str, object] = MappingProxyType({}),
+) -> str | None:
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs)
+    return _get_session_id_for_spend_log(
+        kwargs={"litellm_trace_id": "trace-abc", "litellm_params": litellm_params},
+        metadata=metadata,
+        standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
+        omit_when_missing=False,
+        batch_trace_session_id=batch_trace_session_id,
+    )
+
+
+_CALLER_SESSION_PARAMS: Final = MappingProxyType({"litellm_session_id": "sess-1", "metadata": {"trace_id": "otel-1"}})
+
+
+def test_caller_session_wins_over_the_per_request_trace_id(monkeypatch: pytest.MonkeyPatch):
+    assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS) == "sess-1"
+
+
+def test_batch_session_outranks_a_caller_session(monkeypatch: pytest.MonkeyPatch):
+    assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, batch_trace_session_id="batch-uid-1") == "batch-uid-1"
+
+
+def test_caller_session_keeps_the_legacy_trace_id_with_correlation_logging_off(monkeypatch: pytest.MonkeyPatch):
+    assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, correlation_in_logs=False) == "trace-abc"
+
+
+@pytest.mark.parametrize(("correlation_in_logs", "expected_session_id"), [(True, "convo-1"), (False, "trace-abc")])
+def test_session_displaced_by_a_trace_header_wins_only_with_correlation_logging(
+    monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, expected_session_id: str
+):
+    header_chain: Final = {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc", "session_id": "trace-abc"}}
+    session_id: Final = _chat_session_id(
+        monkeypatch,
+        header_chain,
+        correlation_in_logs=correlation_in_logs,
+        metadata={CALLER_SESSION_ID_METADATA_KEY: "convo-1"},
+    )
+    assert session_id == expected_session_id
+
+
+@pytest.mark.parametrize(
+    ("litellm_params", "metadata"),
+    [
+        pytest.param({"litellm_session_id": "convo\x00x"}, {}, id="body-session"),
+        pytest.param({"litellm_session_id": "trace-abc"}, {CALLER_SESSION_ID_METADATA_KEY: "convo\x00x"}, id="beside-trace-header"),
+    ],
+)
+def test_caller_session_postgres_cannot_store_keeps_the_trace_id(
+    monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object], metadata: Mapping[str, object]
+):
+    assert _chat_session_id(monkeypatch, litellm_params, metadata=metadata) == "trace-abc"
+
+
+@pytest.mark.parametrize(
+    "litellm_params",
+    [
+        pytest.param({"metadata": {}}, id="no-session"),
+        pytest.param(
+            {"litellm_session_id": "minted-1", "metadata": {"session_id": "minted-1", SESSION_ID_GENERATED_METADATA_KEY: True}},
+            id="minted-by-generate-policy",
+        ),
+        pytest.param({"litellm_session_id": "job-7", "metadata": {"trace_id": "job-7"}}, id="copied-from-trace-id"),
+    ],
+)
+def test_session_that_is_not_a_caller_conversation_keeps_the_trace_id(
+    monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object]
+):
+    assert _chat_session_id(monkeypatch, litellm_params) == "trace-abc"
+
+
 def test_omit_policy_still_suppresses_batch_sessions():
     session_id: Final = _get_session_id_for_spend_log(
         kwargs={},

tests/unit/proxy/test_litellm_pre_call_utils.py

@@ -17,6 +17,7 @@ from starlette.datastructures import Headers
 
 import litellm
 from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
     ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
     SERVER_STREAMING_CLASSIFICATION_KEY,
     SERVER_STREAMING_CLASSIFICATION_MARKER,
@@ -3308,6 +3309,93 @@ def test_add_litellm_metadata_from_request_headers_x_litellm_trace_id_sets_chain
     assert data["litellm_trace_id"] == "foo"
 
 
+@pytest.mark.parametrize(("correlation_in_logs", "expected_spend_log_session"), [(True, "convo-1"), (False, None)])
+@pytest.mark.parametrize("session_in_metadata", [False, True], ids=["top-level", "metadata"])
+def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only(
+    monkeypatch: pytest.MonkeyPatch,
+    correlation_in_logs: bool,
+    expected_spend_log_session: str | None,
+    session_in_metadata: bool,
+):
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs)
+    metadata: Final[dict[str, str]] = {"session_id": "convo-1"} if session_in_metadata else {}
+    data: Final[dict[str, object]] = (
+        {"metadata": metadata} if session_in_metadata else {"metadata": metadata, "litellm_session_id": "convo-1"}
+    )
+    LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
+        headers={"x-litellm-trace-id": "req-trace-1"}, data=data, _metadata_variable_name="metadata"
+    )
+    assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_spend_log_session
+    assert data["litellm_session_id"] == "req-trace-1"
+    assert metadata["session_id"] == "req-trace-1"
+    assert data["litellm_trace_id"] == "req-trace-1"
+
+
+@pytest.mark.parametrize(
+    "session_header", ["x-litellm-session-id", "x-claude-code-session-id", "x-session-id"]
+)
+@pytest.mark.parametrize("with_trace_header", [False, True], ids=["session-header-only", "beside-trace-header"])
+def test_session_headers_still_beat_the_body_session_with_correlation_logging(
+    monkeypatch: pytest.MonkeyPatch, session_header: str, with_trace_header: bool
+):
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
+    metadata: Final[dict[str, str]] = {}
+    data: Final[dict[str, object]] = {"metadata": metadata, "litellm_session_id": "convo-1"}
+    trace_header: Final = {"x-litellm-trace-id": "req-trace-1"} if with_trace_header else {}
+    LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
+        headers={**trace_header, session_header: "hdr-session-1"}, data=data, _metadata_variable_name="metadata"
+    )
+    spend_log_session: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) or data["litellm_session_id"]
+    assert spend_log_session == "hdr-session-1"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("forged_in", ["metadata", "litellm_metadata"])
+@pytest.mark.parametrize(
+    "headers",
+    [{"x-litellm-session-id": "hdr-session-1"}, {}],
+    ids=["session-header", "no-headers"],
+)
+async def test_client_metadata_cannot_set_the_spend_log_caller_session(
+    monkeypatch: pytest.MonkeyPatch, forged_in: str, headers: dict[str, str]
+):
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
+    updated: Final = await add_litellm_data_to_request(
+        data={
+            "model": "gpt-5.4-mini",
+            "messages": [{"role": "user", "content": "hi"}],
+            forged_in: {CALLER_SESSION_ID_METADATA_KEY: "forged-session"},
+        },
+        request=_make_request_mock("/v1/chat/completions", headers),
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        proxy_config=MagicMock(),
+        general_settings={},
+        version="test-version",
+    )
+    assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"]
+
+
+@pytest.mark.asyncio
+async def test_spend_log_caller_session_stays_out_of_requester_metadata(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
+    updated: Final = await add_litellm_data_to_request(
+        data={
+            "model": "gpt-5.4-mini",
+            "messages": [{"role": "user", "content": "hi"}],
+            "litellm_session_id": "convo-1",
+            "metadata": {"team_label": "kept"},
+        },
+        request=_make_request_mock("/v1/chat/completions", {"x-litellm-trace-id": "req-trace-1"}),
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        proxy_config=MagicMock(),
+        general_settings={},
+        version="test-version",
+    )
+    assert updated["metadata"][CALLER_SESSION_ID_METADATA_KEY] == "convo-1"
+    assert updated["metadata"]["requester_metadata"]["team_label"] == "kept"
+    assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"]["requester_metadata"]
+
+
 def test_add_litellm_metadata_from_request_headers_x_litellm_session_id_sets_chain_id():
     """x-litellm-session-id sets both metadata and top-level litellm_session_id/litellm_trace_id for call chaining."""
     headers = {"x-litellm-session-id": "bar"}

tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py

@@ -17,6 +17,7 @@ from openai.types.responses.response_function_web_search import (
 from pydantic import TypeAdapter
 
 import litellm
+from litellm.constants import CALLER_SESSION_ID_METADATA_KEY
 from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -6499,3 +6500,69 @@ def test_transform_request_reads_service_tier_from_responses_api_request() -> No
     )
 
     assert result["service_tier"] == "priority"
+
+
+def _continued_chain_request(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    async def session_lookup(previous_response_id: str) -> ChatCompletionSession:
+        assert previous_response_id == "resp_turn_1"
+        return ChatCompletionSession(
+            messages=[{"role": "user", "content": "turn 1"}], litellm_session_id="chain-session"
+        )
+
+    return asyncio.run(
+        LiteLLMCompletionResponsesConfig.async_responses_api_session_handler(
+            previous_response_id="resp_turn_1",
+            litellm_completion_request={
+                "model": _CLAUDE_ON_BEDROCK,
+                "custom_llm_provider": "bedrock",
+                "messages": [{"role": "user", "content": "turn 2"}],
+                "metadata": request_kwargs.get("metadata"),
+            },
+            instructions=None,
+            session_lookup=session_lookup,
+            request_kwargs=request_kwargs,
+        )
+    )
+
+
+@pytest.mark.parametrize(
+    ("request_kwargs", "logged_metadata_key"),
+    [
+        (
+            {
+                "metadata": {"tag": "client"},
+                "litellm_metadata": {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"},
+            },
+            "litellm_metadata",
+        ),
+        ({"metadata": {"user_api_key": "hashed"}}, "metadata"),
+    ],
+)
+def test_a_continued_chain_logs_its_spend_under_the_chain_session_with_request_correlation(
+    monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object], logged_metadata_key: str
+):
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
+    sent_kwargs: Final = deepcopy(request_kwargs)
+
+    request: Final = _continued_chain_request(request_kwargs)
+
+    assert request["litellm_trace_id"] == "chain-session"
+    assert "litellm_session_id" not in request
+    assert request[logged_metadata_key] == {
+        **TypeAdapter(dict[str, object]).validate_python(request_kwargs[logged_metadata_key]),
+        CALLER_SESSION_ID_METADATA_KEY: "chain-session",
+    }
+    assert request_kwargs == sent_kwargs
+
+
+def test_a_continued_chain_leaves_the_turns_metadata_alone_without_request_correlation(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(litellm, "request_correlation_in_logs", False)
+    caller_metadata: Final = {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"}
+
+    request: Final = _continued_chain_request({"metadata": caller_metadata, "litellm_metadata": caller_metadata})
+
+    assert request["litellm_trace_id"] == "chain-session"
+    assert "litellm_metadata" not in request
+    assert request["metadata"] == {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"}

tests/unit/responses/litellm_completion_transformation/test_session_handler.py

@@ -1,4 +1,5 @@
 import json
+import sqlite3
 from typing import Final
 from unittest.mock import AsyncMock, patch
 
@@ -787,3 +788,80 @@ async def test_message_history_replays_real_key_named_tool_payloads() -> None:
     tool_message: Final = result["messages"][2]
     assert json.loads(assistant_message["tool_calls"][0]["function"]["arguments"]) == function_arguments
     assert json.loads(tool_message["content"]) == function_output
+
+
+class _SqliteBackedPrismaDB:
+    def __init__(self, rows: list[tuple[str, str, str, str, str | None, str | None]]) -> None:
+        self._connection = sqlite3.connect(":memory:")
+        self._connection.row_factory = sqlite3.Row
+        self._connection.execute(
+            'CREATE TABLE "LiteLLM_SpendLogs" '
+            '(request_id TEXT, api_key TEXT, session_id TEXT, "endTime" TEXT, "user" TEXT, team_id TEXT)'
+        )
+        self._connection.executemany('INSERT INTO "LiteLLM_SpendLogs" VALUES (?, ?, ?, ?, ?, ?)', rows)
+
+    async def query_raw(self, query: str, *args: object) -> list[dict[str, object]]:
+        return [dict(row) for row in self._connection.execute(query.replace("$1", "?"), args)]
+
+
+@pytest.mark.asyncio
+async def test_session_lookup_does_not_return_another_keys_rows_for_a_shared_session_id():
+    """A caller-supplied litellm_session_id can collide with another key's session; history must stay per key."""
+    sqlite_db: Final = _SqliteBackedPrismaDB(
+        [
+            ("victim-1", "victim-key", "shared-session", "2026-01-01T00:00:01", "", ""),
+            ("victim-2", "victim-key", "shared-session", "2026-01-01T00:00:02", "", ""),
+            ("attacker-1", "attacker-key", "shared-session", "2026-01-01T00:00:03", "", ""),
+            ("attacker-2", "attacker-key", "other-session", "2026-01-01T00:00:04", "", ""),
+        ]
+    )
+    fake_prisma_client = _FakePrismaClient(results=[])
+    fake_prisma_client.db = sqlite_db
+
+    with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client):
+        spend_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id("attacker-1")
+        victim_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id("victim-1")
+
+    assert [row["request_id"] for row in spend_logs] == ["attacker-1"]
+    assert [row["request_id"] for row in victim_logs] == ["victim-1", "victim-2"]
+
+
+async def _history_request_ids(sqlite_db: _SqliteBackedPrismaDB, response_id: str) -> list[str]:
+    fake_prisma_client: Final = _FakePrismaClient(results=[])
+    fake_prisma_client.db = sqlite_db
+    with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client):
+        spend_logs: Final = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(response_id)
+    return [row["request_id"] for row in spend_logs]
+
+
+@pytest.mark.asyncio
+async def test_session_lookup_keeps_jwt_users_and_teams_apart_when_they_share_an_empty_api_key():
+    sqlite_db: Final = _SqliteBackedPrismaDB(
+        [
+            ("alice-1", "", "shared-session", "2026-01-01T00:00:01", "alice", "team-a"),
+            ("bob-1", "", "shared-session", "2026-01-01T00:00:02", "bob", "team-a"),
+            ("service-a-1", "", "shared-session", "2026-01-01T00:00:03", "", "team-a"),
+            ("service-b-1", "", "shared-session", "2026-01-01T00:00:04", "", "team-b"),
+            ("alice-2", "", "shared-session", "2026-01-01T00:00:05", "alice", "team-a"),
+        ]
+    )
+
+    assert await _history_request_ids(sqlite_db, "alice-1") == ["alice-1", "alice-2"]
+    assert await _history_request_ids(sqlite_db, "bob-1") == ["bob-1"]
+    assert await _history_request_ids(sqlite_db, "service-a-1") == ["service-a-1"]
+    assert await _history_request_ids(sqlite_db, "service-b-1") == ["service-b-1"]
+
+
+@pytest.mark.asyncio
+async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_keys_apart():
+    sqlite_db: Final = _SqliteBackedPrismaDB(
+        [
+            ("alice-old-key", "hashed-old", "shared-session", "2026-01-01T00:00:01", "alice", "team-a"),
+            ("ownerless-1", "hashed-other", "shared-session", "2026-01-01T00:00:02", "", ""),
+            ("alice-new-key", "hashed-new", "shared-session", "2026-01-01T00:00:03", "alice", "team-a"),
+            ("ownerless-2", "hashed-another", "shared-session", "2026-01-01T00:00:04", "", ""),
+        ]
+    )
+
+    assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"]
+    assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"]

tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py

@@ -0,0 +1,288 @@
+import pytest
+
+import litellm
+from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
+    MID_STREAM_CONTINUATION_KWARG,
+    MID_STREAM_CONTINUATION_MARKER,
+    ContinuationOutput,
+    ContinuationPrefillDeploymentCheck,
+    continuation_output_ceilings,
+    deployment_supports_prefill,
+)
+
+PREFILL_MODEL = "anthropic/prefill-capable-test-model"
+NON_PREFILL_MODEL = "openai/prefill-unknown-test-model"
+UNMAPPED_MODEL = "openai/unmapped-test-model"
+
+
+@pytest.fixture(autouse=True)
+def _synthetic_cost_map_entries() -> None:
+    litellm.register_model(
+        {
+            PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True},
+            NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"},
+        },
+        persist_across_reloads=False,
+    )
+    litellm.get_model_info.cache_clear()
+
+
+def _deployment(model: str, dep_id: str) -> dict[str, object]:
+    return {"litellm_params": {"model": model}, "model_info": {"id": dep_id}}
+
+
+def test_deployment_supports_prefill_reads_capability():
+    assert deployment_supports_prefill(_deployment(PREFILL_MODEL, "a")) is True
+    assert deployment_supports_prefill(_deployment(NON_PREFILL_MODEL, "b")) is False
+    assert deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}}) is True
+    assert deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}, "model_info": "bogus"}) is True
+
+
+def test_deployment_model_info_override_wins_over_cost_map():
+    # model_info True opts in a model that is not in the cost map
+    assert (
+        deployment_supports_prefill(
+            {"litellm_params": {"model": "vendor/custom-model"}, "model_info": {"supports_assistant_prefill": True}}
+        )
+        is True
+    )
+    # model_info False opts out a model the cost map would otherwise allow
+    assert (
+        deployment_supports_prefill(
+            {"litellm_params": {"model": PREFILL_MODEL}, "model_info": {"supports_assistant_prefill": False}}
+        )
+        is False
+    )
+    # model_info without the key falls through to the cost map
+    assert deployment_supports_prefill(_deployment(PREFILL_MODEL, "z")) is True
+
+
+@pytest.mark.parametrize(
+    "deployment,expected",
+    [
+        ({"litellm_params": {"model": "bedrock/arn:custom-import", "base_model": PREFILL_MODEL}}, True),
+        ({"litellm_params": {"model": "azure/my-deployment"}, "model_info": {"base_model": PREFILL_MODEL}}, True),
+        (
+            {
+                "litellm_params": {"model": "azure/my-deployment", "base_model": NON_PREFILL_MODEL},
+                "model_info": {"base_model": PREFILL_MODEL},
+            },
+            True,
+        ),
+        (
+            {
+                "litellm_params": {"model": PREFILL_MODEL, "base_model": NON_PREFILL_MODEL},
+                "model_info": {"base_model": NON_PREFILL_MODEL},
+            },
+            False,
+        ),
+    ],
+    ids=[
+        "litellm_params_base_model",
+        "model_info_base_model",
+        "model_info_base_model_wins",
+        "base_model_wins_over_model",
+    ],
+)
+def test_deployment_supports_prefill_reads_the_cost_map_key_of_the_base_model(deployment, expected):
+    assert deployment_supports_prefill(deployment) is expected
+
+
+def test_deployment_supports_prefill_rejects_malformed_deployments():
+    assert deployment_supports_prefill({}) is False
+    assert deployment_supports_prefill({"litellm_params": {}}) is False
+    assert deployment_supports_prefill("not-a-dict") is False
+
+
+@pytest.mark.asyncio
+async def test_filter_is_noop_without_continuation_marker():
+    """A normal (non-continuation) request must be passed through untouched, even
+    if some deployments cannot prefill."""
+    check = ContinuationPrefillDeploymentCheck()
+    deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")]
+
+    for request_kwargs in ({}, None, {MID_STREAM_CONTINUATION_KWARG: False}):
+        result = await check.async_filter_deployments(
+            model="group", healthy_deployments=deployments, messages=None, request_kwargs=request_kwargs
+        )
+        assert result == deployments
+
+
+@pytest.mark.asyncio
+async def test_filter_ignores_forged_client_flag():
+    """A client cannot steer routing: a plain truthy value under the marker key
+    (which the proxy could forward from the request body) is not the internal
+    sentinel, so the filter leaves the deployment list untouched."""
+    check = ContinuationPrefillDeploymentCheck()
+    deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")]
+
+    for forged in (True, "true", 1, {"any": "json"}):
+        result = await check.async_filter_deployments(
+            model="group",
+            healthy_deployments=deployments,
+            messages=None,
+            request_kwargs={MID_STREAM_CONTINUATION_KWARG: forged},
+        )
+        assert result == deployments
+
+
+@pytest.mark.asyncio
+async def test_filter_keeps_only_prefill_capable_on_continuation():
+    check = ContinuationPrefillDeploymentCheck()
+    deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")]
+
+    result = await check.async_filter_deployments(
+        model="group",
+        healthy_deployments=deployments,
+        messages=None,
+        request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER},
+    )
+    assert [d["model_info"]["id"] for d in result] == ["a"]
+
+
+@pytest.mark.asyncio
+async def test_filter_empties_group_when_no_prefill_capable_deployment():
+    """No prefill-capable deployment -> empty result, so the router advances the
+    fallback chain and ultimately surfaces the original error."""
+    check = ContinuationPrefillDeploymentCheck()
+    deployments = [_deployment(NON_PREFILL_MODEL, "b"), _deployment(UNMAPPED_MODEL, "c")]
+
+    result = await check.async_filter_deployments(
+        model="group",
+        healthy_deployments=deployments,
+        messages=None,
+        request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER},
+    )
+    assert result == []
+
+
+@pytest.mark.asyncio
+async def test_filter_drops_deployments_whose_own_params_constrain_the_continuation():
+    check = ContinuationPrefillDeploymentCheck()
+    plain = _deployment(PREFILL_MODEL, "plain")
+    constrained = [
+        {"litellm_params": {"model": PREFILL_MODEL, **params}, "model_info": {"id": dep_id}}
+        for dep_id, params in (
+            ("json", {"response_format": {"type": "json_object"}}),
+            ("two_choices", {"extra_body": {"n": 2}}),
+            ("forced_tool", {"tool_choice": {"type": "function", "function": {"name": "f"}}}),
+            ("merged_reasoning", {"merge_reasoning_content_in_choices": True}),
+        )
+    ]
+
+    result = await check.async_filter_deployments(
+        model="group",
+        healthy_deployments=[plain, *constrained],
+        messages=None,
+        request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER},
+    )
+    assert [d["model_info"]["id"] for d in result] == ["plain"]
+
+
+def _text_chunk(text: str) -> litellm.ModelResponseStream:
+    return litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}])
+
+
+@pytest.mark.parametrize(
+    "prefill_content,fallback_texts,expected_texts",
+    [
+        ("Hello", ("", "Hel", "lo wor", "ld"), ("", "", " wor", "ld")),
+        ("Hello", ("Hello",), ("",)),
+        ("Hello", (" world",), (" world",)),
+        ("Hello", ("Hel", "p me"), ("Hel", "p me")),
+        ("Hello", ("Hel",), ("Hel",)),
+        ([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")),
+    ],
+    ids=[
+        "echo_split_across_chunks_is_dropped",
+        "exact_echo_then_end_is_dropped",
+        "continuation_without_echo_passes_through",
+        "near_echo_is_kept_in_order",
+        "stream_ending_mid_match_is_kept",
+        "content_block_prefill_echo_is_dropped",
+    ],
+)
+@pytest.mark.asyncio
+async def test_continuation_output_drops_only_a_full_echo_of_the_prefill(
+    prefill_content, fallback_texts, expected_texts
+):
+    output = ContinuationOutput(
+        "Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", ()
+    )
+
+    async def fallback_stream():
+        for text in fallback_texts:
+            yield _text_chunk(text)
+
+    released = [chunk async for chunk in output.splice(fallback_stream())]
+
+    assert tuple(chunk.choices[0].delta.content for chunk in released) == expected_texts
+
+
+@pytest.mark.parametrize(
+    "prefill,fallback_texts,expected_texts",
+    [
+        ("Transformers use ", ("self-attention",), ("self-attention",)),
+        ("Hello ", ("Hello ", " world"), ("", "world")),
+        ("Hello \n", ("\nworld",), ("\nworld",)),
+        ("A.\n\n", ("\n Hi", "\nthere"), (" Hi", "\nthere")),
+        ("\n\n", ("\nHi",), ("\nHi",)),
+        ("\nHello\n", ("\n",), ("",)),
+    ],
+    ids=[
+        "continuation_without_whitespace_passes_through",
+        "echo_then_re_emitted_space_are_both_dropped",
+        "different_whitespace_is_kept",
+        "seam_ends_at_the_first_divergence",
+        "whitespace_only_prefill_has_no_seam",
+        "re_emitted_whitespace_at_stream_end_is_dropped",
+    ],
+)
+@pytest.mark.asyncio
+async def test_continuation_output_drops_only_whitespace_the_client_already_has(
+    prefill, fallback_texts, expected_texts
+):
+    output = ContinuationOutput(prefill, {"role": "assistant", "content": prefill, "prefix": True}, "group", ())
+
+    async def fallback_stream():
+        for text in fallback_texts:
+            yield _text_chunk(text)
+
+    released = [chunk async for chunk in output.splice(fallback_stream())]
+
+    assert tuple(chunk.choices[0].delta.content for chunk in released) == expected_texts
+
+
+@pytest.mark.parametrize(
+    "request_kwargs,expected",
+    [
+        ({"max_tokens": 100}, {"max_tokens": 70}),
+        ({"max_tokens": 100, "max_completion_tokens": 40}, {"max_tokens": 70, "max_completion_tokens": 10}),
+        ({}, {}),
+        (
+            {"max_tokens": 100, "extra_body": {"max_tokens": 50, "top_k": 3}},
+            {"max_tokens": 70, "extra_body": {"max_tokens": 20, "top_k": 3}},
+        ),
+        ({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}),
+        ({"extra_body": {"top_k": 3}}, {}),
+    ],
+    ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"],
+)
+def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends(
+    request_kwargs, expected
+):
+    assert continuation_output_ceilings(request_kwargs, 30) == expected
+
+
+@pytest.mark.parametrize(
+    "request_kwargs",
+    [
+        {"max_tokens": 30},
+        {"max_tokens": 20},
+        {"max_tokens": 100, "max_completion_tokens": 30},
+        {"max_tokens": 100, "extra_body": {"max_tokens": 30}},
+    ],
+    ids=["max_tokens_met", "max_tokens_exceeded", "one_of_two_met", "extra_body_met"],
+)
+def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs):
+    assert continuation_output_ceilings(request_kwargs, 30) is None

tests/unit/test_router_model_cost_isolation.py

@@ -544,13 +544,14 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing():
     )
 
 
-def test_regex_lookaround_flag_stays_on_the_deployment_that_set_it() -> None:
-    """A deployment's ``supports_regex_lookaround`` override must not land on the shared
+@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"])
+def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None:
+    """A deployment's capability override must not land on the shared
     ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it."""
     backend_model = "bedrock/us.xai.grok-4.6"
     deploy_id = "grok-deploy-keep-regex"
 
-    builtin_flag = litellm.get_model_info(model=backend_model).get("supports_regex_lookaround")
+    builtin_flag = litellm.get_model_info(model=backend_model).get(flag)
     model_keys = {
         deploy_id: litellm.model_cost.get(deploy_id),
         backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)),
@@ -561,13 +562,13 @@ def test_regex_lookaround_flag_stays_on_the_deployment_that_set_it() -> None:
                 {
                     "model_name": "grok-keep-regex",
                     "litellm_params": {"model": backend_model},
-                    "model_info": {"id": deploy_id, "supports_regex_lookaround": not builtin_flag},
+                    "model_info": {"id": deploy_id, flag: not builtin_flag},
                 }
             ],
         )
 
-        assert litellm.model_cost[deploy_id]["supports_regex_lookaround"] is (not builtin_flag)
-        assert litellm.get_model_info(model=backend_model).get("supports_regex_lookaround") is builtin_flag
+        assert litellm.model_cost[deploy_id][flag] is (not builtin_flag)
+        assert litellm.get_model_info(model=backend_model).get(flag) is builtin_flag
     finally:
         _restore_model_cost_entries(model_keys)