From d5ce7dbb8cb8c387493ee5afba79c16f91ad9beb Mon Sep 17 00:00:00 2001 From: Deepanshu Pal <40927968+deepanshupal@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:45:05 +0530 Subject: [PATCH 1/3] fix(proxy)!: honor caller litellm_session_id in spend log session_id (#44199) * fix(responses): scope session history lookup to the referenced key * Implement SQLite-backed DB for session tests Added a SQLite-backed database class for testing and a new test to ensure session isolation between different API keys. * fix(proxy): honor caller litellm_session_id in spend log session_id Updated docstring to clarify session ID resolution logic for spend log rows. * test(proxy): caller litellm_session_id wins over the trace id * style(proxy): fix blank lines around new tests * fix(proxy): read the caller session from the logging payload for spend logs * fix(responses): match user and team too when replaying session history * fix(proxy): keep bridged Responses chains whole and scope session history by owner * fix(proxy): keep a metadata session next to a trace header and let session headers win * fix(proxy): send the trace-header session only to spend logs and let session headers win beside a trace header * fix(proxy): drop a client-sent caller session and keep it out of requester metadata * fix(proxy): keep the trace id as the spend log session when the caller session contains a NUL byte * fix(responses): keep a continued bridged chain's session under request_correlation_in_logs With the flag on, the caller session outranked the chain session the bridge sets for a continued Responses chain, so a chain whose session changed or first appeared on a later turn split. The continuation now logs under the chain's session and drops the turn's caller session key, as on main. Adds the /audit integration cells for the spend log caller session. * fix(responses): log a continued bridged chain under its session through the spend-log key only The previous commit set the chain session as the turn's litellm_session_id, which also moved the OTel conversation id of a continued turn away from its trace header. The bridge now writes the chain session into the internal spend-log session key of the metadata the spend log reads, so only the spend log session changes, as on main. Adds an OTel cell for a continued chain. * test(spend): cover bridged chains that share one caller session * test(spend): cover the caller session on configured pass-through routes * test(spend): type the SQLite fake and split stacked comprehensions in the caller session tests --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/litellm_pre_call_utils.py | 52 +- .../spend_tracking/spend_tracking_utils.py | 35 +- .../session_handler.py | 25 +- .../transformation.py | 26 +- .../spend/test_spend_log_caller_session.py | 783 ++++++++++++++++++ .../test_spend_tracking_utils.py | 80 +- .../unit/proxy/test_litellm_pre_call_utils.py | 88 ++ .../test_litellm_completion_responses.py | 67 ++ .../test_session_handler.py | 78 ++ 10 files changed, 1208 insertions(+), 27 deletions(-) create mode 100644 tests/integration/spend/test_spend_log_caller_session.py diff --git a/litellm/constants.py b/litellm/constants.py index 271e42ee661..eb03d034c7e 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1629,6 +1629,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" +CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( "Truncation is a DB storage safeguard. " diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 6bd31a42e9e..675bf0830f9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -27,6 +27,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm._uuid import uuid from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -187,6 +188,30 @@ def _caller_trace_field(data: Mapping[str, object], metadata_variable_name: str, return requester_value if isinstance(requester_value, str) and requester_value else None +def _caller_body_session_id(data: Mapping[str, object], metadata_variable_name: str) -> str | None: + top_level: Final = data.get("litellm_session_id") + if isinstance(top_level, str) and top_level: + return top_level + return _caller_trace_field(data, metadata_variable_name, "session_id") + + +def _caller_session_beside_trace_header( + normalized_headers: Mapping[str, str], data: Mapping[str, object], metadata_variable_name: str +) -> str | None: + session_headers: Final = {key: value for key, value in normalized_headers.items() if key != "x-litellm-trace-id"} + return get_chain_id_from_headers(session_headers) or _caller_body_session_id(data, metadata_variable_name) + + +def _drop_client_caller_session(request_body: Mapping[str, object]) -> None: + for metadata_key in ("metadata", "litellm_metadata"): + if isinstance(metadata := request_body.get(metadata_key), dict): + metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None) + + +def _requester_metadata_snapshot(metadata: Mapping[str, object]) -> Mapping[str, object]: + return copy.deepcopy({key: value for key, value in metadata.items() if key != CALLER_SESSION_ID_METADATA_KEY}) + + def _stampable_key_hash(user_api_key_dict: UserAPIKeyAuth) -> str | None: """Only proxy-validated keys are stamped, proven by the unforgeable via_virtual_key marker AND a known non-secret shape: the sha256 hex digest @@ -1623,6 +1648,8 @@ class LiteLLMProxyRequestSetup: from litellm.proxy._types import LitellmMetadataFromRequestHeaders metadata_from_headers: Final = LitellmMetadataFromRequestHeaders() + request_body: Final = cast(Mapping[str, object], data) # cast-ok: request body is a str-keyed JSON object + _drop_client_caller_session(request_body) spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers) if spend_logs_metadata is not None: metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata @@ -1631,6 +1658,9 @@ class LiteLLMProxyRequestSetup: # Finally update the requests metadata with the `metadata_from_headers` ######################################################################################### + normalized_headers: Final = MappingProxyType( + {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)} + ) agent_id_from_header: Final = headers.get("x-litellm-agent-id") # Explicit litellm headers take precedence; fall back to any x-*-session-id header. chain_id: Final = get_chain_id_from_headers(dict(headers)) @@ -1639,6 +1669,13 @@ class LiteLLMProxyRequestSetup: metadata_from_headers["agent_id"] = agent_id_from_header verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header) + spend_log_session_id: Final = ( + _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name) + if litellm.request_correlation_in_logs + and chain_id is not None + and chain_id == normalized_headers.get("x-litellm-trace-id") + else None + ) if chain_id: metadata_from_headers["trace_id"] = chain_id metadata_from_headers["session_id"] = chain_id @@ -1663,7 +1700,6 @@ class LiteLLMProxyRequestSetup: # - but lets a caller's existing traceparent/baggage headers (from # real OTel instrumentation) correlate with litellm's own logs instead # of generating an unrelated trace_id. - normalized_headers: Final = MappingProxyType({k.lower(): v for k, v in headers.items() if isinstance(k, str)}) if "litellm_trace_id" not in data: traceparent: Final = normalized_headers.get("traceparent") if isinstance(traceparent, str): @@ -1689,11 +1725,7 @@ class LiteLLMProxyRequestSetup: baggage: Final = normalized_headers.get("baggage") if isinstance(baggage, str): session_id_from_baggage: Final = _session_id_from_baggage(baggage) - caller_session_id: Final = _caller_trace_field( - cast(Mapping[str, object], data), # cast-ok: request body is a str-keyed JSON object - _metadata_variable_name, - "session_id", - ) + caller_session_id: Final = _caller_trace_field(request_body, _metadata_variable_name, "session_id") if session_id_from_baggage and ( caller_session_id is None or caller_session_id == session_id_from_baggage ): @@ -1702,7 +1734,11 @@ class LiteLLMProxyRequestSetup: verbose_proxy_logger.debug("Extracted session_id from W3C baggage header") if isinstance(data[_metadata_variable_name], dict): - data[_metadata_variable_name].update(metadata_from_headers) + data[_metadata_variable_name].update( + {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id} + if spend_log_session_id + else metadata_from_headers + ) return data @staticmethod @@ -2417,7 +2453,7 @@ async def add_litellm_data_to_request( # them — from leaking into requester_metadata where guardrails and audit # paths may read from it. if "metadata" in data and isinstance(data["metadata"], dict): - data[_metadata_variable_name]["requester_metadata"] = copy.deepcopy(data["metadata"]) + data[_metadata_variable_name]["requester_metadata"] = _requester_metadata_snapshot(data["metadata"]) if _metadata_variable_name == "litellm_metadata": data[_metadata_variable_name].update( _promoted_trace_control_fields( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 49644f1aa12..fe55092591e 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -33,6 +34,8 @@ from litellm.constants import ( from litellm.constants import ( MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB, ) +from litellm.integrations.otel.model.metadata import caller_session_id +from litellm.integrations.otel.model.trace_controls import TraceControls from litellm.litellm_core_utils.classifier_logging import ( CLASSIFIER_AUDIT_FIELDS, classifier_audit_fields, @@ -930,6 +933,16 @@ def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) - return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX) +def _caller_session_for_spend_log(kwargs: Mapping[str, object], metadata: Mapping[str, object] | None) -> str | None: + beside_trace_header: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) if metadata else None + caller_session: Final = ( + beside_trace_header + if isinstance(beside_trace_header, str) and beside_trace_header + else caller_session_id(kwargs, TraceControls()) + ) + return caller_session if caller_session and "\x00" not in caller_session else None + + def _get_session_id_for_spend_log( kwargs: Mapping[str, object], metadata: Mapping[str, object] | None, @@ -937,9 +950,12 @@ def _get_session_id_for_spend_log( omit_when_missing: bool, batch_trace_session_id: str | None = None, ) -> str | None: - """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may - be a copied trace id. Batch call types carry a deterministic session derived from the batch id, which outranks - the per-request trace ids because those differ between the create call and the cost poller's row.""" + """Resolve the session id for the spend log row: `omit` honors only metadata.session_id, + batch sessions outrank everything, then the conversation id the caller sent (also one an x-litellm-trace-id header + displaced from the request's session), then trace ids. + + A session minted by `missing_session_id: generate` or copied from the request's trace id is not a conversation, so + it yields to the trace id, which the Responses bridge points at the previous turn's session to keep a chain whole.""" if omit_when_missing: session_id: Final = metadata.get("session_id") if metadata else None return str(session_id) if session_id else None @@ -948,11 +964,14 @@ def _get_session_id_for_spend_log( if batch_trace_session_id is not None: return batch_trace_session_id - if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None: - return str(standard_logging_payload.get("trace_id")) - if kwargs.get("litellm_trace_id") is not None: - return str(kwargs.get("litellm_trace_id")) - return str(uuid.uuid4()) + if litellm.request_correlation_in_logs and (caller_session := _caller_session_for_spend_log(kwargs, metadata)): + return caller_session + match standard_logging_payload: + case {"trace_id": trace_id} if trace_id is not None: + return str(trace_id) + case _: + litellm_trace_id: Final = kwargs.get("litellm_trace_id") + return str(litellm_trace_id) if litellm_trace_id is not None else str(uuid.uuid4()) def _get_request_duration_ms(start_time: datetime, end_time: datetime) -> int | None: diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 732c32ff2cb..e78ec5e5d3e 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -281,7 +281,13 @@ class ResponsesSessionHandler: SQL query - SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id + SELECT session_id, api_key, "user", team_id FROM spend_logs WHERE request_id = previous_response_id, then + SELECT * FROM spend_logs with that session_id and the same owner + + Only rows with the referenced response's owner are returned: the same user and team, and for a key with + neither, the same key. The session id can be caller supplied, so matching on it alone would mix owners. A + user's or team's regenerated or second key keeps the history; a key with neither does not, and turns written + by another owner (an admin continuing the chain) are not replayed. A just-finished turn gets a short second chance: the worker that served it may still be writing its spend log when the follow-up arrives, and an empty result @@ -303,13 +309,24 @@ class ResponsesSessionHandler: query: Final = """ WITH matching_session AS ( - SELECT session_id + SELECT session_id, api_key, "user", team_id FROM "LiteLLM_SpendLogs" WHERE request_id = $1 ) SELECT * - FROM "LiteLLM_SpendLogs" - WHERE session_id IN (SELECT session_id FROM matching_session) + FROM "LiteLLM_SpendLogs" AS logs + WHERE EXISTS ( + SELECT 1 + FROM matching_session + WHERE matching_session.session_id = logs.session_id + AND matching_session."user" IS NOT DISTINCT FROM logs."user" + AND matching_session.team_id IS NOT DISTINCT FROM logs.team_id + AND ( + COALESCE(matching_session."user", '') <> '' + OR COALESCE(matching_session.team_id, '') <> '' + OR matching_session.api_key = logs.api_key + ) + ) ORDER BY "endTime" ASC; """ diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 2e7fb6da194..587cdfcee25 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -39,7 +39,11 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.caching import InMemoryCache -from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + REDACTED_BY_LITELLM, + REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, +) from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( @@ -563,6 +567,14 @@ class ChatCompletionSession(TypedDict, total=False): ########### End of Initialize Classes used for Responses API ########### +def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata" + metadata: Final = request_kwargs.get(key) + if not isinstance(metadata, dict): + return MappingProxyType({}) + return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}} + + class LiteLLMCompletionResponsesConfig: @staticmethod def get_supported_openai_params(model: str) -> list: @@ -1078,8 +1090,6 @@ class LiteLLMCompletionResponsesConfig: else: # Both are empty - this likely means function_call_output had empty/invalid call_id # Provide a helpful error message - import litellm - raise litellm.BadRequestError( message=( f"Unable to create messages for completion request. " @@ -1094,8 +1104,14 @@ class LiteLLMCompletionResponsesConfig: ) litellm_completion_request["messages"] = combined_messages - litellm_completion_request["litellm_trace_id"] = chat_completion_session.get("litellm_session_id") - return litellm_completion_request + chain_session: Final = chat_completion_session.get("litellm_session_id") + litellm_completion_request["litellm_trace_id"] = chain_session + chain_session_metadata: Final[Mapping[str, object]] = ( + _chain_session_metadata(chain_session, request_kwargs) + if chain_session and litellm.request_correlation_in_logs + else MappingProxyType({}) + ) + return {**litellm_completion_request, **chain_session_metadata} @staticmethod def _transform_response_input_param_to_chat_completion_message( diff --git a/tests/integration/spend/test_spend_log_caller_session.py b/tests/integration/spend/test_spend_log_caller_session.py new file mode 100644 index 00000000000..213583d21a1 --- /dev/null +++ b/tests/integration/spend/test_spend_log_caller_session.py @@ -0,0 +1,783 @@ +import itertools +import json +import re +import threading +import uuid +from urllib.parse import urlsplit +from collections import deque +from collections.abc import Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final + +import anthropic +import httpx +import openai +import psutil +import pytest +import yaml +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + gateway_from_environment, + list_value, + object_value, + string_value, +) +from integration._support.budget import readiness_seconds +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(2 * readiness_seconds(2) + 2 * graceful_stop_seconds() + 240) + +MARKER: Final = re.compile(rb"callersess-[0-9a-f]{32}") +CALLER_SESSION_KEY: Final = "litellm_caller_session_id" +PATHS: Final = MappingProxyType( + {"chat": "/v1/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"} +) +ENDPOINT_STREAMS: Final = tuple(itertools.product(PATHS, (False, True))) +Headers = Mapping[str, str] | Sequence[tuple[str, str]] + + +def _marker() -> str: + return "callersess-" + uuid.uuid4().hex + + +def _token(label: str) -> str: + return f"{label}-{uuid.uuid4().hex}" + + +def _frame(payload: Mapping[str, JsonValue]) -> bytes: + return b"data: " + json.dumps(payload).encode() + b"\n\n" + + +def _chat_reply(text: str, stream: bool) -> Reply: + identity: Final = "chatcmpl-" + uuid.uuid4().hex + usage: Final[dict[str, JsonValue]] = {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"} + ], + "usage": usage, + } + ).encode() + ) + chunk: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + } + return Reply( + content_type="text/event-stream", + chunks=( + _frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": text}}]}), + _frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": usage}), + b"data: [DONE]\n\n", + ), + ) + + +def _upstream(request: Request) -> Reply: + markers: Final = tuple(found.group(0).decode() for found in MARKER.finditer(request.body)) + if request.method != "POST" or not request.target.endswith("/chat/completions") or not markers: + return Reply(status=404, body=b'{"error":"unexpected request"}') + return _chat_reply(f"answer {markers[-1]}", JSON_OBJECT.validate_json(request.body).get("stream") is True) + + +def _user_markers(body: Mapping[str, JsonValue]) -> tuple[str, ...]: + user_contents: Final = tuple( + json.dumps(object_value(message)["content"]) + for message in list_value(body.get("messages", [])) + if object_value(message)["role"] == "user" + ) + return tuple(found.group(0).decode() for content in user_contents if (found := MARKER.search(content.encode()))) + + +def _spans(batch: Request) -> Iterator[dict[str, JsonValue]]: + for resource in list_value(JSON_OBJECT.validate_json(batch.body)["resourceSpans"]): + for scope in list_value(object_value(resource)["scopeSpans"]): + yield from (object_value(span) for span in list_value(object_value(scope)["spans"])) + + +def _attributes(span: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return { + string_value(object_value(attribute)["key"]): object_value(attribute)["value"] + for attribute in list_value(span.get("attributes", [])) + } + + +def _batch_attributes(batches: Sequence[Request]) -> Iterator[dict[str, JsonValue]]: + return (_attributes(span) for span in itertools.chain.from_iterable(_spans(batch) for batch in batches)) + + +@dataclass(frozen=True, slots=True) +class Collector: + wire: Wire + accepted: deque[Request] + guard: threading.Lock + + def span_attributes(self, response_id: str) -> tuple[dict[str, JsonValue], ...]: + with self.guard: + batches: Final = tuple(self.accepted) + attributes: Final = tuple(_batch_attributes(batches)) + return tuple( + found + for found in attributes + if object_value(found.get("gen_ai.response.id", {})).get("stringValue") == response_id + ) + + def conversation_ids(self, conversations: frozenset[str]) -> tuple[str, ...]: + with self.guard: + batches: Final = tuple(self.accepted) + responded: Final = ( + object_value(found.get("gen_ai.conversation.id", {})).get("stringValue") + for found in _batch_attributes(batches) + if "gen_ai.response.id" in found + ) + return tuple( + conversation + for conversation in responded + if isinstance(conversation, str) and conversation in conversations + ) + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + scenario: Scenario + model: str + upstream: Wire + process: OwnedProxy + + def body(self, endpoint: str, marker: str, extra: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + match endpoint: + case "chat": + return {"model": self.model, "messages": [{"role": "user", "content": marker}], **extra} + case "messages": + return { + "model": self.model, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker}], + **extra, + } + case _: + return {"model": self.model, "input": marker, **extra} + + def post( + self, + endpoint: str, + marker: str, + fields: Mapping[str, JsonValue] = MappingProxyType({}), + *, + stream: bool = False, + headers: Headers = (), + key: str | None = None, + ) -> httpx.Response: + payload: Final = self.body(endpoint, marker, fields) | ({"stream": True} if stream else {}) + request_headers: Final = ( + ("Authorization", f"Bearer {self.proxy.key if key is None else key}"), + *(headers.items() if isinstance(headers, Mapping) else headers), + ) + with self.proxy.client.stream("POST", PATHS[endpoint], json=payload, headers=request_headers) as response: + response.read() + return response + + def send( + self, + endpoint: str, + marker: str, + fields: Mapping[str, JsonValue] = MappingProxyType({}), + *, + stream: bool = False, + headers: Headers = (), + key: str | None = None, + ) -> httpx.Response: + response: Final = self.post(endpoint, marker, fields, stream=stream, headers=headers, key=key) + assert response.status_code == 200, response.text + return response + + def upstream_bodies(self, marker: str) -> tuple[dict[str, JsonValue], ...]: + bodies: Final = tuple(JSON_OBJECT.validate_json(request.body) for request in self.upstream.drain()) + return tuple(body for body in bodies if _user_markers(body)[-1:] == (marker,)) + + +@dataclass(frozen=True, slots=True) +class Rigs: + flag_off: Rig + flag_on: Rig + sink: Collector + + +def _spend_rows(call_id: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT session_id, request_id, metadata::text AS metadata_text FROM "LiteLLM_SpendLogs" ' + "WHERE litellm_call_id=%s", + (call_id,), + ) + + +def _spend_row(response: httpx.Response) -> dict[str, JsonValue]: + call_id: Final = response.headers["x-litellm-call-id"] + rows: Final = eventually(lambda: _spend_rows(call_id), lambda values: len(values) >= 1, seconds=70) + assert len(rows) == 1, rows + return rows[0] + + +def _session(response: httpx.Response) -> JsonValue: + return _spend_row(response)["session_id"] + + +def _model(scenario: Scenario, upstream: Wire) -> str: + return scenario.model( + model="hosted_vllm/gpt-4o-mini", + api_base=f"{upstream.url}/v1", + use_chat_completions_api=True, + input_cost_per_token=0.001, + output_cost_per_token=0.002, + ) + + +def _rig_config( + directory: Path, name: str, litellm_settings: Mapping[str, JsonValue], extra: Mapping[str, JsonValue] +) -> Path: + shared: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **shared, + "general_settings": {**object_value(shared["general_settings"]), "store_prompts_in_spend_logs": True}, + "litellm_settings": {**object_value(shared["litellm_settings"]), **litellm_settings}, + **extra, + } + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rigs(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rigs]: + accepted: Final[deque[Request]] = deque() # mutable-ok: sink thread appends each accepted OTLP batch + guard: Final = threading.Lock() + + def sink(request: Request) -> Reply: + with guard: + accepted.append(request) + return Reply() + + directory: Final = tmp_path_factory.mktemp("caller-session") + with ExitStack() as stack: + upstream: Final = stack.enter_context(wire_server(_upstream)) + collector: Final = Collector(stack.enter_context(wire_server(sink)), accepted, guard) + shared: Final = stack.enter_context(gateway_from_environment()) + flag_off: Final = stack.enter_context( + owned_proxy_process(shared, directory, {}, config=_rig_config(directory, "flag-off", {}, {}), workers=2) + ) + flag_on: Final = stack.enter_context( + owned_proxy_process( + shared, + directory, + {"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"}, + config=_rig_config( + directory, + "flag-on", + {"request_correlation_in_logs": True, "callbacks": ["otel"]}, + { + "callback_settings": { + "otel": {"exporter": "http/json", "endpoint": collector.wire.url, "mapper_names": ["genai"]} + } + }, + ), + workers=2, + ) + ) + off_scenario: Final = stack.enter_context(flag_off.gateway.scenario()) + on_scenario: Final = stack.enter_context(flag_on.gateway.scenario()) + yield Rigs( + Rig(flag_off.gateway, off_scenario, _model(off_scenario, upstream), upstream, flag_off), + Rig(flag_on.gateway, on_scenario, _model(on_scenario, upstream), upstream, flag_on), + collector, + ) + + +@pytest.mark.parametrize(("endpoint", "stream"), ENDPOINT_STREAMS) +def test_spend_log_session_is_the_body_session_beside_a_trace_header(rigs: Rigs, endpoint: str, stream: bool) -> None: + marker: Final = _marker() + session: Final = _token("convo") + response: Final = rigs.flag_on.send( + endpoint, + marker, + stream=stream, + headers={"x-litellm-trace-id": _token("trace")}, + fields={"litellm_session_id": session}, + ) + + row: Final = _spend_row(response) + upstream: Final = rigs.flag_on.upstream_bodies(marker) + + assert row["session_id"] == session, row + assert CALLER_SESSION_KEY not in string_value(row["metadata_text"]), row + assert len(upstream) == 1, upstream + assert CALLER_SESSION_KEY not in json.dumps(upstream[0]), upstream + + +def test_sdk_clients_record_the_body_session_beside_a_trace_header(rigs: Rigs) -> None: + rig: Final = rigs.flag_on + base_url: Final = str(rig.proxy.client.base_url) + traces: Final = tuple(_token("trace") for _ in range(3)) + sessions: Final = tuple(_token("convo") for _ in range(3)) + markers: Final = tuple(_marker() for _ in range(3)) + chat_client: Final = openai.OpenAI(base_url=base_url + "/v1", api_key=rig.proxy.key, max_retries=0) + messages_client: Final = anthropic.Anthropic(base_url=base_url, api_key=rig.proxy.key, max_retries=0) + + with chat_client.chat.completions.with_streaming_response.create( + model=rig.model, + messages=[{"role": "user", "content": markers[0]}], + stream=True, + extra_headers={"x-litellm-trace-id": traces[0]}, + extra_body={"litellm_session_id": sessions[0]}, + ) as chat_stream: + chat_call: Final = chat_stream.headers["x-litellm-call-id"] + chat_text: Final = "".join( + chunk.choices[0].delta.content or "" for chunk in chat_stream.parse() if chunk.choices + ) + message: Final = messages_client.messages.with_raw_response.create( + model=rig.model, + max_tokens=16, + messages=[{"role": "user", "content": markers[1]}], + extra_headers={"x-litellm-trace-id": traces[1]}, + extra_body={"litellm_session_id": sessions[1]}, + ) + with chat_client.responses.with_streaming_response.create( + model=rig.model, + input=markers[2], + stream=True, + extra_headers={"x-litellm-trace-id": traces[2]}, + extra_body={"litellm_session_id": sessions[2]}, + ) as responses_stream: + responses_call: Final = responses_stream.headers["x-litellm-call-id"] + completed: Final = tuple(event for event in responses_stream.parse() if event.type == "response.completed") + + first_block: Final = message.parse().content[0] + assert chat_text == f"answer {markers[0]}", chat_text + assert isinstance(first_block, anthropic.types.TextBlock) and first_block.text == f"answer {markers[1]}", message + assert len(completed) == 1, completed + calls: Final = (chat_call, message.headers["x-litellm-call-id"], responses_call) + landed: Final = tuple( + eventually(lambda call=call: _spend_rows(call), lambda values: len(values) == 1, seconds=70)[0]["session_id"] + for call in calls + ) + assert landed == sessions, (landed, calls) + + +def test_conversation_span_keeps_the_trace_header_and_no_internal_key(rigs: Rigs) -> None: + trace: Final = _token("trace") + response: Final = rigs.flag_on.send( + "chat", _marker(), headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": _token("convo")} + ) + response_id: Final = string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + spans: Final = eventually(lambda: rigs.sink.span_attributes(response_id), lambda found: len(found) == 1, seconds=30) + + assert spans[0]["gen_ai.conversation.id"] == {"stringValue": trace}, spans + assert CALLER_SESSION_KEY not in json.dumps(spans[0]), spans + + +def test_a_continued_chains_conversation_spans_keep_each_turns_trace_header(rigs: Rigs) -> None: + traces: Final = (_token("trace"), _token("trace")) + first: Final = _turn(rigs.flag_on, _marker(), headers={"x-litellm-trace-id": traces[0]}) + _turn( + rigs.flag_on, + _marker(), + first, + headers={"x-litellm-trace-id": traces[1]}, + fields={"litellm_session_id": _token("convo")}, + ) + + conversations: Final = eventually( + lambda: rigs.sink.conversation_ids(frozenset(traces)), lambda found: len(found) == 2, seconds=30 + ) + + assert sorted(conversations) == sorted(traces), conversations + + +@pytest.mark.parametrize("endpoint", list(PATHS)) +def test_flag_off_keeps_the_trace_id_beside_a_body_session(rigs: Rigs, endpoint: str) -> None: + trace: Final = _token("trace") + + response: Final = rigs.flag_off.send( + endpoint, _marker(), headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": _token("convo")} + ) + + assert _session(response) == trace + + +@pytest.mark.parametrize("endpoint", list(PATHS)) +def test_response_cache_hit_records_the_body_session_beside_a_trace_header(rigs: Rigs, endpoint: str) -> None: + marker: Final = _marker() + sessions: Final = (_token("convo"), _token("convo")) + responses: Final = tuple( + rigs.flag_on.send( + endpoint, marker, headers={"x-litellm-trace-id": _token("trace")}, fields={"litellm_session_id": session} + ) + for session in sessions + ) + + assert tuple(_session(response) for response in responses) == sessions + assert len(rigs.flag_on.upstream_bodies(marker)) == 1 + + +def test_metadata_session_beside_a_new_trace_per_turn_groups_both_turns(rigs: Rigs) -> None: + session: Final = _token("convo") + responses: Final = tuple( + rigs.flag_on.send( + "chat", + _marker(), + headers={"x-litellm-trace-id": _token("trace")}, + fields={"metadata": {"session_id": session}}, + ) + for _ in range(2) + ) + + assert tuple(_session(response) for response in responses) == (session, session) + now: Final = datetime.now(timezone.utc) + page: Final = rigs.flag_on.proxy.get( + "/spend/logs/ui", + params={ + "session_id": session, + "start_date": (now - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + "end_date": (now + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + }, + ) + listed: Final = sorted(string_value(object_value(row)["request_id"]) for row in list_value(page["data"])) + assert listed == sorted( + string_value(JSON_OBJECT.validate_json(response.content)["id"]) for response in responses + ), page + + +def _shape(shape: str, trace: str, session: str, header: str) -> tuple[Headers, dict[str, JsonValue], str]: + traced: Final = {"x-litellm-trace-id": trace} + match shape: + case "session-header-wins": + return {"x-litellm-session-id": header}, {"litellm_session_id": session}, header + case "trace-header-only": + return traced, {}, trace + case "body-session-only": + return {}, {"litellm_session_id": session}, session + case "int-session": + return traced, {"litellm_session_id": 7}, trace + case "list-session": + return traced, {"litellm_session_id": [session]}, trace + case "empty-session": + return traced, {"litellm_session_id": ""}, trace + case "null-session": + return traced, {"litellm_session_id": None}, trace + case "nul-session": + return traced, {"litellm_session_id": session + "\x00x"}, trace + case "two-trace-headers": + return ( + (("x-litellm-trace-id", trace), ("x-litellm-trace-id", header)), + {"litellm_session_id": session}, + session, + ) + case _: + return traced, {"litellm_session_id": session + "s" * 5120}, session + "s" * 5120 + + +@pytest.mark.parametrize( + "shape", + [ + "session-header-wins", + "trace-header-only", + "body-session-only", + "int-session", + "list-session", + "empty-session", + "null-session", + "nul-session", + "two-trace-headers", + "5kb-session", + ], +) +def test_spend_log_session_for_each_session_shape(rigs: Rigs, shape: str) -> None: + headers, fields, expected = _shape(shape, _token("trace"), _token("convo"), _token("header")) + neighbor_trace: Final = _token("neighbor") + + response: Final = rigs.flag_on.send("chat", _marker(), fields, headers=headers) + neighbor: Final = rigs.flag_on.send("chat", _marker(), headers={"x-litellm-trace-id": neighbor_trace}) + + assert _session(response) == expected + assert _session(neighbor) == neighbor_trace + + +@pytest.mark.parametrize("endpoint", ["chat", "messages"]) +def test_client_sent_internal_session_key_is_ignored(rigs: Rigs, endpoint: str) -> None: + forged: Final = _token("forged") + trace: Final = _token("trace") + forged_metadata: Final[dict[str, JsonValue]] = { + "metadata" if endpoint == "chat" else "litellm_metadata": {CALLER_SESSION_KEY: forged} + } + + untraced: Final = rigs.flag_on.send(endpoint, _marker(), forged_metadata) + traced: Final = rigs.flag_on.send(endpoint, _marker(), forged_metadata, headers={"x-litellm-trace-id": trace}) + + assert _session(untraced) not in (forged, None) + assert _session(traced) == trace + + +def test_identical_requests_each_log_once_with_the_body_session(rigs: Rigs) -> None: + session: Final = _token("convo") + trace: Final = _token("trace") + marker: Final = _marker() + + responses: Final = tuple( + rigs.flag_on.send("chat", marker, headers={"x-litellm-trace-id": trace}, fields={"litellm_session_id": session}) + for _ in range(3) + ) + + assert len({response.headers["x-litellm-call-id"] for response in responses}) == 3 + assert tuple(_session(response) for response in responses) == (session, session, session) + + +def _pass_through_route(rig: Rig, target: str) -> str: + path: Final = f"/caller-session-{uuid.uuid4().hex}" + created: Final = rig.proxy.post("/config/pass_through_endpoint", {"path": path, "target": target, "auth": True}) + endpoint_id: Final = string_value(object_value(list_value(created["endpoints"])[0])["id"]) + rig.scenario.cleanups.callback( + lambda: rig.proxy.request("DELETE", "/config/pass_through_endpoint", params={"endpoint_id": endpoint_id}) + ) + return path + + +@pytest.mark.parametrize("field", ["litellm_session_id", "metadata"]) +def test_pass_through_spend_log_session_is_the_body_session_beside_a_trace_header(rigs: Rigs, field: str) -> None: + marker: Final = _marker() + session: Final = _token("convo") + rig: Final = rigs.flag_on + path: Final = _pass_through_route(rig, f"{rig.upstream.url}/v1/chat/completions") + key: Final = rig.scenario.key(allowed_passthrough_routes=[path]) + fields: Final[dict[str, JsonValue]] = ( + {"litellm_session_id": session} if field == "litellm_session_id" else {"metadata": {"session_id": session}} + ) + + response: Final = rig.proxy.request( + "POST", + path, + rig.body("chat", marker, fields), + key=key, + headers={"x-litellm-trace-id": _token("trace")}, + ) + + rows: Final = eventually( + lambda: read_rows( + 'SELECT session_id FROM "LiteLLM_SpendLogs" WHERE api_key=%s', (sha256(key.encode()).hexdigest(),) + ), + lambda found: len(found) >= 1, + seconds=70, + ) + assert response.status_code == 200, response.text + assert f"answer {marker}" in response.text + assert len(rig.upstream_bodies(marker)) == 1 + assert rows == [{"session_id": session}] + + +def _turn( + rig: Rig, + marker: str, + previous: str | None = None, + *, + key: str | None = None, + headers: Headers = (), + fields: Mapping[str, JsonValue] = MappingProxyType({}), +) -> str: + chained: Final[dict[str, JsonValue]] = {"previous_response_id": previous} if previous else {} + response: Final = rig.send("responses", marker, {**chained, **fields}, headers=headers, key=key) + _spend_row(response) + return string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + +def _turn_fields(shape: str, session: str) -> tuple[dict[str, JsonValue], dict[str, JsonValue]]: + match shape: + case "session-every-turn": + return {"litellm_session_id": session}, {"litellm_session_id": session} + case "session-from-turn-2": + return {}, {"litellm_session_id": session} + case _: + return {"litellm_session_id": _token("convo")}, {"litellm_session_id": _token("convo")} + + +@pytest.mark.parametrize( + ("correlated", "shape", "replays_turn_1"), + [ + (True, "session-every-turn", True), + (True, "session-from-turn-2", True), + (True, "new-session-per-turn", True), + (False, "session-from-turn-2", False), + ], +) +def test_bridged_chain_history(rigs: Rigs, correlated: bool, shape: str, replays_turn_1: bool) -> None: + rig: Final = rigs.flag_on if correlated else rigs.flag_off + first_fields, second_fields = _turn_fields(shape, _token("convo")) + markers: Final = (_marker(), _marker(), _marker()) + first: Final = _turn(rig, markers[0], headers={"x-litellm-trace-id": _token("trace")}, fields=first_fields) + second: Final = _turn(rig, markers[1], first, headers={"x-litellm-trace-id": _token("trace")}, fields=second_fields) + + _turn(rig, markers[2], second) + + replayed: Final = rig.upstream_bodies(markers[2]) + assert len(replayed) == 1, replayed + assert _user_markers(replayed[0]) == (markers if replays_turn_1 else markers[1:]) + + +def test_bridged_chains_sharing_a_metadata_session_replay_each_others_turns(rigs: Rigs) -> None: + shared: Final[dict[str, JsonValue]] = {"metadata": {"session_id": _token("convo")}} + markers: Final = (_marker(), _marker(), _marker()) + first: Final = _turn(rigs.flag_on, markers[0], fields=shared) + _turn(rigs.flag_on, markers[1], fields=shared) + + _turn(rigs.flag_on, markers[2], first) + + replayed: Final = rigs.flag_on.upstream_bodies(markers[2]) + assert len(replayed) == 1, replayed + assert _user_markers(replayed[0]) == markers + + +@dataclass(frozen=True, slots=True) +class Owners: + first: str + second: str + + +def _owners(scenario: Scenario, model: str, shape: str) -> Owners: + user: Final = _token("user") + match shape: + case "other-user": + return Owners( + scenario.key(user_id=user, models=[model]), scenario.key(user_id=_token("user"), models=[model]) + ) + case "same-user": + return Owners(scenario.key(user_id=user, models=[model]), scenario.key(user_id=user, models=[model])) + case "same-team": + team: Final = scenario.team() + return Owners(scenario.key(team_id=team, models=[model]), scenario.key(team_id=team, models=[model])) + case _: + return Owners(scenario.key(models=[model]), scenario.key(models=[model])) + + +@pytest.mark.parametrize( + ("shape", "replays_first_owner"), + [("other-user", False), ("same-user", True), ("same-team", True), ("ownerless-keys", False)], +) +def test_bridged_history_follows_the_previous_responses_owner( + rigs: Rigs, shape: str, replays_first_owner: bool +) -> None: + rig: Final = rigs.flag_off + owners: Final = _owners(rig.scenario, rig.model, shape) + session: Final = _token("convo") + markers: Final = (_marker(), _marker(), _marker()) + _turn(rig, markers[0], key=owners.first, fields={"litellm_session_id": session}) + second: Final = _turn(rig, markers[1], key=owners.second, fields={"litellm_session_id": session}) + + _turn(rig, markers[2], second, key=owners.second) + + replayed: Final = rig.upstream_bodies(markers[2]) + assert len(replayed) == 1, replayed + assert _user_markers(replayed[0]) == (markers if replays_first_owner else markers[1:]) + + +BURST: Final = ENDPOINT_STREAMS * 4 + + +def test_concurrent_burst_across_endpoints_logs_each_session_once(rigs: Rigs) -> None: + sessions: Final = tuple(_token("convo") for _ in BURST) + + def one(index: int) -> httpx.Response: + endpoint, stream = BURST[index] + return rigs.flag_on.send( + endpoint, + _marker(), + stream=stream, + headers={"x-litellm-trace-id": _token("trace")}, + fields={"litellm_session_id": sessions[index]}, + ) + + with ThreadPoolExecutor(max_workers=12) as pool: + responses: Final = tuple(pool.map(one, range(len(BURST)))) + + assert tuple(_session(response) for response in responses) == sessions + + +def _open_upstream_connections(pid: int, upstream: Wire) -> int: + port: Final = urlsplit(upstream.url).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +def _post_or_none(rig: Rig, session: str) -> httpx.Response | None: + try: + return rig.post( + "chat", _marker(), headers={"x-litellm-trace-id": _token("trace")}, fields={"litellm_session_id": session} + ) + except httpx.HTTPError: + return None + + +def test_killing_one_worker_mid_burst_keeps_logging_the_body_session(rigs: Rigs) -> None: + release: Final = threading.Event() + arrived: Final[SimpleQueue[bytes]] = SimpleQueue() + + def held(request: Request) -> Reply: + arrived.put(request.body) + assert release.wait(timeout=60), "the burst was never released" + return _upstream(request) + + process: Final = rigs.flag_on.process + sessions: Final = tuple(_token("convo") for _ in range(20)) + after_session: Final = _token("convo") + with wire_server(held) as wire: + rig: Final = replace(rigs.flag_on, model=_model(rigs.flag_on.scenario, wire), upstream=wire) + workers: Final = eventually(process.worker_pids, lambda found: len(found) == 2, seconds=30) + with ThreadPoolExecutor(max_workers=len(sessions)) as pool: + pending: Final = tuple(pool.submit(_post_or_none, rig, session) for session in sessions) + eventually(arrived.qsize, lambda size: size == len(sessions), seconds=60) + held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire) for pid in workers}) + victim, survivor = sorted(workers, key=held_by.__getitem__) + with process.replacing_worker(victim) as kill: + kill.freeze() + kill() + release.set() + results: Final = tuple(future.result() for future in pending) + after: Final = rig.send( + "chat", + _marker(), + headers={"x-litellm-trace-id": _token("trace")}, + fields={"litellm_session_id": after_session}, + ) + + served: Final = tuple( + (response, session) + for response, session in zip(results, sessions, strict=True) + if response is not None and response.status_code == 200 + ) + assert sum(held_by.values()) == len(sessions), held_by + assert len(served) == held_by[survivor] >= len(sessions) // 2, (held_by, len(served)) + assert _session(after) == after_session + assert tuple(_session(response) for response, _ in served) == tuple(session for _, session in served) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 7291f49af99..de2676bafaa 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -14,8 +14,8 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils -from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL -from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL +from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -243,6 +243,82 @@ def test_batch_session_outranks_the_per_request_trace_id(): assert session_id == "batch-uid-1" +def _chat_session_id( + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, object], + correlation_in_logs: bool = True, + batch_trace_session_id: str | None = None, + metadata: Mapping[str, object] = MappingProxyType({}), +) -> str | None: + monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs) + return _get_session_id_for_spend_log( + kwargs={"litellm_trace_id": "trace-abc", "litellm_params": litellm_params}, + metadata=metadata, + standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING, + omit_when_missing=False, + batch_trace_session_id=batch_trace_session_id, + ) + + +_CALLER_SESSION_PARAMS: Final = MappingProxyType({"litellm_session_id": "sess-1", "metadata": {"trace_id": "otel-1"}}) + + +def test_caller_session_wins_over_the_per_request_trace_id(monkeypatch: pytest.MonkeyPatch): + assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS) == "sess-1" + + +def test_batch_session_outranks_a_caller_session(monkeypatch: pytest.MonkeyPatch): + assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, batch_trace_session_id="batch-uid-1") == "batch-uid-1" + + +def test_caller_session_keeps_the_legacy_trace_id_with_correlation_logging_off(monkeypatch: pytest.MonkeyPatch): + assert _chat_session_id(monkeypatch, _CALLER_SESSION_PARAMS, correlation_in_logs=False) == "trace-abc" + + +@pytest.mark.parametrize(("correlation_in_logs", "expected_session_id"), [(True, "convo-1"), (False, "trace-abc")]) +def test_session_displaced_by_a_trace_header_wins_only_with_correlation_logging( + monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, expected_session_id: str +): + header_chain: Final = {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc", "session_id": "trace-abc"}} + session_id: Final = _chat_session_id( + monkeypatch, + header_chain, + correlation_in_logs=correlation_in_logs, + metadata={CALLER_SESSION_ID_METADATA_KEY: "convo-1"}, + ) + assert session_id == expected_session_id + + +@pytest.mark.parametrize( + ("litellm_params", "metadata"), + [ + pytest.param({"litellm_session_id": "convo\x00x"}, {}, id="body-session"), + pytest.param({"litellm_session_id": "trace-abc"}, {CALLER_SESSION_ID_METADATA_KEY: "convo\x00x"}, id="beside-trace-header"), + ], +) +def test_caller_session_postgres_cannot_store_keeps_the_trace_id( + monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object], metadata: Mapping[str, object] +): + assert _chat_session_id(monkeypatch, litellm_params, metadata=metadata) == "trace-abc" + + +@pytest.mark.parametrize( + "litellm_params", + [ + pytest.param({"metadata": {}}, id="no-session"), + pytest.param( + {"litellm_session_id": "minted-1", "metadata": {"session_id": "minted-1", SESSION_ID_GENERATED_METADATA_KEY: True}}, + id="minted-by-generate-policy", + ), + pytest.param({"litellm_session_id": "job-7", "metadata": {"trace_id": "job-7"}}, id="copied-from-trace-id"), + ], +) +def test_session_that_is_not_a_caller_conversation_keeps_the_trace_id( + monkeypatch: pytest.MonkeyPatch, litellm_params: Mapping[str, object] +): + assert _chat_session_id(monkeypatch, litellm_params) == "trace-abc" + + def test_omit_policy_still_suppresses_batch_sessions(): session_id: Final = _get_session_id_for_spend_log( kwargs={}, diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 96c11be2650..93202d93566 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -17,6 +17,7 @@ from starlette.datastructures import Headers import litellm from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, SERVER_STREAMING_CLASSIFICATION_KEY, SERVER_STREAMING_CLASSIFICATION_MARKER, @@ -3308,6 +3309,93 @@ def test_add_litellm_metadata_from_request_headers_x_litellm_trace_id_sets_chain assert data["litellm_trace_id"] == "foo" +@pytest.mark.parametrize(("correlation_in_logs", "expected_spend_log_session"), [(True, "convo-1"), (False, None)]) +@pytest.mark.parametrize("session_in_metadata", [False, True], ids=["top-level", "metadata"]) +def test_x_litellm_trace_id_keeps_the_body_session_for_spend_logs_only( + monkeypatch: pytest.MonkeyPatch, + correlation_in_logs: bool, + expected_spend_log_session: str | None, + session_in_metadata: bool, +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", correlation_in_logs) + metadata: Final[dict[str, str]] = {"session_id": "convo-1"} if session_in_metadata else {} + data: Final[dict[str, object]] = ( + {"metadata": metadata} if session_in_metadata else {"metadata": metadata, "litellm_session_id": "convo-1"} + ) + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers={"x-litellm-trace-id": "req-trace-1"}, data=data, _metadata_variable_name="metadata" + ) + assert metadata.get(CALLER_SESSION_ID_METADATA_KEY) == expected_spend_log_session + assert data["litellm_session_id"] == "req-trace-1" + assert metadata["session_id"] == "req-trace-1" + assert data["litellm_trace_id"] == "req-trace-1" + + +@pytest.mark.parametrize( + "session_header", ["x-litellm-session-id", "x-claude-code-session-id", "x-session-id"] +) +@pytest.mark.parametrize("with_trace_header", [False, True], ids=["session-header-only", "beside-trace-header"]) +def test_session_headers_still_beat_the_body_session_with_correlation_logging( + monkeypatch: pytest.MonkeyPatch, session_header: str, with_trace_header: bool +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + metadata: Final[dict[str, str]] = {} + data: Final[dict[str, object]] = {"metadata": metadata, "litellm_session_id": "convo-1"} + trace_header: Final = {"x-litellm-trace-id": "req-trace-1"} if with_trace_header else {} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers={**trace_header, session_header: "hdr-session-1"}, data=data, _metadata_variable_name="metadata" + ) + spend_log_session: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) or data["litellm_session_id"] + assert spend_log_session == "hdr-session-1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("forged_in", ["metadata", "litellm_metadata"]) +@pytest.mark.parametrize( + "headers", + [{"x-litellm-session-id": "hdr-session-1"}, {}], + ids=["session-header", "no-headers"], +) +async def test_client_metadata_cannot_set_the_spend_log_caller_session( + monkeypatch: pytest.MonkeyPatch, forged_in: str, headers: dict[str, str] +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + updated: Final = await add_litellm_data_to_request( + data={ + "model": "gpt-5.4-mini", + "messages": [{"role": "user", "content": "hi"}], + forged_in: {CALLER_SESSION_ID_METADATA_KEY: "forged-session"}, + }, + request=_make_request_mock("/v1/chat/completions", headers), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"] + + +@pytest.mark.asyncio +async def test_spend_log_caller_session_stays_out_of_requester_metadata(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + updated: Final = await add_litellm_data_to_request( + data={ + "model": "gpt-5.4-mini", + "messages": [{"role": "user", "content": "hi"}], + "litellm_session_id": "convo-1", + "metadata": {"team_label": "kept"}, + }, + request=_make_request_mock("/v1/chat/completions", {"x-litellm-trace-id": "req-trace-1"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + assert updated["metadata"][CALLER_SESSION_ID_METADATA_KEY] == "convo-1" + assert updated["metadata"]["requester_metadata"]["team_label"] == "kept" + assert CALLER_SESSION_ID_METADATA_KEY not in updated["metadata"]["requester_metadata"] + + def test_add_litellm_metadata_from_request_headers_x_litellm_session_id_sets_chain_id(): """x-litellm-session-id sets both metadata and top-level litellm_session_id/litellm_trace_id for call chaining.""" headers = {"x-litellm-session-id": "bar"} diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 19bbb6481fb..cc5d5c62739 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -17,6 +17,7 @@ from openai.types.responses.response_function_web_search import ( from pydantic import TypeAdapter import litellm +from litellm.constants import CALLER_SESSION_ID_METADATA_KEY from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -6499,3 +6500,69 @@ def test_transform_request_reads_service_tier_from_responses_api_request() -> No ) assert result["service_tier"] == "priority" + + +def _continued_chain_request(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + async def session_lookup(previous_response_id: str) -> ChatCompletionSession: + assert previous_response_id == "resp_turn_1" + return ChatCompletionSession( + messages=[{"role": "user", "content": "turn 1"}], litellm_session_id="chain-session" + ) + + return asyncio.run( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="resp_turn_1", + litellm_completion_request={ + "model": _CLAUDE_ON_BEDROCK, + "custom_llm_provider": "bedrock", + "messages": [{"role": "user", "content": "turn 2"}], + "metadata": request_kwargs.get("metadata"), + }, + instructions=None, + session_lookup=session_lookup, + request_kwargs=request_kwargs, + ) + ) + + +@pytest.mark.parametrize( + ("request_kwargs", "logged_metadata_key"), + [ + ( + { + "metadata": {"tag": "client"}, + "litellm_metadata": {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"}, + }, + "litellm_metadata", + ), + ({"metadata": {"user_api_key": "hashed"}}, "metadata"), + ], +) +def test_a_continued_chain_logs_its_spend_under_the_chain_session_with_request_correlation( + monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object], logged_metadata_key: str +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", True) + sent_kwargs: Final = deepcopy(request_kwargs) + + request: Final = _continued_chain_request(request_kwargs) + + assert request["litellm_trace_id"] == "chain-session" + assert "litellm_session_id" not in request + assert request[logged_metadata_key] == { + **TypeAdapter(dict[str, object]).validate_python(request_kwargs[logged_metadata_key]), + CALLER_SESSION_ID_METADATA_KEY: "chain-session", + } + assert request_kwargs == sent_kwargs + + +def test_a_continued_chain_leaves_the_turns_metadata_alone_without_request_correlation( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "request_correlation_in_logs", False) + caller_metadata: Final = {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"} + + request: Final = _continued_chain_request({"metadata": caller_metadata, "litellm_metadata": caller_metadata}) + + assert request["litellm_trace_id"] == "chain-session" + assert "litellm_metadata" not in request + assert request["metadata"] == {CALLER_SESSION_ID_METADATA_KEY: "turn-session", "user_api_key": "hashed"} diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 002a595a5e7..5db563be4b0 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1,4 +1,5 @@ import json +import sqlite3 from typing import Final from unittest.mock import AsyncMock, patch @@ -787,3 +788,80 @@ async def test_message_history_replays_real_key_named_tool_payloads() -> None: tool_message: Final = result["messages"][2] assert json.loads(assistant_message["tool_calls"][0]["function"]["arguments"]) == function_arguments assert json.loads(tool_message["content"]) == function_output + + +class _SqliteBackedPrismaDB: + def __init__(self, rows: list[tuple[str, str, str, str, str | None, str | None]]) -> None: + self._connection = sqlite3.connect(":memory:") + self._connection.row_factory = sqlite3.Row + self._connection.execute( + 'CREATE TABLE "LiteLLM_SpendLogs" ' + '(request_id TEXT, api_key TEXT, session_id TEXT, "endTime" TEXT, "user" TEXT, team_id TEXT)' + ) + self._connection.executemany('INSERT INTO "LiteLLM_SpendLogs" VALUES (?, ?, ?, ?, ?, ?)', rows) + + async def query_raw(self, query: str, *args: object) -> list[dict[str, object]]: + return [dict(row) for row in self._connection.execute(query.replace("$1", "?"), args)] + + +@pytest.mark.asyncio +async def test_session_lookup_does_not_return_another_keys_rows_for_a_shared_session_id(): + """A caller-supplied litellm_session_id can collide with another key's session; history must stay per key.""" + sqlite_db: Final = _SqliteBackedPrismaDB( + [ + ("victim-1", "victim-key", "shared-session", "2026-01-01T00:00:01", "", ""), + ("victim-2", "victim-key", "shared-session", "2026-01-01T00:00:02", "", ""), + ("attacker-1", "attacker-key", "shared-session", "2026-01-01T00:00:03", "", ""), + ("attacker-2", "attacker-key", "other-session", "2026-01-01T00:00:04", "", ""), + ] + ) + fake_prisma_client = _FakePrismaClient(results=[]) + fake_prisma_client.db = sqlite_db + + with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): + spend_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id("attacker-1") + victim_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id("victim-1") + + assert [row["request_id"] for row in spend_logs] == ["attacker-1"] + assert [row["request_id"] for row in victim_logs] == ["victim-1", "victim-2"] + + +async def _history_request_ids(sqlite_db: _SqliteBackedPrismaDB, response_id: str) -> list[str]: + fake_prisma_client: Final = _FakePrismaClient(results=[]) + fake_prisma_client.db = sqlite_db + with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): + spend_logs: Final = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(response_id) + return [row["request_id"] for row in spend_logs] + + +@pytest.mark.asyncio +async def test_session_lookup_keeps_jwt_users_and_teams_apart_when_they_share_an_empty_api_key(): + sqlite_db: Final = _SqliteBackedPrismaDB( + [ + ("alice-1", "", "shared-session", "2026-01-01T00:00:01", "alice", "team-a"), + ("bob-1", "", "shared-session", "2026-01-01T00:00:02", "bob", "team-a"), + ("service-a-1", "", "shared-session", "2026-01-01T00:00:03", "", "team-a"), + ("service-b-1", "", "shared-session", "2026-01-01T00:00:04", "", "team-b"), + ("alice-2", "", "shared-session", "2026-01-01T00:00:05", "alice", "team-a"), + ] + ) + + assert await _history_request_ids(sqlite_db, "alice-1") == ["alice-1", "alice-2"] + assert await _history_request_ids(sqlite_db, "bob-1") == ["bob-1"] + assert await _history_request_ids(sqlite_db, "service-a-1") == ["service-a-1"] + assert await _history_request_ids(sqlite_db, "service-b-1") == ["service-b-1"] + + +@pytest.mark.asyncio +async def test_session_lookup_follows_the_owner_across_keys_and_keeps_ownerless_keys_apart(): + sqlite_db: Final = _SqliteBackedPrismaDB( + [ + ("alice-old-key", "hashed-old", "shared-session", "2026-01-01T00:00:01", "alice", "team-a"), + ("ownerless-1", "hashed-other", "shared-session", "2026-01-01T00:00:02", "", ""), + ("alice-new-key", "hashed-new", "shared-session", "2026-01-01T00:00:03", "alice", "team-a"), + ("ownerless-2", "hashed-another", "shared-session", "2026-01-01T00:00:04", "", ""), + ] + ) + + assert await _history_request_ids(sqlite_db, "alice-new-key") == ["alice-old-key", "alice-new-key"] + assert await _history_request_ids(sqlite_db, "ownerless-2") == ["ownerless-2"] From 7930c0e43407fcb3317f201921cc6b2c20c1a0e0 Mon Sep 17 00:00:00 2001 From: Ayush Shekhar <106994833+ayushh0110@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:21:32 +0530 Subject: [PATCH 2/3] feat(router): mid-stream fallback continuation for chat completions (#41127) * feat(router): continue chat streams on fallback after a mid-stream break A chat-completions stream that breaks after content has been delivered was re-raised and the fallback deployment never ran (#40404), while the Responses-API path already continues via a prefilled assistant turn. Add the same for chat completions behind `enable_mid_stream_fallback_continuation` (opt-in, async-only), so a post-content break re-enters the fallback chain with the partial text as an assistant prefill instead of surfacing the error. Continuation is only attempted when it is safe: the stream emitted plain assistant text (no tool/function calls, thinking blocks, reasoning items, audio or images), the request is not constrained output (response_format or a forced tool_choice) and not merge-reasoning mode, and the fallback target's model supports assistant prefill. The target check runs as a deployment pre-call filter, so a chain with no prefill-capable deployment empties and the original error is surfaced rather than a duplicated or rejected request being sent. Plain reasoning_content is treated as out-of-band and does not block a continuation, matching the Responses-API path; Anthropic thinking is excluded because its signed thinking blocks cannot ride a text-only prefill. CustomStreamWrapper now latches whether a disqualifying delta was streamed and carries it on MidStreamFallbackError so the router can decide without re-scanning chunks. Default behavior is unchanged. * fix(router): harden mid-stream continuation eligibility and prefill folding Address review of the mid-stream continuation: - keep response_format={"type": "text"} eligible; only json_object / json_schema and other structured formats decline, since "text" is the unconstrained default - fold a new partial into an existing trailing assistant prefill instead of appending a second one, so a nested mid-stream break stays a single prefill turn even on providers that do not merge consecutive assistant messages - split the deployment prefill-capability lookup into two readable steps - cover the merge_reasoning decline branch, the text response_format pass, and the prefill-folding path with tests * fix(router): address review of mid-stream continuation - register the continuation deployment filter as a process-global singleton instead of tracking it per router, so discarding one router no longer removes the type-deduplicated filter that other live routers still depend on - disqualify reasoning_content: it reaches the caller as visible reasoning that a text-only prefill cannot carry, so a reasoning-capable fallback would re-derive it and produce an incoherent response - move MID_STREAM_CONTINUATION_KWARG to constants and lazy-import the filter class in the router, breaking the module-level import cycle CodeQL flagged - trim the added docstrings to the repository comment policy * fix(router): keep mid-stream continuation within the type-discipline budget The LIT* type-discipline gate counts mutable-collection use the ruff pass does not see. Annotate the reads with read-only Sequence/Mapping views, compare response_format against a frozen MappingProxyType sentinel, and mark the genuinely mutable litellm message and deployment-list boundaries with mutable-ok reasons, so LIT001 and LIT002 stay under their limits. * fix(router): clear basedpyright operator error and the import-cycle alert - narrow the accumulated stream delta content to str before appending, so the shared helper no longer trips reportOperatorIssue (str += object) - keep the continuation marker in the pre-call-check module and lazy-import it at the two router call sites instead of routing it through constants, so the router has no module-level dependency that closes an import cycle * fix(router): make the mid-stream continuation marker unforgeable The deployment filter is process-global and read the continuation marker as a truthy value from request_kwargs. Since the proxy can forward arbitrary request-body fields into the router, a client could send `"_mid_stream_continuation": true` on a normal request to filter a mixed model group down to prefill-capable deployments and steer their prompt onto that provider. Set a private sentinel object internally and check it by type instead of truthiness: a JSON request body cannot construct one, so a forged flag is ignored. Adds a regression test that forged client values (true, "true", 1, a dict) leave the deployment list untouched. * feat(router): honor per-deployment model_info for continuation prefill capability The deployment filter only read supports_assistant_prefill from the cost map, so a model configured with `model_info: {supports_assistant_prefill: true}` on the deployment (the usual way to declare a capability for a model that is not in the cost map, or to override it) was wrongly skipped. Check the deployment's model_info first (True or False both authoritative), then fall back to the cost map. Adds tests for the override in both directions. * fix(router): bound mid-stream continuation to the caller's output allowance A continuation handed the fallback the full max_tokens / max_completion_tokens even though part of the answer had already streamed, so the total could exceed the caller's limit and repeated interruptions could re-grant the allowance on every hop, amplifying spend. Reduce each configured ceiling by the tokens already emitted before re-entry, and decline the continuation (surface the original error) when the allowance is exhausted. The reductions compound across hops because the trimmed ceiling is what the next hop sees. Also add the missing explanatory comment on the model_info fall-through except. * refactor(router): extract mid-stream decline to stay within the complexity budget The output-ceiling branch pushed stream_with_fallbacks past the C901 limit. Both decline paths (ineligible after content, and exhausted allowance) re-raised the original provider error the same way, so pull that into _raise_original_mid_stream_error. It removes the duplicated branching, drops the function back under the complexity ceiling, and reads clearer. * fix(cost-map): mark Claude Sonnet 4.6 as not supporting assistant prefill The Anthropic API and Bedrock both reject an assistant-last message on claude-sonnet-4-6 with "This model does not support assistant message prefill", so the continuation filter must not route a mid-stream continuation to it. Flip the flag on every claude-sonnet-4-6 entry and on databricks' opus-4-6 entry, whose anthropic sibling already says false. The prefill check tests now register synthetic cost-map entries instead of pinning real models, which broke when claude-3-opus left the map. * refactor(router): type the streaming fallback kwargs and drop narration comments Removes an unused mutable-ok marker (LIT013) and the explanatory comments Greptile flagged, and types initial_kwargs on both streaming fallback iterators as dict[str, Any] like their Responses siblings so the new continuation helpers take a typed mapping and the basedpyright budget stays at or below main. * fix(router): run post_call_rules over the joined text of a continued stream Each provider stream wrapper validates only the fragment it produced, so a rule that trips on the primary's prefix plus the continuation's text never fired. The router now feeds litellm.post_call_rules the emitted text plus every continuation chunk before yielding it. Also drops a narrating test-helper docstring. * refactor(router): read a deployment's declared prefill support without an empty except * test(router): cover _raise_original_mid_stream_error directly for the router coverage gate * fix(router): fold the continuation prefill into a trailing assistant turn A request that already ends in an assistant message took the partial output as a second assistant turn, which prefill providers reject. The partial now extends that turn in place, with prefix set, so the continuation request never ends in two assistant messages. * fix(router): keep content blocks on a trailing assistant turn when extending it with the partial output A trailing assistant message whose content is a list of blocks was stringified before the partial output was appended, so the fallback received the Python repr of the blocks as its prefill. The partial now lands as one more text block, and a content shape that cannot be extended surfaces the original stream error instead of a garbled continuation. * fix(router): continue the trailing text block in place when extending a prefill with content blocks Anthropic strips trailing whitespace per content block, so appending the partial output as a new text block after an assistant turn that ends in "Sure, " would send the prefill as "Sure,here". The continuation now extends the last text block itself and only opens a new text block when the trailing block is not text (an image, say), so the prefill reads exactly as the text the caller already received * test(router): type the prefill check's deployment helper * fix(router): keep a continued stream on the primary's identity and widen what can continue Continuation chunks now keep the primary stream's id and created time and drop the fallback's second role delta. Requests with n > 1 no longer continue. Prefill support reads model_info.base_model and litellm_params.base_model before the deployment model. enable_mid_stream_fallback_continuation is runtime-updatable through update_settings and registers the prefill filter. The bridge rebuild of MidStreamFallbackError keeps the inner stream's disqualifier. A fallback that repeats the prefill is not doubled in the continuation input. * fix(router): decline mid-stream continuation after a content filter, merged reasoning, or a plain trailing assistant turn Also count the partial stream's completion tokens in the continued stream's usage, accept string values for enable_mid_stream_fallback_continuation, and list the new router helpers in the router coverage check * fix(router): drop re-emitted whitespace at the continuation join and sum both legs' usage consistently * fix(router): keep the full prefill so a nested break keeps the separator * fix(router): bill both calls of a continued stream and fix Bedrock Converse, Databricks, and Vertex DeepSeek prefill routes * test(router): integration cells for mid-stream fallback continuation Covers the continued stream the caller gets, the prefill turn and reduced max_tokens the fallback receives, the prefill-capability filter, the runtime toggle, repeated requests through the response cache, and the spend row of a continued stream * fix(router): log a continued stream under the caller's id, price each cut on its own prompt, and keep prefill flags per deployment * fix(router): price and log each cut leg before its continuation reuses the logging object Price the cut stream's partial and let its failure log finish (bounded) before the fallback overwrites a shared proxy logging object, and skip continuation when an output guardrail is registered or a request fallback carries its own messages. * test(router): integration cells for the output-guardrail and request-fallback declines and the cut call's failure log Cover a registered post_call guardrail and a request fallback carrying its own messages keeping the original error, an input-only guardrail leaving continuation on, and the cut call's failure event naming its own deployment and price. * fix(router): skip mid-stream continuation when a request fallback overrides request params A request-body fallback entry that sets anything besides model (stream, messages, max_tokens) changes the request the prefill continuation relies on, so a stream: false entry ended the answer with no error. * fix(router): skip mid-stream continuation for a fallback group target that overrides request params The override check read only top-level fallback entries, so a target inside a group's list, such as {"primary": [{"model": "backup", "stream": false}]} in the request, the config, or a key's router settings, still ended the answer with no error. * fix(router): check only the request's own fallback chain for request param overrides Scanning every group's chain turned continuation off for all requests when one unrelated group's target overrode a param. Resolve the chain the router will use for this request, the same way the fallback path does. * fix(router): keep the original error when request fallbacks are malformed The request-param override check resolved a standard-shape chain outside the router's own error handling, so an empty entry or a non-list chain raised StopIteration or TypeError and replaced the original mid-stream error. Skip empty entries and treat a non-list chain as no chain. * fix(router): read deployment-configured constraints, decline deployment-id fallbacks without prefill, strip only the last Bedrock text block * fix(router): read a deployment's tool_choice ahead of router defaults in the continuation check, and restore the renamed prefill test names * fix(router): filter continuation deployments on every routing strategy, read extra_body and fallback deployments' own params The deployment filter now also runs on the sync selection path that usage-based-routing v1 and custom strategies use, one predicate reads n, response_format, tool_choice and merge_reasoning_content_in_choices with extra_body laid over them, fallback deployments whose own config constrains the output are skipped, and continuation declines when the cut deployment's id is not in the router (per-request credential deployments) * fix(router): decline a cut mid-echo, reduce extra_body output limits, and check post_call_rules once on nested continuations --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 2 +- litellm/constants.py | 2 + litellm/exceptions.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 49 +- .../prompt_templates/common_utils.py | 5 + .../prompt_templates/factory.py | 30 +- .../litellm_core_utils/streaming_handler.py | 56 +- .../llms/databricks/chat/transformation.py | 4 +- ...odel_prices_and_context_window_backup.json | 32 +- .../spend_tracking/spend_tracking_utils.py | 8 +- litellm/router.py | 294 +++- litellm/router_utils/common_utils.py | 3 +- .../router_utils/fallback_event_handlers.py | 26 + .../continuation_prefill_check.py | 264 ++++ litellm/types/utils.py | 4 +- litellm/utils.py | 14 + model_prices_and_context_window.json | 32 +- .../router_code_coverage.py | 6 + .../test_mid_stream_fallback_continuation.py | 749 ++++++++++ ...llm_core_utils_prompt_templates_factory.py | 58 + .../test_streaming_handler.py | 134 ++ .../test_databricks_chat_transformation.py | 15 + .../test_continuation_prefill_check.py | 288 ++++ tests/unit/test_router/test_router.py | 1218 ++++++++++++++++- .../unit/test_router_model_cost_isolation.py | 13 +- 25 files changed, 3211 insertions(+), 97 deletions(-) create mode 100644 litellm/router_utils/pre_call_checks/continuation_prefill_check.py create mode 100644 tests/integration/streaming/test_mid_stream_fallback_continuation.py create mode 100644 tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py diff --git a/litellm/__init__.py b/litellm/__init__.py index af812dade3c..88e56c14187 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -207,7 +207,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]] [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] -post_call_rules: List[Callable] = [] +post_call_rules: List[Callable[..., object]] = [] turn_off_message_logging: Optional[bool] = False standard_logging_payload_excluded_fields: Optional[List[str]] = ( None # Fields to exclude from StandardLoggingPayload before callbacks receive it diff --git a/litellm/constants.py b/litellm/constants.py index eb03d034c7e..39363cc02e4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -57,6 +57,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( "model_group_retry_policy", "model_group_alias", "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", "enable_tag_filtering", "tag_routing_prefix", "optional_pre_call_checks", @@ -146,6 +147,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 # Streams exceeding this duration are terminated with a Timeout error. # None (default) = no limit. Set env var to a number of seconds to enable globally. LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS") +MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0 # Maximum number of base64 characters to keep in logging payloads. # Data URIs exceeding this are replaced with a size placeholder. diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 31388b00554..b3a0802db13 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1217,6 +1217,7 @@ class MidStreamFallbackError(ServiceUnavailableError): num_retries: int | None = None, generated_content: str = "", is_pre_first_chunk: bool = False, + emitted_disqualifying_content: bool = False, ): original_status: Final = getattr(original_exception, "status_code", None) self.status_code = int(original_status) if original_status is not None else 503 @@ -1229,6 +1230,7 @@ class MidStreamFallbackError(ServiceUnavailableError): self.num_retries = num_retries self.generated_content = generated_content self.is_pre_first_chunk = is_pre_first_chunk + self.emitted_disqualifying_content = emitted_disqualifying_content # Create a response if one wasn't provided if response is None: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e766a2be9c3..fa009ca6327 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2295,6 +2295,36 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["combined_usage_object"] = usage self.model_call_details["response_cost"] = response_cost + def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None: + """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw.""" + self.model_call_details["continued_partial_usage"] = usage + self.model_call_details["continued_partial_cost"] = response_cost + self.model_call_details["continued_response_id"] = response_id + + def _add_continued_partial_spend(self, response: object) -> None: + partial_usage: Final = self.model_call_details.get("continued_partial_usage") + partial_cost: Final = self.model_call_details.get("continued_partial_cost") + own_usage: Final = getattr(response, "usage", None) + own_cost: Final = self.model_call_details.get("response_cost") + if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)): + return + from litellm.cost_calculator import BaseTokenUsageProcessor + + self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[own_usage, partial_usage] + ) + if not isinstance(own_cost, (int, float)): + return + self.model_call_details["response_cost"] = own_cost + partial_cost + if self.cost_breakdown is not None: + self.cost_breakdown = self.cost_breakdown | CostBreakdown( + total_cost=own_cost + partial_cost, + additional_costs={ + **self.cost_breakdown.get("additional_costs", {}), + "mid_stream_partial_cost": partial_cost, + }, + ) + def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" usage: Final = getattr(assembled, "usage", None) @@ -2590,6 +2620,15 @@ class Logging(LiteLLMLoggingBaseClass): return response return response.model_copy(update={"model": model}) + def _with_continued_response_id( + self, + response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse, + ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse: + response_id: Final = self.model_call_details.get("continued_response_id") + if not isinstance(response_id, str) or getattr(response, "id", None) == response_id: + return response + return response.model_copy(update={"id": response_id}) + def _success_handler_helper_fn( self, result: object = None, @@ -2908,8 +2947,11 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = self.response_cost_calculator( result=complete_streaming_response ) + self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) - logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response) + logged_streaming_response: Final = self._with_continued_response_id( + self._with_client_facing_stream_model(complete_streaming_response) + ) self.model_call_details["complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload( @@ -3445,12 +3487,15 @@ class Logging(LiteLLMLoggingBaseClass): self.model, ) self.model_call_details["response_cost"] = None + self._add_continued_partial_spend(complete_streaming_response) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) await self._prepare_baseline_cache_estimate(complete_streaming_response) - logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response) + logged_streaming_response: Final = self._with_continued_response_id( + self._with_client_facing_stream_model(complete_streaming_response) + ) self.model_call_details["async_complete_streaming_response"] = logged_streaming_response ## STANDARDIZED LOGGING PAYLOAD diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 81fc865a5bc..044123d8729 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1894,6 +1894,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS) +def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues: + """Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys.""" + return _message_without_fields(message, frozenset({"prefix"})) + + def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any: """ Filters a value from a dictionary diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 07534e70a27..87e76fe39f7 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4303,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]]) +def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock: + return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block + + +def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock: + """Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it.""" + if message["role"] != "assistant": + return message + blocks: Final = message["content"] + last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None) + return BedrockMessageBlock( + role="assistant", + content=[ + _rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks) + ], + ) + + def _ensure_document_messages_have_text( contents: list[BedrockMessageBlock], ) -> list[BedrockMessageBlock]: @@ -4894,7 +4912,12 @@ class BedrockConverseMessagesProcessor: llm_provider=llm_provider, ) - return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents)) + converse_messages: Final = _ensure_document_messages_have_text( + _rename_duplicate_bedrock_document_names(contents) + ) + if not (to_claude and converse_messages): + return converse_messages + return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])] @staticmethod def translate_thinking_blocks_to_reasoning_content_blocks( @@ -5303,7 +5326,10 @@ def _bedrock_converse_messages_pt( llm_provider=llm_provider, ) - return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents)) + converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents)) + if not (to_claude and converse_messages): + return converse_messages + return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])] def make_valid_bedrock_tool_name(input_tool_name: str) -> str: diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 46ad1b70abe..95d062a190c 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -242,6 +242,7 @@ class CustomStreamWrapper: self.completion_stream = completion_stream self.sent_first_chunk = False self.sent_last_chunk = False + self._failure_logging: asyncio.Task[None] | None = None self._closed = False self._stream_created_time: float = time.time() @@ -269,6 +270,7 @@ class CustomStreamWrapper: self.holding_chunk = "" self.complete_response = "" self.response_uptil_now = "" + self._emitted_disqualifying_content = False _model_info: Final[dict] = litellm_params.model_info or {} _api_base: Final = get_api_base( @@ -1129,6 +1131,7 @@ class CustomStreamWrapper: if self.merge_reasoning_content_in_choices is True: reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None) if reasoning_content: + self._emitted_disqualifying_content = True if self.sent_first_thinking_block is False: # Ensure content is not None before concatenation if model_response.choices[0].delta.content is None: @@ -1865,9 +1868,7 @@ class CustomStreamWrapper: if response.choices: choice = response.choices[0] if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) # HANDLE STREAM OPTIONS self.chunks.append(response) @@ -2084,9 +2085,7 @@ class CustomStreamWrapper: if processed_chunk.choices: choice = processed_chunk.choices[0] if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) # Add mcp_list_tools to first chunk if present if not self.sent_first_chunk and processed_chunk.choices: @@ -2156,9 +2155,7 @@ class CustomStreamWrapper: choice = processed_chunk.choices[0] if isinstance(choice, StreamingChoices): - self.response_uptil_now += choice.delta.get("content", "") or "" - else: - self.response_uptil_now += "" + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) # RETURN RESULT self.chunks.append(processed_chunk) @@ -2172,7 +2169,7 @@ class CustomStreamWrapper: if self.logging_obj is not None: self._record_partial_usage_for_failure() ## LOGGING - asyncio.create_task( + self._failure_logging = asyncio.create_task( self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True) ) await self.aclose() @@ -2305,11 +2302,17 @@ class CustomStreamWrapper: if self.logging_obj is not None: self._record_partial_usage_for_failure() ## LOGGING - asyncio.create_task( + self._failure_logging = asyncio.create_task( self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True) ) self._handle_stream_fallback_error(e) + async def wait_for_failure_logging(self, timeout: float) -> None: + """Let this stream's failure log read its own deployment before a fallback reuses the logging object.""" + if self._failure_logging is None: + return + await asyncio.wait((self._failure_logging,), timeout=timeout) + def _record_partial_usage_for_failure(self) -> None: """ A stream that breaks mid-flight still billed the provider for the chunks @@ -2346,6 +2349,35 @@ class CustomStreamWrapper: recover_error, ) + # Delta fields a text-only prefill continuation cannot carry, so a stream + # that emitted any of them is not eligible for mid-stream continuation. + _CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = ( + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", + ) + + @classmethod + def _delta_disqualifies_continuation(cls, delta: object) -> bool: + get: Final = getattr(delta, "get", None) + if not callable(get): + return False + return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS) + + def _accumulate_streamed_delta(self, delta: object) -> None: + # Shared by the sync, async, and non-aiohttp iteration sites so answer + # text and the disqualifying-content latch stay in step across all three. + get: Final = getattr(delta, "get", None) + content: Final = get("content") if callable(get) else None + self.response_uptil_now += content if isinstance(content, str) else "" + if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta): + self._emitted_disqualifying_content = True + def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn": """ Common error handling for both __next__ and __anext__. @@ -2377,6 +2409,7 @@ class CustomStreamWrapper: original_exception=e.original_exception, generated_content=self.response_uptil_now, is_pre_first_chunk=not self.sent_first_chunk, + emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content, ) # Map to OpenAI exception format. Some providers' mappers (e.g. @@ -2437,6 +2470,7 @@ class CustomStreamWrapper: original_exception=mapped_exception, generated_content=self.response_uptil_now, is_pre_first_chunk=not self.sent_first_chunk, + emitted_disqualifying_content=self._emitted_disqualifying_content, ) @staticmethod diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 2145c19a71a..74bdd2cd733 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( merge_consecutive_system_messages, strip_litellm_internal_message_fields, strip_name_from_message, + strip_prefill_prefix_field, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( @@ -487,6 +488,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): Databricks does not support: - 'name' in user message. - litellm's internal `thinking_blocks` / `reasoning_content` on assistant messages. + - the `prefix` prefill flag Mistral and DeepSeek read, which Claude does not need to continue an assistant turn. - non-Claude json_object "json" checks read only string content, so a single text block is sent as a string. """ new_messages = [] @@ -496,7 +498,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): else: _message = message _message = strip_name_from_message(_message, allowed_name_roles=["user"]) - _message = strip_litellm_internal_message_fields(_message) + _message = strip_prefill_prefix_field(strip_litellm_internal_message_fields(_message)) # Move message-level cache_control into a content block when content is a string. if "cache_control" in _message and isinstance(_message.get("content"), str): _message = self._move_cache_control_into_string_content_block(_message) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b744032fb6c..5fff760d85a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2773,7 +2773,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2808,7 +2808,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2843,7 +2843,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2878,7 +2878,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2913,7 +2913,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2948,7 +2948,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -3743,7 +3743,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -15494,7 +15494,7 @@ }, "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -20832,7 +20832,7 @@ "output_cost_per_token": 2.5000010000000002e-05, "output_dbu_cost_per_token": 0.000357143, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -21047,7 +21047,7 @@ "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -49176,7 +49176,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -51006,7 +51006,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -51179,7 +51179,7 @@ "supported_regions": [ "global" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -51200,7 +51200,7 @@ "supported_regions": [ "us-central1" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -58822,7 +58822,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -72334,7 +72334,7 @@ "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index fe55092591e..9ebea589cc9 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -573,11 +573,11 @@ def get_logging_payload( elif isinstance(_usage, dict): usage = _usage - # A request that failed mid-stream has no usable response_obj usage, but the - # streaming handler may have recovered the usage from the chunks already - # delivered. Honor that override so the partial usage lands in spend tracking. + # A request that failed mid-stream has no usable response_obj usage, and a stream + # continued on a fallback only has the fallback's. combined_usage_object holds the + # usage of every call the client's answer took, so it wins in both cases. _combined_usage: Final = kwargs.get("combined_usage_object") - if not usage and isinstance(_combined_usage, litellm.Usage): + if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage): usage = _combined_usage.model_dump() id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata) diff --git a/litellm/router.py b/litellm/router.py index 151a8a75dc3..3c969610e83 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -38,7 +38,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache, partial from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, TypeVar, Union, cast +from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast import anyio import httpx @@ -68,6 +68,7 @@ from litellm.constants import ( DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, DEFAULT_MAX_LRU_CACHE_SIZE, INTERNAL_CALL_ORIGIN_METADATA_KEY, + MID_STREAM_FAILURE_LOG_WAIT_SECONDS, OUTPUT_TOKEN_CEILING_PARAMS, PROMPT_CACHE_PIN_CACHE_MAX_SIZE, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY, @@ -75,7 +76,7 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) -from litellm.integrations.custom_guardrail import is_guardrail_intervention +from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span @@ -194,6 +195,7 @@ from litellm.router_utils.clientside_credential_handler import ( is_clientside_credential, ) from litellm.router_utils.common_utils import ( + MID_STREAM_CONTINUATION_KWARG, filter_team_based_models, filter_web_search_deployments, format_fallback_outcome_message, @@ -233,7 +235,10 @@ from litellm.router_utils.fallback_event_handlers import ( check_non_standard_fallback_format, clear_pre_routing_selection, committed_retry_budget_for_request, + continuation_fallback_targets, fallback_lookup_groups, + fallback_target_name, + fallback_target_overrides_request_params, fallbacks_disabled_for_request, get_fallback_model_group_for_lookup_groups, get_pre_routing_selection, @@ -258,6 +263,17 @@ from litellm.router_utils.handle_error import ( send_llm_exception_alert, ) from litellm.router_utils.health_state_cache import DeploymentHealthCache +from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_MARKER, + ContinuationOutput, + ContinuationPrefillDeploymentCheck, + ContinuedPartial, + constrains_continuation, + continuation_deployments, + continuation_output_ceilings, + continued_text, + deployment_can_continue, +) from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck, warn_on_unknown_model_group_affinity_flags, @@ -303,6 +319,8 @@ from litellm.router_utils.routing_groups import ( ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch from litellm.scheduler import FlowItem, Scheduler +from litellm.secret_managers.main import str_to_bool +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.litellm_params import RoutingStrategyName from litellm.types.llms.openai import ( AllMessageValues, @@ -371,6 +389,7 @@ from litellm.utils import ( EmbeddingResponse, ModelResponse, Rules, + TextCompletionResponse, function_setup, get_llm_provider, get_non_default_completion_params, @@ -527,6 +546,15 @@ _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeA _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object) _DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected" +_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + +def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]: + """Continue the last text block in place so the prefill reads as the text the caller already received.""" + last: Final = blocks[-1] if blocks else None + if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str): + return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}] + return [*blocks, {"type": "text", "text": text}] def _is_fallback_hop(request_kwargs: Mapping[str, object]) -> bool: @@ -1119,6 +1147,7 @@ class Router: health_check_ignore_transient_errors: bool = False, background_health_check_model_groups: Sequence[str] | None = None, enable_weighted_failover: bool = False, + enable_mid_stream_fallback_continuation: bool = False, fallback_access_check: FallbackAccessCheck | None = None, fallback_budget_check: FallbackBudgetCheck | None = None, auto_router_capability_limit: AutoRouterCapabilityLimit | None = None, @@ -1160,6 +1189,7 @@ class Router: deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600. ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error. enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False. + enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False. fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted). fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback). Returns: @@ -1355,6 +1385,7 @@ class Router: self.fallback_on_unsupported_params = fallback_on_unsupported_params self.enable_health_check_routing = enable_health_check_routing self.enable_weighted_failover = enable_weighted_failover + self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation) self.health_check_ignore_transient_errors = health_check_ignore_transient_errors self.background_health_check_model_groups: frozenset[str] | None = ( frozenset(background_health_check_model_groups) @@ -1549,6 +1580,14 @@ class Router: default_pre_call_checks: Final[OptionalPreCallChecks] = [] self.add_optional_pre_call_checks(default_pre_call_checks) + if self.enable_mid_stream_fallback_continuation: + self._register_continuation_prefill_check() + + @staticmethod + def _register_continuation_prefill_check() -> None: + # Process-global on purpose: Router.discard() must not drop a filter another router still needs. + litellm.logging_callback_manager.add_litellm_callback(ContinuationPrefillDeploymentCheck()) + def discard(self): """ Pseudo-destructor to be invoked to clean up global data structures when router is no longer used. @@ -3143,15 +3182,53 @@ class Router: usage: Final = cast(Usage | None, getattr(fallback_item, "usage", None)) usage_objects: Final = [usage] if usage is not None else [] - if ( - complete_response_object_usage is not None - and hasattr(complete_response_object_usage, "usage") - and complete_response_object_usage.usage is not None - ): + if complete_response_object_usage is not None: usage_objects.append(complete_response_object_usage) combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=usage_objects) setattr(fallback_item, "usage", combined_usage) + @staticmethod + def _price_cut_partial( + cut_stream: CustomStreamWrapper, + partial_response: ModelResponse | TextCompletionResponse | None, + ) -> "ContinuedPartial | None": + """Price the cut stream's partial, plus any partial it inherited, before a fallback can reuse its logging.""" + from litellm.cost_calculator import BaseTokenUsageProcessor + + cut_logging: Final = cut_stream.logging_obj + partial_usage: Final = getattr(partial_response, "usage", None) + if partial_response is None or not isinstance(partial_usage, Usage): + return None + inherited_usage: Final = cut_logging.model_call_details.get("continued_partial_usage") + inherited_cost: Final = cut_logging.model_call_details.get("continued_partial_cost") + inherited_id: Final = cut_logging.model_call_details.get("continued_response_id") + priced_partial: Final = partial_response.model_copy( + update={"model": cut_stream.model} if cut_stream.model else {} + ) + priced_partial._hidden_params = { # pyright: ignore[reportPrivateUsage] # no public accessor + key: value for key, value in get_hidden_params_dict(partial_response).items() if key != "response_cost" + } + return ContinuedPartial( + usage=BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[partial_usage, inherited_usage] + if isinstance(inherited_usage, Usage) + else [partial_usage] + ), + response_cost=(cut_logging.response_cost_calculator(result=priced_partial) or 0.0) + + (inherited_cost if isinstance(inherited_cost, float) else 0.0), + response_id=inherited_id if isinstance(inherited_id, str) else partial_response.id, + ) + + @staticmethod + def _carry_partial_spend_to_fallback(fallback_response: object, cut_partial: "ContinuedPartial | None") -> None: + """Bill what the cut legs consumed on the continuation's success log.""" + fallback_logging: Final = getattr(fallback_response, "logging_obj", None) + if cut_partial is None or not isinstance(fallback_logging, LiteLLMLogging): + return + fallback_logging.record_continued_partial( + usage=cut_partial.usage, response_cost=cut_partial.response_cost, response_id=cut_partial.response_id + ) + @staticmethod def _prepare_fallback_hidden_params( fallback_response: object, @@ -3201,7 +3278,7 @@ class Router: self, model_response: CustomStreamWrapper, messages: list[dict[str, str]], - initial_kwargs: dict, + initial_kwargs: dict[str, Any], deployment_slot: contextlib.AsyncExitStack | None = None, first_chunk: _FirstChunk[AsyncIterator[object]] | None = None, ) -> CustomStreamWrapper: @@ -3264,16 +3341,22 @@ class Router: with anyio.CancelScope(shield=True): await close_model_response() await held_slot.aclose() - if not e.is_pre_first_chunk and ( - e.generated_content or _stream_chunks_have_generated_content(model_response.chunks) - ): - if e.original_exception is not None: - raise e.original_exception from e - raise + committed: Final = bool( + not e.is_pre_first_chunk + and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)) + ) + cut_hidden_params: Final = get_hidden_params(model_response) or {} + continue_after_content: Final = committed and self._mid_stream_continuation_eligible( + e=e, request_kwargs=initial_kwargs, cut_deployment_id=cut_hidden_params.get("model_id") + ) + if committed and not continue_after_content: + self._raise_original_mid_stream_error(e) from litellm.main import stream_chunk_builder - complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks) + complete_response_object: Final = stream_chunk_builder( + chunks=model_response.chunks, messages=messages if continue_after_content else None + ) complete_response_object_usage: Final = cast( Usage | None, getattr(complete_response_object, "usage", None), @@ -3289,7 +3372,41 @@ class Router: "content_policy_fallbacks", self.content_policy_fallbacks ) initial_kwargs["original_function"] = self._acompletion - initial_kwargs["messages"] = messages + continuation_messages: Final = ( + self._build_completion_continuation_input(messages, e.generated_content) + if continue_after_content + else None + ) + continuation_output: Final = ( + ContinuationOutput( + e.generated_content, + continuation_messages[-1], + model_group, + complete_response_object, + checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG) + is not MID_STREAM_CONTINUATION_MARKER, + ) + if continuation_messages is not None + else None + ) + cut_partial: Final = ( + Router._price_cut_partial(model_response, complete_response_object) + if continuation_output is not None + else None + ) + if continue_after_content: + emitted_tokens: Final = int( + getattr(complete_response_object_usage, "completion_tokens", 0) or 0 + ) + reduced_ceilings: Final = continuation_output_ceilings(initial_kwargs, emitted_tokens) + if reduced_ceilings is None or continuation_messages is None: + self._raise_original_mid_stream_error(e) + await model_response.wait_for_failure_logging(timeout=MID_STREAM_FAILURE_LOG_WAIT_SECONDS) + initial_kwargs.update(reduced_ceilings) + initial_kwargs["messages"] = continuation_messages + initial_kwargs[MID_STREAM_CONTINUATION_KWARG] = MID_STREAM_CONTINUATION_MARKER + else: + initial_kwargs["messages"] = messages self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs) fallback_response = await self.async_function_with_fallbacks_common_utils( e=e, @@ -3305,11 +3422,17 @@ class Router: # If fallback returns a streaming response, iterate over it if hasattr(fallback_response, "__aiter__"): + Router._carry_partial_spend_to_fallback(fallback_response, cut_partial) prepared_fallback_hidden_params = Router._adopt_fallback_response_headers( wrapper_ref, fallback_response ) fallback_headers_are_settled = False - async for fallback_item in fallback_response: + fallback_items: Final = ( + continuation_output.splice(fallback_response) + if continuation_output is not None + else fallback_response + ) + async for fallback_item in fallback_items: if not fallback_headers_are_settled: fallback_headers_are_settled = True # a fallback that failed over again only repoints itself once it yields @@ -3322,7 +3445,10 @@ class Router: and isinstance(fallback_item, ModelResponseStream) and hasattr(fallback_item, "usage") ): - self._combine_fallback_usage(fallback_item, complete_response_object_usage) + self._combine_fallback_usage( + fallback_item, + complete_response_object_usage if continuation_output is not None else None, + ) yield fallback_item else: # If fallback returns a non-streaming response, yield None @@ -3479,6 +3605,97 @@ class Router: total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0), ) + @staticmethod + def _raise_original_mid_stream_error(e: "MidStreamFallbackError") -> NoReturn: + """Surface the provider error the stream wrapper carried instead of the internal MidStreamFallbackError.""" + if e.original_exception is not None: + raise e.original_exception from e + raise e + + def _params_sent_to_deployment( + self, request_kwargs: Mapping[str, object], deployment: Deployment | None + ) -> Mapping[str, object]: + """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies.""" + deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {} + defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None} + deployment_tool_choice: Final = ( + {"tool_choice": deployment_params["tool_choice"]} + if deployment_params.get("tool_choice") is not None + else {} + ) + return MappingProxyType({**deployment_params, **defaults, **deployment_tool_choice, **request_kwargs}) + + def _names_deployment_that_cannot_continue(self, target: object) -> bool: + name: Final = fallback_target_name(target) + if name is None or name in self.model_names or not self.has_model_id(name): + return False + deployment: Final = self.get_deployment(model_id=name) + return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True)) + + def _mid_stream_continuation_eligible( + self, + e: "MidStreamFallbackError", + request_kwargs: Mapping[str, object], + cut_deployment_id: object = None, + ) -> bool: + """Whether a stream that broke after plain assistant text may continue via prefill.""" + if not self.enable_mid_stream_fallback_continuation: + return False + if not e.generated_content or e.emitted_disqualifying_content: + return False + if isinstance(e.original_exception, litellm.ContentPolicyViolationError): + return False + cut_deployment: Final = ( + self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None + ) + if isinstance(cut_deployment_id, str) and cut_deployment is None: + return False + if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)): + return False + requested_group: Final = request_kwargs.get("model") + lookup_groups: Final = fallback_lookup_groups( + request_kwargs, requested_group if isinstance(requested_group, str) else None + ) + targets: Final = continuation_fallback_targets(request_kwargs.get("fallbacks", self.fallbacks), lookup_groups) + if any(fallback_target_overrides_request_params(target) for target in targets): + return False + if any(self._names_deployment_that_cannot_continue(target) for target in targets): + return False + return not any( + isinstance(callback, CustomGuardrail) + and callback._event_hook_is_event_type( # pyright: ignore[reportPrivateUsage] # the only check of a guardrail's hooks + GuardrailEventHooks.post_call + ) + for callback in litellm.callbacks + ) + + @staticmethod + def _build_completion_continuation_input( + messages: Sequence[Mapping[str, object]], + generated_content: str, + ) -> Sequence[Mapping[str, object]] | None: + """Append the partial output as an assistant prefill, or extend a trailing `prefix: True` assistant turn in + place so the request never ends in two assistant messages. None when the request ends in a plain assistant + turn, in a prefill turn whose content has a shape this cannot extend, or when the cut came mid-echo of the + prefill.""" + last: Final = messages[-1] if messages else None + if last is None or last.get("role") != "assistant": + return [*messages, {"role": "assistant", "content": generated_content, "prefix": True}] + if last.get("prefix") is not True: + return None + new_text: Final = continued_text(last, generated_content) + if new_text is None: + return None + content: Final = last.get("content") + if content is None or isinstance(content, str): + return [*messages[:-1], {**last, "content": f"{content or ''}{new_text}", "prefix": True}] + try: + blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content) + except ValidationError: + return None + extended: Final = _extend_trailing_text_block(blocks, new_text) + return [*messages[:-1], {**last, "content": extended, "prefix": True}] + @staticmethod def _build_responses_continuation_input( input_val: Union[str, "ResponseInputParam"] | None, @@ -3880,13 +4097,6 @@ class Router: raise e.original_exception from e raise - from litellm.main import stream_chunk_builder - - complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks) - complete_response_object_usage: Final = cast( - Usage | None, - getattr(complete_response_object, "usage", None), - ) try: model_group: Final = cast(str, initial_kwargs.get("model")) fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", router_self.fallbacks) @@ -3932,7 +4142,7 @@ class Router: and isinstance(fallback_item, ModelResponseStream) and hasattr(fallback_item, "usage") ): - router_self._combine_fallback_usage(fallback_item, complete_response_object_usage) + router_self._combine_fallback_usage(fallback_item, None) yield fallback_item else: yield None @@ -13004,6 +13214,7 @@ class Router: "retry_policy", "model_group_alias", "enable_weighted_failover", + "enable_mid_stream_fallback_continuation", "enable_tag_filtering", "tag_routing_prefix", "fallback_on_unsupported_params", @@ -13060,12 +13271,10 @@ class Router: rebuild_routing_groups = True elif var == "optional_pre_call_checks": self.set_optional_pre_call_checks(kwargs[var]) + elif var == "enable_mid_stream_fallback_continuation": + self._set_mid_stream_fallback_continuation(kwargs[var]) elif var == "retry_policy": - value = kwargs[var] - if isinstance(value, dict): - value = RetryPolicy(**value) - if value is None or isinstance(value, RetryPolicy): - setattr(self, var, value) + self._set_retry_policy(kwargs[var]) else: value = kwargs[var] # only run routing strategy init if it has changed @@ -13092,6 +13301,20 @@ class Router: self._routing_groups_input = routing_groups_input verbose_router_logger.debug("Updated Router settings: %s", self.get_settings()) + @staticmethod + def _is_flag_enabled(value: object) -> bool: + return value is True or (isinstance(value, str) and str_to_bool(value) is True) + + def _set_mid_stream_fallback_continuation(self, enabled: object) -> None: + self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enabled) + if self.enable_mid_stream_fallback_continuation: + self._register_continuation_prefill_check() + + def _set_retry_policy(self, value: object) -> None: + policy: Final = RetryPolicy.model_validate(value) if isinstance(value, dict) else value + if policy is None or isinstance(policy, RetryPolicy): + self.retry_policy = policy + def _get_client(self, deployment, kwargs, client_type=None): """ Returns the appropriate client based on the given deployment, kwargs, and client_type. @@ -15119,9 +15342,8 @@ class Router: request_kwargs=request_kwargs, ) strategy, strategy_selector = self._get_routing_context(model, request_kwargs) - pick_attributes: Final = _deployment_pick_attributes( - model, _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) - ) + routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs) + pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs) if isinstance(healthy_deployments, dict): if (healthy_deployments.get("model_info") or {}).get("blocked") is True: @@ -15159,7 +15381,9 @@ class Router: ) healthy_deployments = _pre_cooldown_deployments - healthy_deployments = self._filter_blocked_deployments(healthy_deployments) + healthy_deployments = continuation_deployments( + self._filter_blocked_deployments(healthy_deployments), routing_kwargs + ) # filter pre-call checks if self.enable_pre_call_checks and (messages is not None or input is not None): diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index e12304c6ed1..c4e4d9ccc0b 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -27,7 +27,8 @@ from litellm.types.utils import LlmProviders, ProviderSpecificModelInfo _V = TypeVar("_V") -ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors"}) +MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation" +ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG}) _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object]) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 1f37c6c8cb1..06bd3329cc1 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -615,6 +615,32 @@ def get_fallback_model_group( return fallback_model_group, generic_fallback_idx +def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]: + group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple( # mutable-ok: mirrors the fallbacks config shape + entry for entry in entries if isinstance(entry, Mapping) and entry + ) + chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups) + return chain if isinstance(chain, list) else () + + +def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]: + entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else () + return ( + entries + if check_non_standard_fallback_format(fallbacks=entries) + else _resolved_fallback_chain(entries, lookup_groups) + ) + + +def fallback_target_overrides_request_params(target: object) -> bool: + return isinstance(target, Mapping) and "model" in target and len(target) > 1 + + +def fallback_target_name(target: object) -> str | None: + name: Final = target.get("model") if isinstance(target, Mapping) else target + return name if isinstance(name, str) else None + + PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id") PROVIDER_SCOPED_RESOURCE_FUNCTION_NAMES: Final = frozenset( { diff --git a/litellm/router_utils/pre_call_checks/continuation_prefill_check.py b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py new file mode 100644 index 00000000000..8f211ef67db --- /dev/null +++ b/litellm/router_utils/pre_call_checks/continuation_prefill_check.py @@ -0,0 +1,264 @@ +"""Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave +the output unconstrained.""" + +from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import accumulate +from os.path import commonprefix +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.rules import Rules +from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import ModelResponse, ModelResponseStream, Usage +from litellm.utils import supports_assistant_prefill + + +class _ContinuationMarker: + """Only the router constructs one, so a JSON request body cannot forge the continuation flag.""" + + +MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker() + +_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) +_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"}) +_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens") + + +def _str_keyed(value: object) -> Mapping[str, object]: + try: + return _STR_KEYED_DICT_ADAPTER.validate_python(value) + except ValidationError: + return _NO_FIELDS + + +def deployment_supports_prefill(deployment: object) -> bool: + deployment_map: Final = _str_keyed(deployment) + model_info: Final = _str_keyed(deployment_map.get("model_info")) + declared: Final = model_info.get("supports_assistant_prefill") + if isinstance(declared, bool): + return declared + litellm_params: Final = _str_keyed(deployment_map.get("litellm_params")) + candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model")) + model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None) + return model is not None and supports_assistant_prefill(model=model) + + +def constrains_continuation(params: Mapping[str, object]) -> bool: + sent: Final = {**params, **_str_keyed(params.get("extra_body"))} + choice_count: Final = sent.get("n") + response_format: Final = sent.get("response_format") + tool_choice: Final = sent.get("tool_choice") + return ( + (isinstance(choice_count, int) and choice_count > 1) + or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT) + or tool_choice == "required" + or isinstance(tool_choice, Mapping) + or sent.get("merge_reasoning_content_in_choices") is True + ) + + +def deployment_can_continue(deployment: object) -> bool: + litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params")) + return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params) + + +def continuation_deployments( + healthy_deployments: list[dict[str, object]], # mutable-ok: router deployment-list contract + request_kwargs: Mapping[str, object] | None, +) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list + marker: Final = (request_kwargs or {}).get(MID_STREAM_CONTINUATION_KWARG) + if not isinstance(marker, _ContinuationMarker): + return healthy_deployments + return [deployment for deployment in healthy_deployments if deployment_can_continue(deployment)] + + +class ContinuationPrefillDeploymentCheck(CustomLogger): + async def async_filter_deployments( + self, + model: str, + healthy_deployments: list[dict[str, object]], # mutable-ok: CustomLogger deployment-list contract + messages: Sequence[AllMessageValues] | None, + request_kwargs: Mapping[str, object] | None = None, + parent_otel_span: Span | None = None, + ) -> list[dict[str, object]]: # mutable-ok: returns a mutable deployment list + return continuation_deployments(healthy_deployments, request_kwargs) + + +def _delta_text(chunk: ModelResponseStream) -> str: + if not chunk.choices: + return "" + content: Final = chunk.choices[0].delta.content + return content if isinstance(content, str) else "" + + +_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]]) + + +def _prefill_text(prefill_message: Mapping[str, object]) -> str: + content: Final = prefill_message.get("content") + if isinstance(content, str): + return content + try: + blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content) + except ValidationError: + return "" + return "".join( + text for block in blocks if block.get("type") == "text" and isinstance(text := block.get("text"), str) + ) + + +def _resendable_whitespace(prefill: str) -> str: + return prefill[len(prefill.rstrip()) :] if prefill.strip() else "" + + +def continued_text(prefill_message: Mapping[str, object], generated_content: str) -> str | None: + prefill: Final = _prefill_text(prefill_message) + if generated_content and len(generated_content) < len(prefill) and prefill.startswith(generated_content): + return None + echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0 + after_echo: Final = generated_content[echo:] + return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :] + + +def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]: + return MappingProxyType( + {key: value - emitted_tokens for key in _OUTPUT_CEILING_KEYS if isinstance(value := params.get(key), int)} + ) + + +def continuation_output_ceilings( + request_kwargs: Mapping[str, object], emitted_tokens: int +) -> Mapping[str, object] | None: + """The caller's output ceilings, top-level and in extra_body, minus the tokens already emitted, or None once + nothing is left.""" + extra_body: Final = _str_keyed(request_kwargs.get("extra_body")) + top_level: Final = _reduced_ceilings(request_kwargs, emitted_tokens) + in_extra_body: Final = _reduced_ceilings(extra_body, emitted_tokens) + if any(ceiling <= 0 for ceiling in (*top_level.values(), *in_extra_body.values())): + return None + if not in_extra_body: + return top_level + return MappingProxyType({**top_level, "extra_body": {**extra_body, **in_extra_body}}) + + +def _with_first_delta(chunk: ModelResponseStream, update: Mapping[str, object]) -> ModelResponseStream: + choice: Final = chunk.choices[0] + delta: Final = choice.delta.model_copy(update=update) + return chunk.model_copy(update={"choices": [choice.model_copy(update={"delta": delta}), *chunk.choices[1:]]}) + + +def _without_leading_text(chunk: ModelResponseStream, count: int) -> ModelResponseStream: + if count <= 0: + return chunk + return _with_first_delta(chunk, {"content": _delta_text(chunk)[count:]}) + + +def _without_role(chunk: ModelResponseStream) -> ModelResponseStream: + if not chunk.choices or chunk.choices[0].delta.role is None: + return chunk + return _with_first_delta(chunk, {"role": None}) + + +def _stream_identity(primary_response: object) -> Mapping[str, object]: + if not isinstance(primary_response, ModelResponse): + return _NO_FIELDS + return MappingProxyType({"id": primary_response.id, "created": primary_response.created}) + + +def _drop_leading_text(chunks: tuple[ModelResponseStream, ...], count: int) -> tuple[ModelResponseStream, ...]: + lengths: Final = tuple(len(_delta_text(chunk)) for chunk in chunks) + offsets: Final = tuple(accumulate(lengths, initial=0))[:-1] + return tuple( + _without_leading_text(chunk, min(length, count - offset)) + for chunk, offset, length in zip(chunks, offsets, lengths, strict=True) + ) + + +@dataclass(frozen=True, slots=True) +class ContinuedPartial: + usage: Usage + response_cost: float + response_id: str + + +class ContinuationOutput: + """Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the + prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing + whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and + runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A + continuation nested in another leaves the rules to the outermost one, the only one holding the whole text.""" + + def __init__( + self, + emitted_text: str, + prefill_message: Mapping[str, object], + model: str, + primary_response: object, + checks_post_call_rules: bool = True, + ) -> None: + self._text = emitted_text + self._checks_post_call_rules: Final = checks_post_call_rules + self._prefill_text: Final = _prefill_text(prefill_message) + self._model: Final = model + self._identity: Final = _stream_identity(primary_response) + self._rules: Final = Rules() + self._held: tuple[ModelResponseStream, ...] = () + self._echo_settled = not self._prefill_text + self._seam = _resendable_whitespace(self._prefill_text) + + async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]: + async for item in stream: + if not isinstance(item, ModelResponseStream): + yield item + continue + for released in self._observed(self._past_seam(self._released(item))): + yield self._in_primary_stream(released) + for released in self._observed(self._past_seam(self._flushed())): + yield self._in_primary_stream(released) + + def _in_primary_stream(self, chunk: ModelResponseStream) -> ModelResponseStream: + return _without_role(chunk.model_copy(update=self._identity) if self._identity else chunk) + + def _flushed(self) -> tuple[ModelResponseStream, ...]: + held: Final = self._held + self._held = () + self._echo_settled = True + return held + + def _released(self, chunk: ModelResponseStream) -> tuple[ModelResponseStream, ...]: + if self._echo_settled: + return (chunk,) + held: Final = (*self._held, chunk) + text: Final = "".join(_delta_text(held_chunk) for held_chunk in held) + if len(text) < len(self._prefill_text) and self._prefill_text.startswith(text): + self._held = held + return () + self._held = () + self._echo_settled = True + if not text.startswith(self._prefill_text): + return held + return _drop_leading_text(held, len(self._prefill_text)) + + def _past_seam(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]: + return tuple(self._without_seam(chunk) for chunk in chunks) + + def _without_seam(self, chunk: ModelResponseStream) -> ModelResponseStream: + text: Final = _delta_text(chunk) + matched: Final = len(commonprefix((text, self._seam))) + self._seam = self._seam[matched:] if matched == len(text) else "" + return _without_leading_text(chunk, matched) + + def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]: + if not self._checks_post_call_rules or not litellm.post_call_rules: + return chunks + for chunk in chunks: + self._text += _delta_text(chunk) + self._rules.post_call_rules(input=self._text, model=self._model) + return chunks diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0bcb732a0f5..7df8cc7452f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4045,7 +4045,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"}) -DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_regex_lookaround"}) +DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset( + {"supports_regex_lookaround", "supports_assistant_prefill"} +) SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = ( frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__) diff --git a/litellm/utils.py b/litellm/utils.py index 65408663f6f..ea36a68ad5a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3060,6 +3060,20 @@ def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None ) +def supports_assistant_prefill(model: str, custom_llm_provider: str | None = None) -> bool: + """ + Whether the model can continue a prefilled assistant message (Anthropic's + ``prefix: True`` trick and equivalents). Missing metadata reads as False, + so a mid-stream fallback continuation is only routed to a model known to + support it. + """ + return _supports_factory( + model=model, + custom_llm_provider=custom_llm_provider, + key="supports_assistant_prefill", + ) + + def supports_thinking_cache_preservation(model: str, custom_llm_provider: str | None = None) -> bool: return supports_factory( model=model, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b744032fb6c..5fff760d85a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2773,7 +2773,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2808,7 +2808,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2843,7 +2843,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2878,7 +2878,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2913,7 +2913,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -2948,7 +2948,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -3743,7 +3743,7 @@ "max_tokens": 128000, "mode": "chat", "output_cost_per_token": 1.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -15494,7 +15494,7 @@ }, "supports_adaptive_thinking": true, "supports_legacy_thinking": true, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -20832,7 +20832,7 @@ "output_cost_per_token": 2.5000010000000002e-05, "output_dbu_cost_per_token": 0.000357143, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -21047,7 +21047,7 @@ "output_cost_per_token": 1.5000020000000002e-05, "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_legacy_thinking": true, "supports_prompt_caching": true, @@ -49176,7 +49176,7 @@ "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.5e-05, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_prompt_caching": true, @@ -51006,7 +51006,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -51179,7 +51179,7 @@ "supported_regions": [ "global" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -51200,7 +51200,7 @@ "supported_regions": [ "us-central1" ], - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, @@ -58822,7 +58822,7 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, @@ -72334,7 +72334,7 @@ "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "supports_assistant_prefill": true, + "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, "supports_pdf_input": true, diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index a3ed70f4a56..6762e4cd35c 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -114,6 +114,12 @@ ignored_function_names = [ "_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py "_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py "_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py + "_register_continuation_prefill_check", # Tested through the continuation tests in test_router.py + "_set_mid_stream_fallback_continuation", # Tested through the update_settings continuation test in test_router.py + "_is_flag_enabled", # Tested through the string-flag continuation tests in test_router.py + "_params_sent_to_deployment", # Tested through the configured-constraint continuation tests in test_router.py + "_names_deployment_that_cannot_continue", # Tested through the deployment-id continuation tests in test_router.py + "_set_retry_policy", # Tested through update_settings in test_router_retry_policy_update.py "_resolve_pre_call_check_input_tokens", # Tested through the token-count failure pre-call check tests in test_router.py "_params_neither_dropped_nor_allowed", # Tested through the allowed-or-dropped param pre-call check tests in test_router.py "_deployment_rejects_params", # Tested through the unsupported-param pre-call check tests in test_router.py diff --git a/tests/integration/streaming/test_mid_stream_fallback_continuation.py b/tests/integration/streaming/test_mid_stream_fallback_continuation.py new file mode 100644 index 00000000000..8ec52f26dde --- /dev/null +++ b/tests/integration/streaming/test_mid_stream_fallback_continuation.py @@ -0,0 +1,749 @@ +from __future__ import annotations + +import functools +import hashlib +import itertools +import json +import math +import operator +import threading +import uuid +from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import openai +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows, scratch_database +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from openai.types.chat import ChatCompletionChunk +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(4 * graceful_stop_seconds() + 120) + +SETTING: Final = "enable_mid_stream_fallback_continuation" +PARTIAL: Final = ("Alpha beta", " gamma") +JOINED_PARTIAL: Final = "".join(PARTIAL) +RELAY_TEXT: Final = " delta" +BACKUP_TEXT: Final = " omega." +PLAIN_TEXT: Final = "plain answer" +TOOL_PREAMBLE: Final = "Let me check" +BACKUP_USAGE: Final = (40, 6) +MAX_TOKENS: Final = 200 +PRIMARY_PRICES: Final = (0.001, 0.002) +BACKUP_PRICES: Final = (0.003, 0.005) +SHARED_BACKEND: Final = "openai/gpt-4o-mini" +SEPARATE_BACKEND: Final = "openai/gpt-4o" +WEATHER_TOOL: Final[Mapping[str, JsonValue]] = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +TOOL_CALL_DELTA: Final[Mapping[str, JsonValue]] = { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": ""}, +} +NOTHING_SEEN: Final[tuple[ChatCompletionChunk, ...]] = () +ALLOW_EVERYTHING: Final = "def apply_guardrail(inputs, request_data, input_type):\n return allow()\n" +EVENT_BATCH: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +@functools.cache +def cut_gate(marker: str, hop: str, attempt: int) -> threading.Event: + return threading.Event() + + +@functools.cache +def attempts(marker: str, hop: str) -> Iterator[int]: + return itertools.count() + + +def frame(identity: str, delta: Mapping[str, JsonValue], finish: str | None = None) -> bytes: + body: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish}], + } + return b"data: " + json.dumps(body).encode() + b"\n\n" + + +def complete_stream(identity: str, text: str, usage: tuple[int, int]) -> Reply: + usage_frame: Final[dict[str, JsonValue]] = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [], + "usage": {"prompt_tokens": usage[0], "completion_tokens": usage[1], "total_tokens": sum(usage)}, + } + return Reply( + content_type="text/event-stream", + chunks=( + frame(identity, {"role": "assistant", "content": text}), + frame(identity, {}, finish="stop"), + b"data: " + json.dumps(usage_frame).encode() + b"\n\n", + b"data: [DONE]\n\n", + ), + ) + + +def cut_after_caller_sees(marker: str, hop: str, *frames: bytes) -> Reply: + return Reply( + content_type="text/event-stream", + chunks=(b"".join(frames), b"data: never sent\n\n"), + abort_after=1, + gate_after_first=cut_gate(marker, hop, next(attempts(marker, hop))), + ) + + +def first_user_text(request: Request) -> str: + messages: Final = JSON_OBJECT.validate_json(request.body)["messages"] + assert isinstance(messages, list), messages + first: Final = object_value(messages[0])["content"] + assert isinstance(first, str), first + return first + + +def marker_of(request: Request) -> str: + return first_user_text(request).split()[-1] + + +def is_chat(request: Request) -> bool: + return request.target.endswith("/chat/completions") + + +def model_list() -> Reply: + return Reply(body=b'{"object": "list", "data": []}') + + +def primary(request: Request) -> Reply: + if not is_chat(request): + return model_list() + scenario: Final = first_user_text(request).split()[0] + marker: Final = marker_of(request) + identity: Final = f"chatcmpl-{marker}-primary" + if scenario == "reject": + return Reply(status=500, body=b'{"error": {"message": "primary is down", "type": "server_error"}}') + if scenario == "tool-cut": + return cut_after_caller_sees( + marker, + "primary", + frame(identity, {"role": "assistant", "content": TOOL_PREAMBLE}), + frame(identity, {"tool_calls": [dict(TOOL_CALL_DELTA)]}), + ) + return cut_after_caller_sees( + marker, + "primary", + frame(identity, {"role": "assistant", "content": PARTIAL[0]}), + frame(identity, {"content": PARTIAL[1]}), + ) + + +def relay(request: Request) -> Reply: + if not is_chat(request): + return model_list() + marker: Final = marker_of(request) + return cut_after_caller_sees( + marker, "relay", frame(f"chatcmpl-{marker}-relay", {"role": "assistant", "content": RELAY_TEXT}) + ) + + +def backup(request: Request) -> Reply: + if not is_chat(request): + return model_list() + return complete_stream(f"chatcmpl-{marker_of(request)}-backup", BACKUP_TEXT, BACKUP_USAGE) + + +def no_prefill(request: Request) -> Reply: + if not is_chat(request): + return model_list() + return complete_stream(f"chatcmpl-{marker_of(request)}-plain", PLAIN_TEXT, (9, 2)) + + +def accept_events(request: Request) -> Reply: + return Reply() + + +def registered_guardrail(name: str, mode: str) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "custom_code", + "mode": mode, + "default_on": False, + "custom_code": ALLOW_EVERYTHING, + }, + } + + +@dataclass(frozen=True, slots=True) +class Rig: + gateway: Gateway + database_url: str + primary: Wire + relay: Wire + backup: Wire + no_prefill: Wire + events: Wire + + +def joined_text(chunks: Iterable[ChatCompletionChunk]) -> str: + return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) + + +def has_tool_call(chunks: Iterable[ChatCompletionChunk]) -> bool: + return any(choice.delta.tool_calls for chunk in chunks for choice in chunk.choices) + + +@dataclass(frozen=True, slots=True) +class Release: + gate: threading.Event + ready: Callable[[tuple[ChatCompletionChunk, ...]], bool] + + +def after_text(marker: str, hop: str, text: str, attempt: int = 0) -> Release: + return Release(cut_gate(marker, hop, attempt), lambda seen: joined_text(seen) == text) + + +@dataclass(frozen=True, slots=True) +class Streamed: + chunks: tuple[ChatCompletionChunk, ...] + error: openai.APIError | None + + @property + def text(self) -> str: + return joined_text(self.chunks) + + @property + def ids(self) -> frozenset[str]: + return frozenset(chunk.id for chunk in self.chunks) + + @property + def role_deltas(self) -> int: + return sum(1 for chunk in self.chunks for choice in chunk.choices if choice.delta.role is not None) + + @property + def finish_reasons(self) -> tuple[str, ...]: + return tuple(choice.finish_reason for chunk in self.chunks for choice in chunk.choices if choice.finish_reason) + + @property + def usage(self) -> tuple[int, int]: + reported: Final = tuple(chunk.usage for chunk in self.chunks if chunk.usage is not None) + assert reported, self.chunks + return (reported[-1].prompt_tokens, reported[-1].completion_tokens) + + +def deployment( + name: str, + wire: Wire, + prices: tuple[float, float], + *, + backend: str = SHARED_BACKEND, + prefill: bool | None = None, +) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": backend, + "api_base": wire.url + "/v1", + "api_key": "synthetic-continuation-key", + "input_cost_per_token": prices[0], + "output_cost_per_token": prices[1], + }, + **({"model_info": {"supports_assistant_prefill": prefill}} if prefill is not None else {}), + } + + +def write_config( + directory: Path, + model_entries: Sequence[Mapping[str, JsonValue]], + fallbacks: Sequence[Mapping[str, JsonValue]], + router_settings: Mapping[str, JsonValue], + guardrails: Sequence[Mapping[str, JsonValue]], +) -> Path: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final = { + **base, + "model_list": [dict(entry) for entry in model_entries], + "litellm_settings": { + **object_value(base["litellm_settings"]), + "callbacks": ["generic_api"], + "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, + }, + "guardrails": [dict(entry) for entry in guardrails], + "router_settings": { + **object_value(base["router_settings"]), + "num_retries": 0, + "fallbacks": [dict(entry) for entry in fallbacks], + **router_settings, + }, + } + path: Final = directory / f"continuation-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def booted_rig( + directory: Path, + fallbacks: Sequence[Mapping[str, JsonValue]], + router_settings: Mapping[str, JsonValue], + workers: int, + guardrails: Sequence[Mapping[str, JsonValue]] = (), +) -> Generator[Rig]: + with ExitStack() as stack: + shared: Final = stack.enter_context(gateway_from_environment()) + database_url: Final = stack.enter_context(scratch_database()) + wires: Final = tuple( + stack.enter_context(wire_server(respond)) for respond in (primary, relay, backup, no_prefill, accept_events) + ) + primary_wire, relay_wire, backup_wire, no_prefill_wire, events_wire = wires + config: Final = write_config( + directory, + ( + *( + deployment(name, primary_wire, PRIMARY_PRICES) + for name in ("primary", "primary-skip", "primary-none", "primary-relay", "primary-shared") + ), + deployment("relay", relay_wire, PRIMARY_PRICES, prefill=True), + deployment("backup", backup_wire, BACKUP_PRICES, prefill=True), + deployment("no-prefill", no_prefill_wire, PRIMARY_PRICES, backend=SEPARATE_BACKEND), + deployment("no-prefill-shared", no_prefill_wire, PRIMARY_PRICES), + ), + fallbacks, + router_settings, + guardrails, + ) + gateway: Final = stack.enter_context( + owned_proxy( + shared, + directory, + {"DATABASE_URL": database_url, "GENERIC_LOGGER_ENDPOINT": events_wire.url}, + config=config, + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=workers, + ) + ) + yield Rig(gateway, database_url, primary_wire, relay_wire, backup_wire, no_prefill_wire, events_wire) + + +@pytest.fixture(scope="module") +def configured(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + fallbacks: Final[tuple[Mapping[str, JsonValue], ...]] = ( + {"primary": ["backup"]}, + {"primary-skip": ["no-prefill", "backup"]}, + {"primary-none": ["no-prefill"]}, + {"primary-relay": ["relay"]}, + {"relay": ["backup"]}, + {"primary-shared": ["no-prefill-shared"]}, + ) + with booted_rig( + tmp_path_factory.mktemp("continuation-configured"), + fallbacks, + {SETTING: True}, + 2, + (registered_guardrail("input-check", "pre_call"),), + ) as rig: + yield rig + + +@pytest.fixture(scope="module") +def output_guarded(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with booted_rig( + tmp_path_factory.mktemp("continuation-output-guarded"), + ({"primary": ["backup"]},), + {SETTING: True}, + 2, + (registered_guardrail("output-check", "post_call"),), + ) as rig: + yield rig + + +@pytest.fixture(scope="module") +def unset(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + with booted_rig( + tmp_path_factory.mktemp("continuation-unset"), ({"primary": ["no-prefill", "backup"]},), {}, 1 + ) as rig: + yield rig + + +def new_key(rig: Rig) -> str: + key: Final = rig.gateway.post("/key/generate", {})["key"] + assert isinstance(key, str), key + return key + + +def appended(seen: tuple[ChatCompletionChunk, ...], chunk: ChatCompletionChunk) -> tuple[ChatCompletionChunk, ...]: + return (*seen, chunk) + + +def observed( + response: Iterable[ChatCompletionChunk], releases: Sequence[Release] +) -> Iterator[ChatCompletionChunk | openai.APIError]: + try: + for seen in itertools.accumulate(response, appended, initial=NOTHING_SEEN): + for release in releases: + if release.ready(seen): + release.gate.set() + yield from seen[-1:] + except openai.APIError as error: + yield error + + +def stream( + rig: Rig, + key: str, + model: str, + prompt: str, + releases: Sequence[Release] = (), + extra: Mapping[str, JsonValue] | None = None, +) -> Streamed: + base_url: Final = f"{rig.gateway.client.base_url}".rstrip("/") + "/v1" + try: + with openai.OpenAI(base_url=base_url, api_key=key, max_retries=0, timeout=60) as client: + response: Final = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + stream=True, + stream_options={"include_usage": True}, + max_tokens=MAX_TOKENS, + extra_body=dict(extra or {}), + ) + outcome: Final = tuple(observed(response, releases)) + finally: + for release in releases: + release.gate.set() + errors: Final = tuple(item for item in outcome if isinstance(item, openai.APIError)) + return Streamed( + tuple(item for item in outcome if isinstance(item, ChatCompletionChunk)), errors[0] if errors else None + ) + + +def chat_bodies(wire: Wire, marker: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + JSON_OBJECT.validate_json(request.body) + for request in wire.drain() + if is_chat(request) and marker_of(request) == marker + ) + + +def spend_rows(rig: Rig, key: str) -> list[dict[str, JsonValue]]: + return eventually( + lambda: read_rows( + 'SELECT request_id, status, spend, prompt_tokens, completion_tokens, metadata FROM "LiteLLM_SpendLogs" ' + "WHERE api_key=%s", + (hashlib.sha256(key.encode()).hexdigest(),), + database_url=rig.database_url, + ), + lambda rows: len(rows) >= 1, + seconds=70, + ) + + +def amount(value: JsonValue) -> float: + assert isinstance(value, (int, float)), value + return float(value) + + +def key_spend(rig: Rig, key: str) -> float: + return amount(object_value(rig.gateway.get("/key/info", {"key": key})["info"])["spend"]) + + +def close(observed_value: float, expected: float) -> bool: + return math.isclose(observed_value, expected, rel_tol=1e-9, abs_tol=1e-12) + + +def cost(tokens: tuple[int, int], prices: tuple[float, float]) -> float: + return tokens[0] * prices[0] + tokens[1] * prices[1] + + +def prefill_turn(text: str) -> dict[str, JsonValue]: + return {"role": "assistant", "content": text, "prefix": True} + + +@dataclass(frozen=True, slots=True) +class Billed: + rows: list[dict[str, JsonValue]] + partial: tuple[int, int] + + +def billed_once(rig: Rig, key: str, received: Streamed) -> Billed: + rows: Final = spend_rows(rig, key) + assert [row["status"] for row in rows] == ["success"], rows + row_tokens: Final = (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) + assert isinstance(row_tokens[0], int) and isinstance(row_tokens[1], int), rows + assert received.usage == row_tokens, (received.usage, row_tokens) + partial: Final = (row_tokens[0] - BACKUP_USAGE[0], row_tokens[1] - BACKUP_USAGE[1]) + assert partial[0] > 0 and partial[1] > 0, (row_tokens, BACKUP_USAGE) + return Billed(rows, partial) + + +def assert_spend_prices_every_token(rig: Rig, key: str, billed: Billed) -> None: + partial_cost: Final = cost(billed.partial, PRIMARY_PRICES) + breakdown: Final = object_value(object_value(billed.rows[0]["metadata"])["cost_breakdown"]) + spend: Final = amount(billed.rows[0]["spend"]) + reported_partial_cost: Final = amount(object_value(breakdown["additional_costs"])["mid_stream_partial_cost"]) + assert close(spend, cost(BACKUP_USAGE, BACKUP_PRICES) + partial_cost) and close( + reported_partial_cost, partial_cost + ), f"partial={billed.partial} spend={spend} breakdown={json.dumps(breakdown)}" + eventually(lambda: key_spend(rig, key), lambda value: close(value, spend), seconds=70) + assert spend_rows(rig, key) == billed.rows + + +def test_text_cut_continues_on_the_prefill_fallback_and_bills_both_calls_once(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + prompt: Final = f"text-cut {marker}" + received: Final = stream(configured, key, "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL),)) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks + assert received.ids == {f"chatcmpl-{marker}-primary"}, received.ids + assert received.role_deltas == 1, received.chunks + assert received.finish_reasons == ("stop",), received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + continued: Final = chat_bodies(configured.backup, marker) + assert len(continued) == 1, continued + assert continued[0]["messages"] == [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)] + assert "_mid_stream_continuation" not in continued[0], continued[0] + billed: Final = billed_once(configured, key, received) + assert continued[0]["max_tokens"] == MAX_TOKENS - billed.partial[1], (continued[0], billed.partial) + assert_spend_prices_every_token(configured, key, billed) + + +def test_spend_row_of_a_continued_stream_carries_the_id_the_caller_received(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + received: Final = stream( + configured, key, "primary", f"text-cut {marker}", (after_text(marker, "primary", JOINED_PARTIAL),) + ) + assert received.error is None, received.error + assert [row["request_id"] for row in spend_rows(configured, key)] == [received.chunks[0].id] + + +def test_tool_call_before_the_cut_keeps_the_original_error(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary", + f"tool-cut {marker}", + (Release(cut_gate(marker, "primary", 0), has_tool_call),), + {"tools": [dict(WEATHER_TOOL)]}, + ) + assert received.error is not None, received.chunks + assert (received.text, has_tool_call(received.chunks)) == (TOOL_PREAMBLE, True), received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + assert chat_bodies(configured.backup, marker) == () + + +def test_continuation_skips_a_fallback_without_prefill_support(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + prompt: Final = f"text-cut {marker}" + received: Final = stream( + configured, new_key(configured), "primary-skip", prompt, (after_text(marker, "primary", JOINED_PARTIAL),) + ) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks + assert chat_bodies(configured.no_prefill, marker) == () + assert [body["messages"] for body in chat_bodies(configured.backup, marker)] == [ + [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)] + ] + + +def test_no_prefill_capable_fallback_keeps_the_original_error(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary-none", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + ) + assert received.error is not None, received.chunks + assert "No deployments available" not in str(received.error), received.error + assert received.text == JOINED_PARTIAL, received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + assert chat_bodies(configured.no_prefill, marker) == () + + +def test_undeclared_fallback_sharing_a_backend_with_a_prefill_deployment_keeps_the_original_error( + configured: Rig, +) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary-shared", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + ) + assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks + assert chat_bodies(configured.no_prefill, marker) == () + + +def test_failure_before_content_still_falls_back_to_a_fallback_without_prefill_support(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + prompt: Final = f"reject {marker}" + received: Final = stream(configured, new_key(configured), "primary-none", prompt) + assert received.error is None, received.error + assert received.text == PLAIN_TEXT, received.chunks + assert [body["messages"] for body in chat_bodies(configured.no_prefill, marker)] == [ + [{"role": "user", "content": prompt}] + ] + + +def stream_with_two_cuts(rig: Rig, key: str, marker: str) -> Streamed: + return stream( + rig, + key, + "primary-relay", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL), after_text(marker, "relay", JOINED_PARTIAL + RELAY_TEXT)), + ) + + +def test_second_cut_continues_with_both_partials_in_one_prefill(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + received: Final = stream_with_two_cuts(configured, key, marker) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + RELAY_TEXT + BACKUP_TEXT, received.chunks + assert received.role_deltas == 1, received.chunks + user_turn: Final = {"role": "user", "content": f"text-cut {marker}"} + relayed: Final = chat_bodies(configured.relay, marker) + continued: Final = chat_bodies(configured.backup, marker) + assert [body["messages"] for body in relayed] == [[user_turn, prefill_turn(JOINED_PARTIAL)]] + assert [body["messages"] for body in continued] == [[user_turn, prefill_turn(JOINED_PARTIAL + RELAY_TEXT)]] + partial: Final = billed_once(configured, key, received).partial + relay_ceiling: Final = relayed[0]["max_tokens"] + assert isinstance(relay_ceiling, int), relayed[0] + assert 0 < MAX_TOKENS - relay_ceiling < partial[1], (relayed[0], partial) + assert continued[0]["max_tokens"] == MAX_TOKENS - partial[1], (continued[0], partial) + + +def test_second_cut_bills_every_call_on_one_row(configured: Rig) -> None: + key: Final = new_key(configured) + received: Final = stream_with_two_cuts(configured, key, uuid.uuid4().hex) + assert received.error is None, received.error + assert_spend_prices_every_token(configured, key, billed_once(configured, key, received)) + + +def test_repeating_a_continued_request_never_returns_only_the_continuation(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + key: Final = new_key(configured) + prompt: Final = f"text-cut {marker}" + answers: Final = tuple( + stream(configured, key, "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL, attempt),)) + for attempt in range(2) + ) + assert tuple((answer.error, answer.text) for answer in answers) == ((None, JOINED_PARTIAL + BACKUP_TEXT),) * 2 + + +def test_without_the_setting_a_text_cut_keeps_the_original_error(unset: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + unset, new_key(unset), "primary", f"text-cut {marker}", (after_text(marker, "primary", JOINED_PARTIAL),) + ) + assert received.error is not None, received.chunks + assert received.text == JOINED_PARTIAL, received.chunks + assert len(chat_bodies(unset.primary, marker)) == 1 + assert chat_bodies(unset.no_prefill, marker) == () + assert chat_bodies(unset.backup, marker) == () + + +def router_setting(rig: Rig) -> JsonValue: + return object_value(rig.gateway.get("/router/settings")["current_values"]).get(SETTING) + + +def test_turning_the_setting_on_at_runtime_continues_on_a_prefill_fallback(unset: Rig) -> None: + rig: Final = unset + rig.gateway.post("/config/update", {"router_settings": {SETTING: True}}) + try: + assert router_setting(rig) is True + marker: Final = uuid.uuid4().hex + prompt: Final = f"text-cut {marker}" + received: Final = stream(rig, new_key(rig), "primary", prompt, (after_text(marker, "primary", JOINED_PARTIAL),)) + assert received.error is None, received.error + assert received.text == JOINED_PARTIAL + BACKUP_TEXT, received.chunks + assert chat_bodies(rig.no_prefill, marker) == () + assert [body["messages"] for body in chat_bodies(rig.backup, marker)] == [ + [{"role": "user", "content": prompt}, prefill_turn(JOINED_PARTIAL)] + ] + finally: + rig.gateway.post("/config/update", {"router_settings": {SETTING: False}}) + + +def test_a_registered_output_guardrail_keeps_the_original_error(output_guarded: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + output_guarded, + new_key(output_guarded), + "primary", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + ) + assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks + assert len(chat_bodies(output_guarded.primary, marker)) == 1 + assert chat_bodies(output_guarded.backup, marker) == () + + +def test_a_request_fallback_bringing_its_own_messages_keeps_the_original_error(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + {"fallbacks": [{"model": "backup", "messages": [{"role": "user", "content": f"start over {marker}"}]}]}, + ) + assert (received.error is not None, received.text) == (True, JOINED_PARTIAL), received.chunks + assert len(chat_bodies(configured.primary, marker)) == 1 + assert chat_bodies(configured.backup, marker) == () + + +def carries_tag(event: Mapping[str, JsonValue], marker: str) -> bool: + tags: Final = event.get("request_tags") + return isinstance(tags, list) and marker in tags + + +def tagged_events(batches: Iterable[Request], marker: str) -> tuple[dict[str, JsonValue], ...]: + events: Final = itertools.chain.from_iterable(EVENT_BATCH.validate_json(batch.body) for batch in batches) + return tuple(event for event in events if carries_tag(event, marker)) + + +def logged_events(sink: Wire, marker: str) -> Iterator[tuple[dict[str, JsonValue], ...]]: + return itertools.accumulate((tagged_events(sink.drain(), marker) for _ in itertools.count()), operator.add) + + +def test_the_cut_call_logs_its_failure_under_its_own_deployment_and_price(configured: Rig) -> None: + marker: Final = uuid.uuid4().hex + received: Final = stream( + configured, + new_key(configured), + "primary", + f"text-cut {marker}", + (after_text(marker, "primary", JOINED_PARTIAL),), + {"metadata": {"tags": [marker]}}, + ) + assert received.error is None, received.error + polls: Final = logged_events(configured.events, marker) + events: Final = eventually(lambda: next(polls), lambda logged: len(logged) >= 2, seconds=30) + assert sorted(str(event["status"]) for event in events) == ["failure", "success"], events + failure: Final = next(event for event in events if event["status"] == "failure") + assert failure["model_group"] == "primary", failure + assert str(failure["api_base"]).startswith(configured.primary.url), failure + tokens: Final = (failure["prompt_tokens"], failure["completion_tokens"]) + assert isinstance(tokens[0], int) and isinstance(tokens[1], int) and tokens[0] > 0 and tokens[1] > 0, failure + assert close(amount(failure["response_cost"]), cost((tokens[0], tokens[1]), PRIMARY_PRICES)), failure diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index c878f0e149d..bf977325a85 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -642,6 +642,64 @@ async def test_bedrock_converse_replays_reasoning_to_claude_only_as_base64_of_cl assert assistant_content == [*sent, {"text": "hello"}] +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize( + "model, final_text", + [("us.anthropic.claude-sonnet-4-6", "Leaves fall"), ("us.amazon.nova-pro-v1:0", "Leaves fall \n")], + ids=["claude", "nova"], +) +async def test_bedrock_converse_strips_trailing_whitespace_from_a_final_assistant_turn_for_claude_only( + use_async: bool, model: str, final_text: str +): + """Claude on Converse 400s on a final assistant turn ending in whitespace, which a cut stream's prefill often has.""" + messages: Final = [ + {"role": "user", "content": "Write a haiku."}, + {"role": "assistant", "content": "Sure. "}, + {"role": "user", "content": "About autumn."}, + {"role": "assistant", "content": "Leaves fall \n"}, + ] + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model=model, llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt(messages=messages, model=model, llm_provider="bedrock") + ) + + assert [m["content"] for m in result if m["role"] == "assistant"] == [ + [{"text": "Sure. "}], + [{"text": final_text}], + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_bedrock_converse_keeps_whitespace_between_text_blocks_of_a_final_claude_assistant_turn( + use_async: bool, +): + messages: Final = [ + {"role": "user", "content": "Write a haiku."}, + { + "role": "assistant", + "content": [{"type": "text", "text": "Leaves fall "}, {"type": "text", "text": "softly \n"}], + }, + ] + + result: Final = ( + await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=messages, model="us.anthropic.claude-sonnet-4-6", llm_provider="bedrock" + ) + if use_async + else _bedrock_converse_messages_pt( + messages=messages, model="us.anthropic.claude-sonnet-4-6", llm_provider="bedrock" + ) + ) + + assert result[-1]["content"] == [{"text": "Leaves fall "}, {"text": "softly"}] + + def test_translate_thinking_blocks_keeps_dropping_redacted_data_when_called_the_old_way(): """Outside callers that only pass the thinking blocks keep the old behavior, with no redactedContent sent.""" assert BedrockConverseMessagesProcessor.translate_thinking_blocks_to_reasoning_content_blocks( diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 4347a6639bd..fa649b48a51 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -1019,6 +1019,87 @@ async def test_bridged_stream_mid_stream_fallback_error_is_rebuilt_around_the_pr assert excinfo.value.generated_content == "" +@pytest.mark.asyncio +@pytest.mark.parametrize("merge_reasoning", [True, False], ids=["merged_into_content", "kept_as_reasoning"]) +async def test_mid_stream_break_after_reasoning_disqualifies_continuation(logging_obj: Logging, merge_reasoning: bool): + import httpx + + from litellm.exceptions import MidStreamFallbackError + + class _ReasoningThenCut: + def __init__(self) -> None: + self.chunks = iter( + ( + ModelResponseStream(choices=[{"index": 0, "delta": {"reasoning_content": "Let me think"}}]), + ModelResponseStream(choices=[{"index": 0, "delta": {"content": "The answer"}}]), + ) + ) + + def __aiter__(self) -> "_ReasoningThenCut": + return self + + async def __anext__(self) -> ModelResponseStream: + next_chunk: Final = next(self.chunks, None) + if next_chunk is None: + raise httpx.ReadTimeout("connection dropped") + return next_chunk + + response: Final = CustomStreamWrapper( + completion_stream=_ReasoningThenCut(), model="deepseek-reasoner", logging_obj=logging_obj + ) + response.merge_reasoning_content_in_choices = merge_reasoning + + with pytest.raises(MidStreamFallbackError) as excinfo: + async for _ in response: + pass + + assert excinfo.value.is_pre_first_chunk is False + assert excinfo.value.emitted_disqualifying_content is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "outer_delta,inner_flag,expected", + [ + ({"tool_calls": [{"index": 0}]}, False, True), + ({"content": "Let me check."}, True, True), + ({"content": "Let me check."}, False, False), + ], + ids=["bridge_wrapper_sent_a_tool_call", "inner_stream_sent_a_tool_call", "plain_text_only"], +) +async def test_bridged_stream_rebuild_keeps_the_continuation_disqualifier( + logging_obj: Logging, outer_delta: dict[str, object], inner_flag: bool, expected: bool +): + from litellm.exceptions import InternalServerError, MidStreamFallbackError + + provider_error: Final = InternalServerError(message="upstream 500", llm_provider="azure", model="gpt-6.1-sol") + inner_error: Final = MidStreamFallbackError( + message=str(provider_error), + model="gpt-6.1-sol", + llm_provider="azure", + original_exception=provider_error, + is_pre_first_chunk=False, + emitted_disqualifying_content=inner_flag, + ) + + async def _raise_inner_error(**kwargs: object) -> NoReturn: + raise inner_error + + response: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-6.1-sol", + logging_obj=logging_obj, + custom_llm_provider="azure", + make_call=_raise_inner_error, + ) + response._accumulate_streamed_delta(outer_delta) + + with pytest.raises(MidStreamFallbackError) as excinfo: + await response.__anext__() + + assert excinfo.value.emitted_disqualifying_content is expected + + @pytest.mark.asyncio @pytest.mark.parametrize("sync_mode", [True, False]) async def test_streaming_rate_limit_midstream_fallback_keeps_provider_headers(logging_obj: Logging, sync_mode: bool): @@ -6230,6 +6311,59 @@ class TestStableStreamingResponseId: assert wrapper.model_response_creator().id == "chatcmpl-from-provider" +class TestContinuationDisqualifiers: + """The mid-stream continuation eligibility hinges on classifying which deltas + carry output a text-only prefill cannot represent.""" + + @pytest.mark.parametrize( + "field", + [ + "tool_calls", + "function_call", + "reasoning_content", + "thinking_blocks", + "reasoning_items", + "audio", + "images", + "annotations", + ], + ) + def test_disqualifying_fields_flagged(self, field): + # reasoning_content included: it reaches the caller as visible reasoning a + # text-only prefill cannot carry, so a reasoning fallback would re-derive it + assert CustomStreamWrapper._delta_disqualifies_continuation({field: [{"x": 1}]}) is True + + @pytest.mark.parametrize("delta", [{"content": "hi"}, {}, {"role": "assistant"}]) + def test_plain_text_not_flagged(self, delta): + assert CustomStreamWrapper._delta_disqualifies_continuation(delta) is False + + def test_non_mapping_delta_is_safe(self): + assert CustomStreamWrapper._delta_disqualifies_continuation(object()) is False + + def test_accumulate_grows_text_and_leaves_flag_clear_for_plain_text(self): + wrapper = object.__new__(CustomStreamWrapper) + wrapper.response_uptil_now = "" + wrapper._emitted_disqualifying_content = False + + wrapper._accumulate_streamed_delta({"content": "Hel"}) + wrapper._accumulate_streamed_delta({"content": "lo"}) + + assert wrapper.response_uptil_now == "Hello" + assert wrapper._emitted_disqualifying_content is False + + def test_accumulate_latches_flag_on_disqualifying_delta(self): + wrapper = object.__new__(CustomStreamWrapper) + wrapper.response_uptil_now = "" + wrapper._emitted_disqualifying_content = False + + wrapper._accumulate_streamed_delta({"content": "Hi"}) + wrapper._accumulate_streamed_delta({"tool_calls": [{"index": 0}]}) + wrapper._accumulate_streamed_delta({"content": "there"}) + + assert wrapper.response_uptil_now == "Hithere" + assert wrapper._emitted_disqualifying_content is True + + @pytest.mark.asyncio async def test_async_stream_without_usage_counts_tokens_off_the_event_loop(): from tests.large_text import text diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 650517c92ca..1e353a7a82a 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -461,6 +461,21 @@ def test_transform_messages_sanitizes_empty_content(): assert result[1]["content"] == "Hi" +def test_transform_messages_drops_the_prefill_prefix_flag(): + """Databricks rejects unknown message keys, so a continuation prefill reaches Claude without ``prefix``.""" + messages = [ + {"role": "user", "content": "How do transformers work?"}, + {"role": "assistant", "content": "Transformers use", "prefix": True}, + ] + + result = DatabricksConfig().transform_messages( + messages=messages, model="databricks-claude-sonnet-4-6", is_async=False + ) + + assert result[-1] == {"role": "assistant", "content": "Transformers use"} + assert messages[-1]["prefix"] is True + + def test_transform_request_preserves_unity_model_service_name(): config = DatabricksConfig() result = config.transform_request( diff --git a/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py new file mode 100644 index 00000000000..8fa1c799863 --- /dev/null +++ b/tests/unit/router_utils/pre_call_checks/test_continuation_prefill_check.py @@ -0,0 +1,288 @@ +import pytest + +import litellm +from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ContinuationOutput, + ContinuationPrefillDeploymentCheck, + continuation_output_ceilings, + deployment_supports_prefill, +) + +PREFILL_MODEL = "anthropic/prefill-capable-test-model" +NON_PREFILL_MODEL = "openai/prefill-unknown-test-model" +UNMAPPED_MODEL = "openai/unmapped-test-model" + + +@pytest.fixture(autouse=True) +def _synthetic_cost_map_entries() -> None: + litellm.register_model( + { + PREFILL_MODEL: {"litellm_provider": "anthropic", "mode": "chat", "supports_assistant_prefill": True}, + NON_PREFILL_MODEL: {"litellm_provider": "openai", "mode": "chat"}, + }, + persist_across_reloads=False, + ) + litellm.get_model_info.cache_clear() + + +def _deployment(model: str, dep_id: str) -> dict[str, object]: + return {"litellm_params": {"model": model}, "model_info": {"id": dep_id}} + + +def test_deployment_supports_prefill_reads_capability(): + assert deployment_supports_prefill(_deployment(PREFILL_MODEL, "a")) is True + assert deployment_supports_prefill(_deployment(NON_PREFILL_MODEL, "b")) is False + assert deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}}) is True + assert deployment_supports_prefill({"litellm_params": {"model": PREFILL_MODEL}, "model_info": "bogus"}) is True + + +def test_deployment_model_info_override_wins_over_cost_map(): + # model_info True opts in a model that is not in the cost map + assert ( + deployment_supports_prefill( + {"litellm_params": {"model": "vendor/custom-model"}, "model_info": {"supports_assistant_prefill": True}} + ) + is True + ) + # model_info False opts out a model the cost map would otherwise allow + assert ( + deployment_supports_prefill( + {"litellm_params": {"model": PREFILL_MODEL}, "model_info": {"supports_assistant_prefill": False}} + ) + is False + ) + # model_info without the key falls through to the cost map + assert deployment_supports_prefill(_deployment(PREFILL_MODEL, "z")) is True + + +@pytest.mark.parametrize( + "deployment,expected", + [ + ({"litellm_params": {"model": "bedrock/arn:custom-import", "base_model": PREFILL_MODEL}}, True), + ({"litellm_params": {"model": "azure/my-deployment"}, "model_info": {"base_model": PREFILL_MODEL}}, True), + ( + { + "litellm_params": {"model": "azure/my-deployment", "base_model": NON_PREFILL_MODEL}, + "model_info": {"base_model": PREFILL_MODEL}, + }, + True, + ), + ( + { + "litellm_params": {"model": PREFILL_MODEL, "base_model": NON_PREFILL_MODEL}, + "model_info": {"base_model": NON_PREFILL_MODEL}, + }, + False, + ), + ], + ids=[ + "litellm_params_base_model", + "model_info_base_model", + "model_info_base_model_wins", + "base_model_wins_over_model", + ], +) +def test_deployment_supports_prefill_reads_the_cost_map_key_of_the_base_model(deployment, expected): + assert deployment_supports_prefill(deployment) is expected + + +def test_deployment_supports_prefill_rejects_malformed_deployments(): + assert deployment_supports_prefill({}) is False + assert deployment_supports_prefill({"litellm_params": {}}) is False + assert deployment_supports_prefill("not-a-dict") is False + + +@pytest.mark.asyncio +async def test_filter_is_noop_without_continuation_marker(): + """A normal (non-continuation) request must be passed through untouched, even + if some deployments cannot prefill.""" + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")] + + for request_kwargs in ({}, None, {MID_STREAM_CONTINUATION_KWARG: False}): + result = await check.async_filter_deployments( + model="group", healthy_deployments=deployments, messages=None, request_kwargs=request_kwargs + ) + assert result == deployments + + +@pytest.mark.asyncio +async def test_filter_ignores_forged_client_flag(): + """A client cannot steer routing: a plain truthy value under the marker key + (which the proxy could forward from the request body) is not the internal + sentinel, so the filter leaves the deployment list untouched.""" + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")] + + for forged in (True, "true", 1, {"any": "json"}): + result = await check.async_filter_deployments( + model="group", + healthy_deployments=deployments, + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: forged}, + ) + assert result == deployments + + +@pytest.mark.asyncio +async def test_filter_keeps_only_prefill_capable_on_continuation(): + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(PREFILL_MODEL, "a"), _deployment(NON_PREFILL_MODEL, "b")] + + result = await check.async_filter_deployments( + model="group", + healthy_deployments=deployments, + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + assert [d["model_info"]["id"] for d in result] == ["a"] + + +@pytest.mark.asyncio +async def test_filter_empties_group_when_no_prefill_capable_deployment(): + """No prefill-capable deployment -> empty result, so the router advances the + fallback chain and ultimately surfaces the original error.""" + check = ContinuationPrefillDeploymentCheck() + deployments = [_deployment(NON_PREFILL_MODEL, "b"), _deployment(UNMAPPED_MODEL, "c")] + + result = await check.async_filter_deployments( + model="group", + healthy_deployments=deployments, + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + assert result == [] + + +@pytest.mark.asyncio +async def test_filter_drops_deployments_whose_own_params_constrain_the_continuation(): + check = ContinuationPrefillDeploymentCheck() + plain = _deployment(PREFILL_MODEL, "plain") + constrained = [ + {"litellm_params": {"model": PREFILL_MODEL, **params}, "model_info": {"id": dep_id}} + for dep_id, params in ( + ("json", {"response_format": {"type": "json_object"}}), + ("two_choices", {"extra_body": {"n": 2}}), + ("forced_tool", {"tool_choice": {"type": "function", "function": {"name": "f"}}}), + ("merged_reasoning", {"merge_reasoning_content_in_choices": True}), + ) + ] + + result = await check.async_filter_deployments( + model="group", + healthy_deployments=[plain, *constrained], + messages=None, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + assert [d["model_info"]["id"] for d in result] == ["plain"] + + +def _text_chunk(text: str) -> litellm.ModelResponseStream: + return litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}]) + + +@pytest.mark.parametrize( + "prefill_content,fallback_texts,expected_texts", + [ + ("Hello", ("", "Hel", "lo wor", "ld"), ("", "", " wor", "ld")), + ("Hello", ("Hello",), ("",)), + ("Hello", (" world",), (" world",)), + ("Hello", ("Hel", "p me"), ("Hel", "p me")), + ("Hello", ("Hel",), ("Hel",)), + ([{"type": "text", "text": "Sure: "}, {"type": "text", "text": "Hello"}], ("Sure: Hello", " world"), ("", " world")), + ], + ids=[ + "echo_split_across_chunks_is_dropped", + "exact_echo_then_end_is_dropped", + "continuation_without_echo_passes_through", + "near_echo_is_kept_in_order", + "stream_ending_mid_match_is_kept", + "content_block_prefill_echo_is_dropped", + ], +) +@pytest.mark.asyncio +async def test_continuation_output_drops_only_a_full_echo_of_the_prefill( + prefill_content, fallback_texts, expected_texts +): + output = ContinuationOutput( + "Hello", {"role": "assistant", "content": prefill_content, "prefix": True}, "group", () + ) + + async def fallback_stream(): + for text in fallback_texts: + yield _text_chunk(text) + + released = [chunk async for chunk in output.splice(fallback_stream())] + + assert tuple(chunk.choices[0].delta.content for chunk in released) == expected_texts + + +@pytest.mark.parametrize( + "prefill,fallback_texts,expected_texts", + [ + ("Transformers use ", ("self-attention",), ("self-attention",)), + ("Hello ", ("Hello ", " world"), ("", "world")), + ("Hello \n", ("\nworld",), ("\nworld",)), + ("A.\n\n", ("\n Hi", "\nthere"), (" Hi", "\nthere")), + ("\n\n", ("\nHi",), ("\nHi",)), + ("\nHello\n", ("\n",), ("",)), + ], + ids=[ + "continuation_without_whitespace_passes_through", + "echo_then_re_emitted_space_are_both_dropped", + "different_whitespace_is_kept", + "seam_ends_at_the_first_divergence", + "whitespace_only_prefill_has_no_seam", + "re_emitted_whitespace_at_stream_end_is_dropped", + ], +) +@pytest.mark.asyncio +async def test_continuation_output_drops_only_whitespace_the_client_already_has( + prefill, fallback_texts, expected_texts +): + output = ContinuationOutput(prefill, {"role": "assistant", "content": prefill, "prefix": True}, "group", ()) + + async def fallback_stream(): + for text in fallback_texts: + yield _text_chunk(text) + + released = [chunk async for chunk in output.splice(fallback_stream())] + + assert tuple(chunk.choices[0].delta.content for chunk in released) == expected_texts + + +@pytest.mark.parametrize( + "request_kwargs,expected", + [ + ({"max_tokens": 100}, {"max_tokens": 70}), + ({"max_tokens": 100, "max_completion_tokens": 40}, {"max_tokens": 70, "max_completion_tokens": 10}), + ({}, {}), + ( + {"max_tokens": 100, "extra_body": {"max_tokens": 50, "top_k": 3}}, + {"max_tokens": 70, "extra_body": {"max_tokens": 20, "top_k": 3}}, + ), + ({"extra_body": {"max_completion_tokens": 45}}, {"extra_body": {"max_completion_tokens": 15}}), + ({"extra_body": {"top_k": 3}}, {}), + ], + ids=["max_tokens", "both_ceilings", "no_ceiling", "extra_body_max_tokens", "extra_body_only", "extra_body_no_ceiling"], +) +def test_continuation_output_ceilings_take_the_emitted_tokens_off_every_ceiling_the_request_sends( + request_kwargs, expected +): + assert continuation_output_ceilings(request_kwargs, 30) == expected + + +@pytest.mark.parametrize( + "request_kwargs", + [ + {"max_tokens": 30}, + {"max_tokens": 20}, + {"max_tokens": 100, "max_completion_tokens": 30}, + {"max_tokens": 100, "extra_body": {"max_tokens": 30}}, + ], + ids=["max_tokens_met", "max_tokens_exceeded", "one_of_two_met", "extra_body_met"], +) +def test_continuation_output_ceilings_decline_once_a_ceiling_has_no_tokens_left(request_kwargs): + assert continuation_output_ceilings(request_kwargs, 30) is None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 3bd051d71d4..6bc52ab3dfc 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -1,4 +1,5 @@ import ast, asyncio, importlib, time +import contextlib import copy import functools import gc @@ -12,7 +13,7 @@ import warnings from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping from datetime import datetime, timedelta, timezone from pathlib import Path -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import TYPE_CHECKING, Final, Literal, cast from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -2934,6 +2935,1216 @@ async def test_acompletion_streaming_iterator(): print("\n=== All tests passed! ===") +def _make_midstream_source(error, chunks=None, first_chunk=None): + from unittest.mock import MagicMock + + opening_chunk = ( + first_chunk if first_chunk is not None else MagicMock(choices=[MagicMock(delta=MagicMock(content="Hello"))]) + ) + + class _Source: + def __init__(self): + self.index = 0 + self.chunks = chunks if chunks is not None else [] + self.model = "gpt-4" + self.custom_llm_provider = "openai" + self.logging_obj = MagicMock() + + def __aiter__(self): + return self + + async def __anext__(self): + if self.index == 0: + self.index += 1 + return opening_chunk + raise error + + async def wait_for_failure_logging(self, timeout): + return None + + return _Source() + + +class _FakeFallbackStream: + def __init__(self, item): + self._item = item + self._done = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self._done: + raise StopAsyncIteration + self._done = True + return self._item + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_continues_after_content_when_eligible(): + """Flag on + plain-text break: the router re-enters the fallback chain with an + assistant-prefill continuation and the mid-stream marker, then streams the + fallback's output instead of re-raising.""" + from unittest.mock import AsyncMock, MagicMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ) + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + error = MidStreamFallbackError( + message="Connection lost", + model="gpt-4", + llm_provider="openai", + generated_content="Hello", + is_pre_first_chunk=False, + emitted_disqualifying_content=False, + ) + source = _make_midstream_source(error) + fallback_chunk = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": "world"}}]) + initial_kwargs = {"model": "gpt-4", "stream": True} + + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=source, messages=[{"role": "user", "content": "Hi"}], initial_kwargs=initial_kwargs + ) + collected = [chunk async for chunk in result] + + mock_fallback.assert_awaited_once() + passed_kwargs = mock_fallback.await_args.kwargs["kwargs"] + assert passed_kwargs[MID_STREAM_CONTINUATION_KWARG] is MID_STREAM_CONTINUATION_MARKER + assert passed_kwargs["messages"][-1] == {"role": "assistant", "content": "Hello", "prefix": True} + assert fallback_chunk in collected + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "continuation_text,joined_output_trips_rule", + [("6789", True), ("6788", False)], + ids=["rule_trips_only_on_the_joined_text", "clean_continuation_streams"], +) +async def test_acompletion_streaming_iterator_continuation_runs_post_call_rules_on_the_joined_text( + monkeypatch, continuation_text, joined_output_trips_rule +): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + monkeypatch.setattr(litellm, "post_call_rules", [lambda output: "123-45-6789" not in output]) + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="123-45-", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_chunk = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": continuation_text}}]) + + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + if joined_output_trips_rule: + with pytest.raises(litellm.APIResponseValidationError): + async for _ in result: + pass + else: + assert fallback_chunk in [chunk async for chunk in result] + + +@pytest.mark.asyncio +async def test_a_second_cut_runs_post_call_rules_only_on_the_text_the_caller_received(monkeypatch): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ) + + checked_texts = [] + monkeypatch.setattr(litellm, "post_call_rules", [lambda output: checked_texts.append(output) is None]) + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + def cut(generated_content): + return MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content=generated_content, + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + + messages = [{"role": "user", "content": "Hi"}] + middle_leg = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source( + cut(" there"), + first_chunk=litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " there"}}]), + ), + messages=[*messages, {"role": "assistant", "content": "Hello", "prefix": True}], + initial_kwargs={"model": "gpt-4", "stream": True, MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + backup_leg = _FakeFallbackStream( + litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " friend"}}]) + ) + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(side_effect=[middle_leg, backup_leg]) + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(cut("Hello")), + messages=messages, + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + _ = [chunk async for chunk in result] + + assert checked_texts == ["Hello there", "Hello there friend"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallback_texts", + [("Hel", "lo", " world"), (" world",)], + ids=["fallback_echoes_the_prefill", "fallback_continues_after_the_prefill"], +) +async def test_acompletion_streaming_iterator_continuation_never_repeats_the_partial_output(fallback_texts): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "mistral/mistral-small-latest", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + + async def fallback_stream(): + for text in fallback_texts: + yield litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}]) + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(return_value=fallback_stream()) + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + + continuation = [chunk for chunk in collected if isinstance(chunk, litellm.ModelResponseStream)] + assert "".join(chunk.choices[0].delta.content for chunk in continuation) == " world" + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_continuation_stays_in_the_primary_stream(): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + primary_chunk = litellm.ModelResponseStream( + id="chatcmpl-primary", created=111, choices=[{"index": 0, "delta": {"role": "assistant", "content": "Hello"}}] + ) + + async def fallback_stream(): + yield litellm.ModelResponseStream( + id="msg-fallback", created=222, choices=[{"index": 0, "delta": {"role": "assistant", "content": " wor"}}] + ) + yield litellm.ModelResponseStream( + id="msg-fallback", created=222, choices=[{"index": 0, "delta": {"content": "ld"}}] + ) + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(return_value=fallback_stream()) + ): + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error, chunks=[primary_chunk]), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + + continuation = [chunk for chunk in collected if isinstance(chunk, litellm.ModelResponseStream)] + assert [(chunk.id, chunk.created, chunk.choices[0].delta.role) for chunk in continuation] == [ + ("chatcmpl-primary", 111, None), + ("chatcmpl-primary", 111, None), + ] + assert "".join(chunk.choices[0].delta.content for chunk in continuation) == " world" + + +def _continuation_router(): + return litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + +def _usage_chunk(prompt_tokens, completion_tokens): + from litellm.types.utils import Usage + + return litellm.ModelResponseStream( + model="gpt-4", + choices=[], + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ) + + +async def _drain_with_fallback(router, error, primary_chunks, messages, fallback_items): + from unittest.mock import AsyncMock, patch + + async def fallback_stream(): + for item in fallback_items: + yield item + + with patch.object( + router, "async_function_with_fallbacks_common_utils", new=AsyncMock(return_value=fallback_stream()) + ) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error, chunks=primary_chunks), + messages=messages, + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + streamed = [chunk for chunk in collected if isinstance(chunk, litellm.ModelResponseStream)] + return mock_fallback.await_args.kwargs["kwargs"], streamed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("primary_reports_usage", [True, False], ids=["usage_reported", "usage_not_reported"]) +async def test_acompletion_streaming_iterator_continuation_usage_sums_both_legs(primary_reports_usage): + from litellm.exceptions import MidStreamFallbackError + + messages = [{"role": "user", "content": "Explain transformers in one paragraph."}] + partial = "Transformers use " + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content=partial, + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + content_chunk = litellm.ModelResponseStream( + model="gpt-4", choices=[{"index": 0, "delta": {"role": "assistant", "content": partial}}] + ) + primary_chunks = [content_chunk, _usage_chunk(21, 3)] if primary_reports_usage else [content_chunk] + primary_usage = litellm.stream_chunk_builder(chunks=list(primary_chunks), messages=messages).usage + + _, streamed = await _drain_with_fallback( + _continuation_router(), error, primary_chunks, messages, [_usage_chunk(91, 5)] + ) + + assert primary_usage.prompt_tokens > 0 + assert streamed[-1].usage.prompt_tokens == primary_usage.prompt_tokens + 91 + assert streamed[-1].usage.completion_tokens == primary_usage.completion_tokens + 5 + + +class _CutSSE(httpx.AsyncByteStream): + def __init__(self, content: str) -> None: + self._content: Final = content + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield f'data: {json.dumps({"id": "cut", "object": "chat.completion.chunk", "created": 1700000000, "model": "m", "choices": [{"index": 0, "delta": {"role": "assistant", "content": self._content}, "finish_reason": None}]})}\n\n'.encode() + raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") + + +_BACKUP_SSE: Final = "".join( + f"data: {json.dumps(chunk)}\n\n" + for chunk in ( + {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]}, + {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}}, + ) +) + "data: [DONE]\n\n" + + +class _SpendRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.successes: Final[list[dict[str, object]]] = [] + self.failures: Final[list[dict[str, object]]] = [] + self.spend_rows: Final[list[dict[str, object]]] = [] + self.succeeded: Final = asyncio.Event() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + self.successes.append(kwargs["standard_logging_object"]) + self.spend_rows.append(get_logging_payload(kwargs, response_obj, start_time, end_time)) + self.succeeded.set() + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self.failures.append(kwargs["standard_logging_object"]) + + +class _SlowFailureRecorder(_SpendRecorder): + def __init__(self) -> None: + super().__init__() + self.failed_deployment_ids: Final[list[str]] = [] + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + await asyncio.sleep(0.05) + self.failed_deployment_ids.append(kwargs["litellm_params"]["metadata"]["model_info"]["id"]) + await super().async_log_failure_event(kwargs, response_obj, start_time, end_time) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cut_groups", [("primary",), ("primary", "middle")], ids=["one_break", "two_breaks"]) +async def test_continued_stream_success_log_bills_every_cut_leg( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, cut_groups: tuple[str, ...] +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _SpendRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_CutSSE("Transformers use")) + ) + respx_mock.post(host="api.deepseek.com").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_CutSSE(" learned")) + ) + respx_mock.post(host="api.mistral.ai").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=_BACKUP_SSE) + ) + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "k1", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + }, + { + "model_name": "middle", + "litellm_params": { + "model": "deepseek/deepseek-chat", + "api_key": "k2", + "input_cost_per_token": 0.003, + "output_cost_per_token": 0.004, + }, + }, + {"model_name": "backup", "litellm_params": {"model": "mistral/mistral-large-latest", "api_key": "k3"}}, + ], + fallbacks=[{"primary": ["middle" if "middle" in cut_groups else "backup"]}, {"middle": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + + response: Final = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "How do transformers work?"}], + stream=True, + stream_options={"include_usage": True}, + ) + chunks: Final = [chunk async for chunk in response] + await asyncio.wait_for(recorder.succeeded.wait(), timeout=10) + await asyncio.sleep(0.5) + + backup_prompt_cost, backup_completion_cost = litellm.cost_per_token( + model="mistral/mistral-large-latest", prompt_tokens=91, completion_tokens=5 + ) + success: Final = recorder.successes[-1] + assert len(recorder.failures) == len(cut_groups) + assert [failure["response_cost"] for failure in recorder.failures] == pytest.approx( + [ + failure["prompt_tokens"] * input_price + failure["completion_tokens"] * output_price + for failure, (input_price, output_price) in zip(recorder.failures, ((0.001, 0.002), (0.003, 0.004))) + ] + ) + assert success["response_cost"] == pytest.approx( + sum(failure["response_cost"] for failure in recorder.failures) + backup_prompt_cost + backup_completion_cost + ) + assert success["prompt_tokens"] == sum(failure["prompt_tokens"] for failure in recorder.failures) + 91 + assert success["completion_tokens"] == sum(failure["completion_tokens"] for failure in recorder.failures) + 5 + assert success["cost_breakdown"]["total_cost"] == pytest.approx(success["response_cost"]) + spend_row: Final = recorder.spend_rows[-1] + assert (spend_row["spend"], spend_row["prompt_tokens"], spend_row["completion_tokens"]) == ( + success["response_cost"], + success["prompt_tokens"], + success["completion_tokens"], + ) + client_usage: Final = next(chunk.usage for chunk in reversed(chunks) if getattr(chunk, "usage", None)) + assert (client_usage.prompt_tokens, client_usage.completion_tokens) == ( + success["prompt_tokens"], + success["completion_tokens"], + ) + assert {chunk.id for chunk in chunks} == {spend_row["request_id"]} + + +@pytest.mark.asyncio +async def test_continued_stream_sharing_the_proxy_logging_object_bills_the_cut_to_its_own_deployment( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + recorder: Final = _SlowFailureRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_CutSSE("Transformers use")) + ) + respx_mock.post(host="api.mistral.ai").mock( + return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, text=_BACKUP_SSE) + ) + router: Final = Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "k1", + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + }, + "model_info": {"id": "primary-deployment"}, + }, + {"model_name": "backup", "litellm_params": {"model": "mistral/mistral-large-latest", "api_key": "k3"}}, + ], + fallbacks=[{"primary": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + messages: Final = [{"role": "user", "content": "How do transformers work?"}] + shared_logging, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + model="primary", + messages=messages, + stream=True, + litellm_call_id="shared-call", + ) + + response: Final = await router.acompletion( + model="primary", + messages=messages, + stream=True, + stream_options={"include_usage": True}, + litellm_call_id="shared-call", + litellm_logging_obj=shared_logging, + ) + _ = [chunk async for chunk in response] + await asyncio.wait_for(recorder.succeeded.wait(), timeout=10) + await asyncio.sleep(0.5) + + backup_prompt_cost, backup_completion_cost = litellm.cost_per_token( + model="mistral/mistral-large-latest", prompt_tokens=91, completion_tokens=5 + ) + (failure,) = recorder.failures + assert (failure["model_group"], failure["model_id"]) == ("primary", "primary-deployment") + assert recorder.failed_deployment_ids == ["primary-deployment"] + assert failure["response_cost"] == pytest.approx( + failure["prompt_tokens"] * 0.001 + failure["completion_tokens"] * 0.002 + ) + assert recorder.successes[-1]["response_cost"] == pytest.approx( + failure["response_cost"] + backup_prompt_cost + backup_completion_cost + ) + + +def test_carry_partial_spend_to_fallback_adds_the_partial_the_cut_stream_inherited(): + from litellm.types.utils import Usage + + cut_logging: Final = LiteLLMLogging( + model="deepseek/deepseek-chat", messages=[], stream=True, call_type="acompletion", + start_time=datetime.now(), litellm_call_id="cut", function_id="cut", + ) + cut_logging.model_call_details["response_cost"] = 0.5 + cut_logging.update_environment_variables( + litellm_params={"custom_llm_provider": "deepseek"}, optional_params={}, model="deepseek/deepseek-chat" + ) + cut_logging.record_continued_partial( + usage=Usage(prompt_tokens=40, completion_tokens=7, total_tokens=47), + response_cost=0.001, + response_id="chatcmpl-primary", + ) + fallback_logging: Final = LiteLLMLogging( + model="mistral-large-latest", messages=[], stream=True, call_type="acompletion", + start_time=datetime.now(), litellm_call_id="fallback", function_id="fallback", + ) + + Router._carry_partial_spend_to_fallback( + SimpleNamespace(logging_obj=fallback_logging), + Router._price_cut_partial( + cast(CustomStreamWrapper, SimpleNamespace(logging_obj=cut_logging, model="deepseek/deepseek-chat")), + litellm.ModelResponse( + id="chatcmpl-relay", usage=Usage(prompt_tokens=60, completion_tokens=3, total_tokens=63) + ), + ), + ) + + relay_prompt_cost, relay_completion_cost = litellm.cost_per_token( + model="deepseek/deepseek-chat", prompt_tokens=60, completion_tokens=3 + ) + carried_usage: Final = fallback_logging.model_call_details["continued_partial_usage"] + assert (carried_usage.prompt_tokens, carried_usage.completion_tokens) == (100, 10) + assert fallback_logging.model_call_details["continued_partial_cost"] == pytest.approx( + relay_prompt_cost + relay_completion_cost + 0.001 + ) + assert fallback_logging.model_call_details["continued_response_id"] == "chatcmpl-primary" + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_plain_fallback_keeps_the_fallback_usage(): + from litellm.exceptions import MidStreamFallbackError + + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="anthropic", generated_content="", + is_pre_first_chunk=True, + ) + role_chunk = litellm.ModelResponseStream( + model="gpt-4", choices=[{"index": 0, "delta": {"role": "assistant", "content": ""}}] + ) + + _, streamed = await _drain_with_fallback( + _continuation_router(), + error, + [role_chunk, _usage_chunk(21, 0)], + [{"role": "user", "content": "Hi"}], + [_usage_chunk(91, 5)], + ) + + assert (streamed[-1].usage.prompt_tokens, streamed[-1].usage.completion_tokens) == (91, 5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "partial,fallback_texts,expected_answer", + [ + ("Transformers use ", (" self", "-attention"), "Transformers use self-attention"), + ("def f():\n ", ("\n", " return 1"), "def f():\n return 1"), + ("First.\n", ("\n\nSecond",), "First.\n\nSecond"), + ], + ids=["space", "indentation", "paragraph_break"], +) +async def test_acompletion_streaming_iterator_continuation_does_not_double_trailing_whitespace( + partial, fallback_texts, expected_answer +): + from litellm.exceptions import MidStreamFallbackError + + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content=partial, + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_items = [ + litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": text}}]) for text in fallback_texts + ] + + passed_kwargs, streamed = await _drain_with_fallback( + _continuation_router(), error, [], [{"role": "user", "content": "Hi"}], fallback_items + ) + + assert passed_kwargs["messages"][-1]["content"] == partial + assert partial + "".join(chunk.choices[0].delta.content for chunk in streamed) == expected_answer + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", [True, "true", " True "]) +async def test_update_settings_turns_on_mid_stream_continuation(monkeypatch, enabled): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ContinuationPrefillDeploymentCheck, + ) + + monkeypatch.setattr(litellm, "callbacks", []) + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + ) + router.update_settings(enable_mid_stream_fallback_continuation=enabled) + + error = MidStreamFallbackError( + message="Connection lost", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_chunk = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " world"}}]) + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + collected = [chunk async for chunk in result] + + assert mock_fallback.await_args.kwargs["kwargs"][MID_STREAM_CONTINUATION_KWARG] is MID_STREAM_CONTINUATION_MARKER + assert fallback_chunk in collected + assert any(isinstance(callback, ContinuationPrefillDeploymentCheck) for callback in litellm.callbacks) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error_kwargs,request_kwargs", + [ + ({"emitted_disqualifying_content": True}, {}), + ({"emitted_disqualifying_content": False}, {"response_format": {"type": "json_object"}}), + ({"emitted_disqualifying_content": False}, {"tool_choice": "required"}), + ({"emitted_disqualifying_content": False}, {"merge_reasoning_content_in_choices": True}), + ({"emitted_disqualifying_content": False}, {"n": 2}), + ], + ids=["tool_or_thinking_emitted", "json_mode", "forced_tool_choice", "merged_reasoning", "multiple_choices"], +) +async def test_acompletion_streaming_iterator_declines_ineligible_after_content(error_kwargs, request_kwargs): + """Flag on but the break is not continuation-safe: the router re-raises and + never enters the fallback chain, so no duplicated/rejected request is sent.""" + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, **error_kwargs, + ) + source = _make_midstream_source(error) + initial_kwargs = {"model": "gpt-4", "stream": True, **request_kwargs} + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=source, messages=[{"role": "user", "content": "Hi"}], initial_kwargs=initial_kwargs + ) + with pytest.raises(MidStreamFallbackError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_raises_content_filter_instead_of_continuing(): + """A content filter that stops a stream mid-answer reaches the client as ContentPolicyViolationError, + never as a continuation that lets a backup finish the filtered answer.""" + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "azure/gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + content_filter = litellm.ContentPolicyViolationError(message="filtered", model="gpt-4", llm_provider="azure") + error = MidStreamFallbackError( + message="filtered", model="gpt-4", llm_provider="azure", generated_content="Step one is to obtain the", + is_pre_first_chunk=False, emitted_disqualifying_content=False, original_exception=content_filter, + ) + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + with pytest.raises(litellm.ContentPolicyViolationError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +async def _continues_after_content( + initial_kwargs: dict[str, object], + primary_params: Mapping[str, object] = MappingProxyType({}), + default_litellm_params: dict[str, object] | None = None, + cut_model_id: str = "primary-deployment", +) -> bool: + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "k1", **primary_params}, + "model_info": {"id": "primary-deployment"}, + }, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + { + "model_name": "openai-backup", + "litellm_params": {"model": "gpt-4o", "api_key": "k3"}, + "model_info": {"id": "no-prefill-deployment", "supports_assistant_prefill": False}, + }, + { + "model_name": "claude-backup", + "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k4"}, + "model_info": {"id": "prefill-deployment", "supports_assistant_prefill": True}, + }, + { + "model_name": "json-claude-backup", + "litellm_params": { + "model": "anthropic/claude-3-opus-20240229", + "api_key": "k5", + "response_format": {"type": "json_object"}, + }, + "model_info": {"id": "json-prefill-deployment", "supports_assistant_prefill": True}, + }, + ], + fallbacks=[{"gpt-4": ["backup"]}], + default_litellm_params=default_litellm_params, + enable_mid_stream_fallback_continuation=True, + ) + error: Final = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + fallback_chunk: Final = litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": " world"}}]) + cut_stream: Final = _make_midstream_source(error) + cut_stream._hidden_params = {"model_id": cut_model_id} + with patch.object( + router, + "async_function_with_fallbacks_common_utils", + new=AsyncMock(return_value=_FakeFallbackStream(fallback_chunk)), + ) as mock_fallback: + result: Final = await router._acompletion_streaming_iterator( + model_response=cut_stream, + messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True, **initial_kwargs}, + ) + with contextlib.suppress(MidStreamFallbackError): + _ = [chunk async for chunk in result] + return mock_fallback.await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "event_hook,continues", + [("post_call", False), (None, False), ("pre_call", True)], + ids=["post_call_guardrail", "every_hook_guardrail", "input_only_guardrail"], +) +async def test_mid_stream_continuation_declines_while_an_output_guardrail_is_registered( + monkeypatch: pytest.MonkeyPatch, event_hook: str | None, continues: bool +) -> None: + from litellm.types.guardrails import GuardrailEventHooks + + guardrail: Final = CustomGuardrail( + guardrail_name="output-check", event_hook=GuardrailEventHooks(event_hook) if event_hook else None + ) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + assert await _continues_after_content({}) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallbacks,continues", + [ + ([{"model": "backup", "messages": [{"role": "user", "content": "Say hi"}]}], False), + ([{"model": "backup", "stream": False}], False), + ([{"model": "backup", "max_tokens": 300}], False), + ([{"model": "backup"}], True), + ([{"gpt-4": [{"model": "backup", "stream": False}]}], False), + ([{"*": [{"model": "backup", "max_tokens": 300}]}], False), + ([{"gpt-4": [{"model": "backup"}, "backup"]}], True), + ([{"gpt-4": ["backup"]}, {"other-group": [{"model": "backup", "stream": False}]}], True), + ([{}], True), + ([{"gpt-4": 5}], True), + ], + ids=[ + "entry_with_its_own_messages", + "entry_turning_off_streaming", + "entry_with_its_own_max_tokens", + "entry_with_model_only", + "group_target_turning_off_streaming", + "generic_group_target_with_its_own_max_tokens", + "group_targets_with_model_only", + "override_only_in_another_groups_chain", + "empty_entry", + "non_list_chain", + ], +) +async def test_mid_stream_continuation_declines_a_request_fallback_that_overrides_request_params( + fallbacks: list[object], continues: bool +) -> None: + assert await _continues_after_content({"fallbacks": fallbacks}) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fallbacks,continues", + [ + ([{"gpt-4": ["no-prefill-deployment"]}], False), + ([{"gpt-4": ["backup", "no-prefill-deployment"]}], False), + ([{"model": "no-prefill-deployment"}], False), + ([{"gpt-4": ["prefill-deployment"]}], True), + ([{"gpt-4": ["openai-backup"]}], True), + ([{"gpt-4": ["json-prefill-deployment"]}], False), + ], + ids=[ + "deployment_id_without_prefill", + "deployment_id_without_prefill_later_in_chain", + "flat_entry_naming_a_deployment_id_without_prefill", + "deployment_id_with_prefill", + "group_without_prefill_left_to_the_deployment_filter", + "deployment_id_with_its_own_response_format", + ], +) +async def test_mid_stream_continuation_declines_a_fallback_naming_a_deployment_that_cannot_continue( + fallbacks: list[object], continues: bool +) -> None: + assert await _continues_after_content({"fallbacks": fallbacks}) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "primary_params,default_litellm_params,continues", + [ + ({"response_format": {"type": "json_object"}}, None, False), + ({"n": 2}, None, False), + ({"tool_choice": "required"}, None, False), + ({"merge_reasoning_content_in_choices": True}, None, False), + ({}, {"response_format": {"type": "json_object"}}, False), + ({"response_format": {"type": "text"}}, None, True), + ({"temperature": 0.2}, None, True), + ({"tool_choice": "required"}, {"tool_choice": "auto"}, False), + ({"response_format": {"type": "json_object"}}, {"response_format": {"type": "text"}}, True), + ({"extra_body": {"n": 2}}, None, False), + ], + ids=[ + "deployment_response_format", + "deployment_n", + "deployment_tool_choice", + "deployment_merged_reasoning", + "router_default_response_format", + "deployment_plain_text_format", + "deployment_unrelated_param", + "deployment_tool_choice_over_router_default", + "router_default_format_over_deployment", + "deployment_extra_body_n", + ], +) +async def test_mid_stream_continuation_reads_constraints_configured_on_the_cut_deployment( + primary_params: Mapping[str, object], default_litellm_params: dict[str, object] | None, continues: bool +) -> None: + assert await _continues_after_content({}, primary_params, default_litellm_params) is continues + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra_body,continues", + [({"response_format": {"type": "json_object"}}, False), ({"n": 2}, False), ({"top_k": 5}, True)], + ids=["json_format_in_extra_body", "n_in_extra_body", "unrelated_extra_body_key"], +) +async def test_mid_stream_continuation_reads_constraints_the_request_sends_in_extra_body( + extra_body: dict[str, object], continues: bool +) -> None: + assert await _continues_after_content({"extra_body": extra_body}) is continues + + +@pytest.mark.asyncio +async def test_mid_stream_continuation_declines_when_the_cut_deployment_is_not_in_the_router() -> None: + assert await _continues_after_content({}, cut_model_id="per-request-credential-deployment") is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "routing_strategy", + [ + "simple-shuffle", + "usage-based-routing", + "usage-based-routing-v2", + "least-busy", + "latency-based-routing", + "cost-based-routing", + ], +) +async def test_continuation_routes_only_to_deployments_that_can_continue_under_every_routing_strategy( + routing_strategy: str, +) -> None: + from litellm.router_utils.pre_call_checks.continuation_prefill_check import ( + MID_STREAM_CONTINUATION_KWARG, + MID_STREAM_CONTINUATION_MARKER, + ) + from litellm.types.router import RouterRateLimitError + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "backup", + "litellm_params": {"model": "gpt-4o", "api_key": "k1"}, + "model_info": {"id": "no-prefill-deployment", "supports_assistant_prefill": False}, + }, + ], + routing_strategy=routing_strategy, + enable_mid_stream_fallback_continuation=True, + ) + messages: Final = [{"role": "user", "content": "Hi"}] + plain_pick: Final = await router.async_get_available_deployment( + model="backup", messages=messages, request_kwargs={} + ) + assert plain_pick["model_info"]["id"] == "no-prefill-deployment" + with pytest.raises(RouterRateLimitError): + await router.async_get_available_deployment( + model="backup", + messages=messages, + request_kwargs={MID_STREAM_CONTINUATION_KWARG: MID_STREAM_CONTINUATION_MARKER}, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "flag_kwargs", [{}, {"enable_mid_stream_fallback_continuation": "false"}], ids=["default", "string_false"] +) +async def test_acompletion_streaming_iterator_flag_off_declines_after_content(flag_kwargs): + """Regression guard for the opt-in default: with the flag off, a post-content + break re-raises and never falls back, exactly as before this feature.""" + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + **flag_kwargs, + ) + error = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + source = _make_midstream_source(error) + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=source, messages=[{"role": "user", "content": "Hi"}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + with pytest.raises(MidStreamFallbackError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +def test_build_completion_continuation_input_appends_assistant_prefill(): + messages = [{"role": "user", "content": "hi"}] + built = litellm.Router._build_completion_continuation_input(messages, "partial answer") + assert built[:-1] == messages + assert built[-1] == {"role": "assistant", "content": "partial answer", "prefix": True} + + +@pytest.mark.parametrize( + "prefill_content,second_partial,expected_content", + [ + ("part one ", " part two ", "part one part two "), + ("part one ", "part two ", "part one part two "), + ("First.\n\n", "Second", "First.\n\nSecond"), + ([{"type": "text", "text": "part one "}], " part two ", [{"type": "text", "text": "part one part two "}]), + ], + ids=["separator_sent_again", "separator_not_sent_again", "paragraph_break_not_sent_again", "content_blocks"], +) +def test_build_completion_continuation_input_folds_into_existing_prefill( + prefill_content, second_partial, expected_content +): + """A nested break must not leave two trailing assistant turns: the new partial folds into the prior prefill, + which keeps reading as the text the caller already received.""" + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": prefill_content, "prefix": True}] + built = litellm.Router._build_completion_continuation_input(messages, second_partial) + assert built == [messages[0], {"role": "assistant", "content": expected_content, "prefix": True}] + + +@pytest.mark.parametrize( + "prefill_content,expected_content", + [ + ("Hello", "Hello world"), + ([{"type": "text", "text": "Hello"}], [{"type": "text", "text": "Hello world"}]), + ], + ids=["string_prefill", "content_block_prefill"], +) +def test_build_completion_continuation_input_does_not_double_an_echoed_prefill(prefill_content, expected_content): + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": prefill_content, "prefix": True}] + built = litellm.Router._build_completion_continuation_input(messages, "Hello world") + assert built is not None + assert built[-1] == {"role": "assistant", "content": expected_content, "prefix": True} + + +@pytest.mark.parametrize( + "cut_partial,expected_content", + [("Hel", None), ("Hello", "Hello"), ("Hello wor", "Hello wor")], + ids=["cut_mid_echo", "cut_after_a_full_echo", "cut_after_new_text"], +) +def test_build_completion_continuation_input_declines_a_cut_in_the_middle_of_a_prefill_echo( + cut_partial, expected_content +): + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "Hello", "prefix": True}] + built = litellm.Router._build_completion_continuation_input(messages, cut_partial) + expected = None if expected_content is None else [messages[0], {**messages[1], "content": expected_content}] + assert built == expected + + +@pytest.mark.parametrize( + "assistant_content", ["Earlier answer.", [{"type": "text", "text": "Earlier answer."}]], ids=["string", "blocks"] +) +def test_build_completion_continuation_input_declines_a_trailing_plain_assistant_turn(assistant_content): + """The caller's own assistant turn is history, not a prefill, so the partial is never fused into it.""" + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": assistant_content}] + assert litellm.Router._build_completion_continuation_input(messages, "The capital of France is") is None + + +@pytest.mark.asyncio +async def test_acompletion_streaming_iterator_declines_when_request_ends_in_plain_assistant_turn(): + from unittest.mock import AsyncMock, patch + + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[ + {"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k1"}}, + {"model_name": "backup", "litellm_params": {"model": "anthropic/claude-3-opus-20240229", "api_key": "k2"}}, + ], + fallbacks=[{"gpt-4": ["backup"]}], + enable_mid_stream_fallback_continuation=True, + ) + error = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + + with patch.object(router, "async_function_with_fallbacks_common_utils", new=AsyncMock()) as mock_fallback: + result = await router._acompletion_streaming_iterator( + model_response=_make_midstream_source(error), + messages=[{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Earlier answer."}], + initial_kwargs={"model": "gpt-4", "stream": True}, + ) + with pytest.raises(MidStreamFallbackError): + async for _ in result: + pass + + mock_fallback.assert_not_awaited() + + +def test_build_completion_continuation_input_keeps_structured_assistant_content(): + """Content blocks on a trailing assistant turn stay blocks: the partial continues the last text + block in place (a block boundary would let a provider drop the space between "Sure, " and "here"), + lands as a new text block only after a non-text block, and a content shape that cannot be extended declines.""" + from litellm.router import _extend_trailing_text_block + + messages = [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "Sure, "}], "prefix": True}, + ] + built = litellm.Router._build_completion_continuation_input(messages, "here it is") + assert built is not None + assert [m["role"] for m in built] == ["user", "assistant"] + assert built[-1] == {"role": "assistant", "content": [{"type": "text", "text": "Sure, here it is"}], "prefix": True} + image_block = {"type": "image_url", "image_url": {"url": "https://example.test/a.png"}} + assert _extend_trailing_text_block([image_block], "here it is") == [image_block, {"type": "text", "text": "here it is"}] + assert _extend_trailing_text_block([], "here it is") == [{"type": "text", "text": "here it is"}] + assert ( + litellm.Router._build_completion_continuation_input([{"role": "assistant", "content": 42, "prefix": True}], "x") + is None + ) + + +def test_mid_stream_continuation_eligible_allows_text_response_format(): + """response_format={"type": "text"} is the unconstrained default and must stay + eligible, unlike json_object / json_schema.""" + from litellm.exceptions import MidStreamFallbackError + + router = litellm.Router( + model_list=[{"model_name": "gpt-4", "litellm_params": {"model": "gpt-4", "api_key": "k"}}], + enable_mid_stream_fallback_continuation=True, + ) + e = MidStreamFallbackError( + message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello", + is_pre_first_chunk=False, emitted_disqualifying_content=False, + ) + assert router._mid_stream_continuation_eligible(e=e, request_kwargs={"response_format": {"type": "text"}}) is True + assert router._mid_stream_continuation_eligible(e=e, request_kwargs={"response_format": {"type": "json_object"}}) is False + + +def test_raise_original_mid_stream_error_surfaces_the_provider_exception(): + from litellm.exceptions import MidStreamFallbackError, RateLimitError + + provider_error = RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4") + wrapped = MidStreamFallbackError( + message="rate limited", model="gpt-4", llm_provider="openai", + original_exception=provider_error, generated_content="Hello", + ) + with pytest.raises(RateLimitError) as raised: + litellm.Router._raise_original_mid_stream_error(wrapped) + assert raised.value is provider_error + assert raised.value.__cause__ is wrapped + + bare = MidStreamFallbackError(message="boom", model="gpt-4", llm_provider="openai", generated_content="Hello") + with pytest.raises(MidStreamFallbackError) as bare_raised: + litellm.Router._raise_original_mid_stream_error(bare) + assert bare_raised.value is bare + + @pytest.mark.asyncio async def test_acompletion_streaming_iterator_reraises_original_exception_when_available(): """Async: when the mid-stream MidStreamFallbackError wraps a real provider @@ -8694,6 +9905,11 @@ def test_combine_fallback_usage(): assert chunk.usage.completion_tokens == 5 assert chunk.usage.total_tokens == 15 + Router._combine_fallback_usage(chunk, Usage(prompt_tokens=0, completion_tokens=4, total_tokens=4)) + assert chunk.usage.prompt_tokens == 10 + assert chunk.usage.completion_tokens == 9 + assert chunk.usage.total_tokens == 19 + @pytest.mark.asyncio async def test_acompletion_streaming_iterator_does_not_log_success_on_terminal_failure(): diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index d343f464a77..d8625ff9da3 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -544,13 +544,14 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): ) -def test_regex_lookaround_flag_stays_on_the_deployment_that_set_it() -> None: - """A deployment's ``supports_regex_lookaround`` override must not land on the shared +@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"]) +def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: + """A deployment's capability override must not land on the shared ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it.""" backend_model = "bedrock/us.xai.grok-4.6" deploy_id = "grok-deploy-keep-regex" - builtin_flag = litellm.get_model_info(model=backend_model).get("supports_regex_lookaround") + builtin_flag = litellm.get_model_info(model=backend_model).get(flag) model_keys = { deploy_id: litellm.model_cost.get(deploy_id), backend_model: copy.deepcopy(litellm.model_cost.get(backend_model)), @@ -561,13 +562,13 @@ def test_regex_lookaround_flag_stays_on_the_deployment_that_set_it() -> None: { "model_name": "grok-keep-regex", "litellm_params": {"model": backend_model}, - "model_info": {"id": deploy_id, "supports_regex_lookaround": not builtin_flag}, + "model_info": {"id": deploy_id, flag: not builtin_flag}, } ], ) - assert litellm.model_cost[deploy_id]["supports_regex_lookaround"] is (not builtin_flag) - assert litellm.get_model_info(model=backend_model).get("supports_regex_lookaround") is builtin_flag + assert litellm.model_cost[deploy_id][flag] is (not builtin_flag) + assert litellm.get_model_info(model=backend_model).get(flag) is builtin_flag finally: _restore_model_cost_entries(model_keys) From f81e657903c6ee1969e8173353f875b8785f973f Mon Sep 17 00:00:00 2001 From: Rick <26716961+Bytechoreographer@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:52:15 +0800 Subject: [PATCH 3/3] feat(volcengine): add Doubao-Seedance video generation via the OpenAI /v1/videos surface (#46156) * feat(volcengine): add Doubao-Seedance video generation via the OpenAI /v1/videos surface New VolcEngineVideoConfig maps litellm's OpenAI Videos API onto Volcengine Ark's contents/generations task API: create hits /api/v3/contents/generations/tasks and returns a cgt-... task id, status and content poll /api/v3/contents/generations/tasks/{id}, list maps the same collection (limit -> page_size, Ark's filter.* through extra_query), delete maps DELETE on the task URL, and edit/extension reuse create with a reference_video content item since Ark models both as ordinary generation tasks. Ark's own edit and extension require ratio adaptive (edit also duration -1), so those are pinned unless the caller overrides them. Remix has no Ark equivalent and raises. The body is flat apart from a typed content array: the prompt is a text item and every image/video/audio input is a url item carrying a role, which is the only way to express reference-to-video, edit and extension jobs, so a caller-supplied content array wins outright. input_reference becomes a first_frame item, base64 encoded into a data URI for file inputs. OpenAI params translate per Ark's contract: seconds becomes duration and size yields both a ratio and a named resolution tier, with the ratio snapped onto the set Ark accepts (854x480 reduces to 427:240, which Ark rejects) and the tier taken from the shortest side. Emitting only the ratio leaves the request on the model default and bills at that default's rate. Billing reads the delivered duration and resolution rather than the requested ones, because duration -1 and ratio adaptive both let Ark deliver something else. Ark bills per output token (tokens = width * height * 24 * seconds / 1024), so the cost map carries per-second rates by resolution tier, derived from the console's CNY per million token list rates (2.5 70, 2.0 46, 2.0-fast 37, 2.0-mini 23 without a video input) at the repo's 7.0 CNY/USD convention. Requests that do carry a video input bill at the lower rate instead, which litellm's per-second tiers cannot express, so those are overestimated. usage.total_tokens rides along for invoice reconciliation. Ark echoes the delivered geometry only as a resolution tier plus a ratio, and the 2.0 family's rendered frame does not follow the tier name: its 16:9 "480p" canvas is 864x496, not the 854x480 the tier implies, and its other ratios differ from the tier's nominal shortest side too. A size reconstructed from the tier alone therefore reports pixels the model never produces and scales the 480p rate against the wrong pixel count, so both the reported size and the 2.0 480p rates come from the measured frame (a per-family table, sourced from the Seedance 2.0 Mini model card and re-verified against a live gateway). The create response reports the requested duration and resolution, since only the create, edit and remix calls are costed and Ark's create response carries no usage block, so without it every video logs zero spend. A caller that leaves seconds off is not free either: Ark runs its own default length, which is 5s for the 2.0 family, so that is what the create response bills, while the 2.5 family's default is -1 (Ark picks) and stays unbilled until the status response. The create response also echoes the requested seconds and the frame the job will be rendered at. Editing and extending need the source video as a URL: Ark's task query is a GET while litellm's edit prefetch hook only POSTs, so a task id cannot be resolved into one. The model comes from an explicit extra_body.model, else the encoded video id, else the model the router already resolved, with litellm's provider prefix dropped. The credential's api_base is the Anthropic-compat surface (/api/compatible), so the base is cut at the first /api/ marker rather than by a /api/v3 suffix before the video paths append their own version. The cost-map schema test's exception list gains the 2.0 4K tier, whose $1.28/second rate exceeds its $1 sanity bound, alongside the existing runwayml/seedance2 entry. * fix(volcengine): answer 4xx when a Seedance video is downloaded before it finishes Downloading a task that is still queued or running raised a bare ValueError, which the video content handler synthesizes into a 500. The proxy then retried the same poll twice and reported the caller's own early download as a gateway fault, and the router went looking for a model group fallback for a job that was simply unfinished The download path now raises VolcEngineVideoError carrying the status the OpenAI videos API documents for content that is not ready (404), 400 for a task the provider reported as failed, 404 for an expired task record, and 502 for a succeeded task with no artifact. None of them are 5xx, so the router stops retrying them Regression tests drive avideo_content and assert the caller sees NotFoundError or BadRequestError instead of an internal server error * fix(volcengine): clear the type-discipline and override findings on the Seedance video path The module annotated four mutable collections and carried five `# mutable-ok` markers on lines that suppressed nothing, so the type-discipline gate counted LIT001 x4 and LIT013 x5 against the PR. The content-item and task-dump helpers hand back read-only views of what they build, so they now say Mapping. VideoObject.error is declared as a dict by the type it feeds, so that one marker moves onto the annotation it actually suppresses transform_video_status_retrieve_response also dropped the client keyword the base class declares, which basedpyright reported as an incompatible override * fix(volcengine): bill every Seedance video at create and keep its polls on the deployment that made it Ark bills a task when it finishes while litellm bills it at create, so a request whose cost is not known then is rejected: Ark-picked length (duration -1), frames, a video input, and with it edit and extension. An omitted length is sent as 5s, since the 2.5 family otherwise lets Ark pick it and logged $0. Video ids now carry the requested and the served model group, read from litellm_metadata, so polls and downloads go back to the deployment that created the task. The routing and input_reference helpers move out of the DashScope config into a shared module both providers use. A download before the render finishes answers 400, unreadable or unmapped task bodies answer 502, malformed seconds, size, parameters and Ark fields raise unless drop_params is set, an unpriced resolution tier is refused, and the rendered frame table moves into the cost map. Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (1M context) * fix(volcengine): reject a text item in content so the guarded prompt is the only text sent Input guardrails scan data["prompt"], so a caller could pass a harmless prompt for scanning and a different, unscanned text item in content, which this branch kept and the guarded prompt was dropped. The prompt now always leads the content array and a second text item answers 400. Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (1M context) * fix(volcengine): route video to VOLCENGINE_API_BASE when the deployment sets no api_base * fix(volcengine): let a model with no tier prices keep its resolution instead of rejecting every tier * fix(volcengine): answer 400 for malformed or unknown create fields, send a 720p default tier, and stop the list cursor * fix(volcengine): map size to the closest tier by pixel count, reject draft, remix and list cursors, map user to safety_identifier, read ARK_API_BASE * fix(volcengine): drop unsupported create values under drop_params, take OpenAI's object input_reference, default to 4 seconds, price endpoint ids by base_model * fix(videos): unwrap nested file tuples without recursion * fix(volcengine): price Seedance tiers at Ark's own per-resolution token rates Ark prices 1080p and 4k per token above and below the 480p/720p rate, so the single 46 CNY per million tokens the rows were derived from billed 2.0 1080p and 2.5 1080p about 10% low and 2.0 4k 77% high. Ark's per-resolution table and its own 16:9 examples agree: 1080p 2.48 CNY/s, 4k 5.05 CNY/s for 2.0 and 3.74 CNY/s for 2.5 1080p, which is 51, 26 and 77 CNY per million tokens. Rewrites the tier test to check each rate against the rendered pixels at that tier's own price instead of assuming every tier scales from the base rate, and drops the 4k cost exception that only existed for the old rate. Co-Authored-By: Bytechoreographer Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> * fix(volcengine): unpack Ark parameters sent through extra_body * fix(volcengine): ignore the extra_body keys litellm maps itself The shared request builder merges the caller's raw extra_body back over the mapped params, so a key this adapter maps itself comes back a second time as a field of its own, and the unknown field check turned the supported request into a 400. 2aefc18b0a covered parameters; size, seconds and user still failed that way. All three are already mapped from the top-level copy the same builder writes, so the leftover is ignored rather than rejected. characters and image stay rejected, since Ark has no equivalent and accepting them would hide a parameter that never took effect. Co-Authored-By: Bytechoreographer Co-Authored-By: AaronHowell <111272993+AaronHowell@users.noreply.github.com> * fix(volcengine): reject litellm request fields inside parameters and accept litellm's own controls A seconds, size, user, prompt or nested parameters inside an Ark parameters object, and an extra_body copy of prompt, model or extra_headers, answer 400 again unless drop_params is on, instead of being ignored. A user that is not a string answers 400 the same way. drop_params, additional_drop_params and a forwarded organization are no longer rejected as unknown Ark fields, and additional_drop_params removes the fields it names. * fix(volcengine): apply additional_drop_params before mapping request fields A drop list naming user, seconds, size, parameters or an unsupported field such as image had no effect, because the fields were already mapped to Ark names before the list was applied. The list now removes them first, so the request sends what it would send if those fields were never set --------- Co-authored-by: AaronHowell <111272993+AaronHowell@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: zijie.zhou Co-authored-by: Bytechoreographer --- .../llms/base_llm/videos/async_task_utils.py | 123 ++ .../llms/dashscope/videos/transformation.py | 102 +- litellm/llms/volcengine/videos/__init__.py | 0 .../llms/volcengine/videos/transformation.py | 904 +++++++++++++ ...odel_prices_and_context_window_backup.json | 55 + litellm/utils.py | 4 + model_prices_and_context_window.json | 55 + provider_endpoints_support.json | 3 +- .../test_dashscope_video_transformation.py | 3 +- .../test_volcengine_video_transformation.py | 1131 +++++++++++++++++ 10 files changed, 2284 insertions(+), 96 deletions(-) create mode 100644 litellm/llms/base_llm/videos/async_task_utils.py create mode 100644 litellm/llms/volcengine/videos/__init__.py create mode 100644 litellm/llms/volcengine/videos/transformation.py create mode 100644 tests/unit/llms/volcengine/test_volcengine_video_transformation.py diff --git a/litellm/llms/base_llm/videos/async_task_utils.py b/litellm/llms/base_llm/videos/async_task_utils.py new file mode 100644 index 00000000000..d2fb3ed7606 --- /dev/null +++ b/litellm/llms/base_llm/videos/async_task_utils.py @@ -0,0 +1,123 @@ +import base64 +from collections.abc import Mapping +from io import BufferedReader, BytesIO +from itertools import accumulate, repeat +from pathlib import Path, PurePath +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.images.utils import ImageEditRequestUtils +from litellm.types.videos.utils import decode_video_id_with_provider + +_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) + + +def _fields(value: object) -> Mapping[str, object] | None: + try: + return _FIELDS_ADAPTER.validate_python(value) + except ValidationError: + return None + + +def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]: + return _fields(getattr(logging_obj, "litellm_params", None)) or {} + + +def _id_safe_name(name: object) -> str | None: + return name if isinstance(name, str) and name and ";" not in name else None + + +def _stamped_models(metadata: object) -> tuple[str, str | None] | None: + fields: Final = _fields(metadata) + if fields is None: + return None + requested: Final = _id_safe_name(fields.get("model_group")) + served: Final = _id_safe_name(fields.get("deployment_model_name")) + if requested is None: + return None + return requested, None if served is None or served == requested or "*" in served else served + + +def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None: + """ + A later status or content request checks the key's model access against the model encoded in the id, and + routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the + group that served the task can send its polls back to the deployment that created it, while a key may be + scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one + the router was asked for on this hop, and the group that served it when that differs and is not a wildcard. + The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none. + """ + params: Final = _logged_litellm_params(logging_obj) + stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata")) + return next((models for models in stamped if models), None) + + +def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]: + """ + The served group survives into the returned id only when the router sent this poll to it, so an edited id + cannot write any name it likes back. + """ + video_id: Final = _logged_litellm_params(logging_obj).get("video_id") + if not isinstance(video_id, str): + return None, None + decoded: Final = decode_video_id_with_provider(video_id) + served: Final = decoded.get("served_model") + routed: Final = routed_video_models(logging_obj) + return decoded.get("model_id") or None, served if routed is None or served in routed else None + + +def _tuple_payload(file_obj: object) -> object: + if not isinstance(file_obj, tuple): + return file_obj + file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj) + return file_tuple[1] if len(file_tuple) >= 2 else None + + +def _file_content(file_obj: object) -> object: + unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj) + return next(item for item in unwrapped if not isinstance(item, tuple)) + + +def input_reference_bytes(file_obj: object) -> bytes: + content: Final = _file_content(file_obj) + if isinstance(content, PurePath): + return Path(content).read_bytes() + if isinstance(content, (BytesIO, BufferedReader)): + current_position: Final = content.tell() + _ = content.seek(0) + data: Final = content.read() + _ = content.seek(current_position) + return data + if isinstance(content, bytes): + return content + if isinstance(content, bytearray): + return bytes(content) + read: Final = getattr(content, "read", None) + if callable(read): + read_data: Final[object] = read() + if isinstance(read_data, bytes): + return read_data + raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple") + + +def _reference_object_url(reference: Mapping[str, object]) -> str: + image_url: Final = reference.get("image_url") + if isinstance(image_url, str) and image_url: + return image_url + raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported") + + +def input_reference_url(image: object) -> str: + """A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form + carries that URL, or a data URL, in ``image_url``.""" + if isinstance(image, str): + return image + reference: Final = _fields(image) + if reference is not None: + return _reference_object_url(reference) + content: Final = input_reference_bytes(image) + content_type: Final = ImageEditRequestUtils.get_image_content_type(content) + encoded: Final = base64.b64encode(content).decode("utf-8") + return f"data:{content_type};base64,{encoded}" diff --git a/litellm/llms/dashscope/videos/transformation.py b/litellm/llms/dashscope/videos/transformation.py index 17b685a0e1a..1db91cbb127 100644 --- a/litellm/llms/dashscope/videos/transformation.py +++ b/litellm/llms/dashscope/videos/transformation.py @@ -3,13 +3,10 @@ DashScope (Alibaba Model Studio) async video task API: create, poll and download delete or remix endpoint, so those raise. """ -import base64 import json from collections.abc import Mapping from datetime import datetime -from io import BufferedReader, BytesIO from math import log -from pathlib import Path, PurePath from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final @@ -19,9 +16,13 @@ from pydantic import BaseModel, Field, StrictBool, StrictInt, StrictStr, TypeAda import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.images.utils import ImageEditRequestUtils from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.videos.async_task_utils import ( + input_reference_url, + polled_video_models, + routed_video_models, +) from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # house cached-client factory has no public alias and its stub leaves params untyped @@ -32,7 +33,6 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelInfo from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject from litellm.types.videos.utils import ( - decode_video_id_with_provider, encode_video_id_with_provider, extract_original_video_id, ) @@ -86,7 +86,6 @@ _EMPTY_PARAMS: Final[dict[str, object]] = {} # mutable-ok: BaseVideoConfig cont _TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_TaskResponse) _PARAMETERS_ADAPTER: Final = TypeAdapter(dict[str, object]) -_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...]) _STATUS_MAP: Final = MappingProxyType( { @@ -383,39 +382,6 @@ def _is_priced_tier(resolution: object, priced_tiers: tuple[str, ...]) -> bool: return isinstance(resolution, str) and resolution.strip().upper() in priced_tiers -def _read_all_bytes(file_obj: object) -> bytes: - if isinstance(file_obj, tuple): - file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj) - return _read_all_bytes(file_tuple[1] if len(file_tuple) >= 2 else None) - if isinstance(file_obj, PurePath): - return Path(file_obj).read_bytes() - if isinstance(file_obj, (BytesIO, BufferedReader)): - current_position: Final = file_obj.tell() - file_obj.seek(0) - content: Final = file_obj.read() - file_obj.seek(current_position) - return content - if isinstance(file_obj, bytes): - return file_obj - if isinstance(file_obj, bytearray): - return bytes(file_obj) - read: Final = getattr(file_obj, "read", None) - if callable(read): - data: Final = read() - if isinstance(data, bytes): - return data - raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple") - - -def _image_url(image: object) -> str: - if isinstance(image, str): - return image - content: Final = _read_all_bytes(image) - content_type: Final = ImageEditRequestUtils.get_image_content_type(content) - encoded: Final = base64.b64encode(content).decode("utf-8") - return f"data:{content_type};base64,{encoded}" - - def _media_reference_type(model: str) -> str | None: """Models without the field predate the media array and take the image on the flat ``img_url`` field.""" info: Final = _dashscope_model_info(model) @@ -558,58 +524,6 @@ def _video_url_from_task(task: _TaskResponse, raw_response: httpx.Response) -> s raise _video_error(raw_response, 502, "Video URL not found in task response. The task may not have succeeded yet.") -def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]: - litellm_params: Final[object] = getattr(logging_obj, "litellm_params", None) - try: - return _PARAMETERS_ADAPTER.validate_python(litellm_params) - except ValidationError: - return _EMPTY_PARAMS - - -def _polled_models(logging_obj: object) -> tuple[str | None, str | None]: - """ - The proxy routes by the models encoded in the polled id, so they must survive into the returned id. The served - group survives only when the router sent this poll to it, so an edited id cannot write any name it likes back. - """ - video_id: Final = _logged_litellm_params(logging_obj).get("video_id") - if not isinstance(video_id, str): - return None, None - decoded: Final = decode_video_id_with_provider(video_id) - served: Final = decoded.get("served_model") - routed: Final = _poll_models(logging_obj) - return decoded.get("model_id") or None, served if routed is None or served in routed else None - - -def _poll_models(logging_obj: object) -> tuple[str, str | None] | None: - """ - A later status or content request checks the key's model access against the model encoded in the id, and - routes by it. DashScope serves the same model name from several regional hosts, so only the group that - served the task can send its polls back to the region that created it, while a key may be scoped to the - alias, routing group or wildcard name it asked for instead. The id carries both names: the one the router - was asked for on this hop, and the group that served it when that differs and is not a wildcard. The router - stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none. - """ - params: Final = _logged_litellm_params(logging_obj) - stamped: Final = (_stamped_poll_models(params.get(key)) for key in ("litellm_metadata", "metadata")) - return next((models for models in stamped if models), None) - - -def _stamped_poll_models(metadata: object) -> tuple[str, str | None] | None: - try: - fields: Final = _PARAMETERS_ADAPTER.validate_python(metadata) - except ValidationError: - return None - requested: Final = _id_safe_name(fields.get("model_group")) - served: Final = _id_safe_name(fields.get("deployment_model_name")) - if requested is None: - return None - return requested, None if served is None or served == requested or "*" in served else served - - -def _id_safe_name(name: object) -> str | None: - return name if isinstance(name, str) and name and ";" not in name else None - - class DashScopeVideoConfig(BaseVideoConfig): def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseVideoConfig contract returns list return [ @@ -804,7 +718,7 @@ class DashScopeVideoConfig(BaseVideoConfig): input_reference: Final = video_create_optional_request_params.get("input_reference") if input_reference is None: return None, None - image_url: Final = _image_url(input_reference) + image_url: Final = input_reference_url(input_reference) media_type: Final = _media_reference_type(model) if media_type is None: return "img_url", image_url @@ -823,7 +737,7 @@ class DashScopeVideoConfig(BaseVideoConfig): requested: Final = self._requested_parameters(request_data) video_obj: Final = _video_object_from_task(task, raw_response, model=model, requested=requested) if custom_llm_provider and video_obj.id: - requested_model, served_model = _poll_models(logging_obj) or (model, None) + requested_model, served_model = routed_video_models(logging_obj) or (model, None) video_obj.id = encode_video_id_with_provider( video_obj.id, custom_llm_provider, requested_model, served_model ) @@ -869,7 +783,7 @@ class DashScopeVideoConfig(BaseVideoConfig): client: "HTTPHandler | None" = None, ) -> VideoObject: task: Final = _parse_task_response(raw_response) - model, served_model = _polled_models(logging_obj) + model, served_model = polled_video_models(logging_obj) video_obj: Final = _video_object_from_task(task, raw_response, model=model) if custom_llm_provider and video_obj.id: video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model, served_model) diff --git a/litellm/llms/volcengine/videos/__init__.py b/litellm/llms/volcengine/videos/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/volcengine/videos/transformation.py b/litellm/llms/volcengine/videos/transformation.py new file mode 100644 index 00000000000..a75a4185186 --- /dev/null +++ b/litellm/llms/volcengine/videos/transformation.py @@ -0,0 +1,904 @@ +import json +from collections.abc import Mapping, Sequence +from math import log +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeVar + +import httpx +from httpx._types import RequestFiles +from pydantic import BaseModel, StrictBool, StrictInt, StrictStr, TypeAdapter, ValidationError + +import litellm +from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.videos.async_task_utils import ( + input_reference_url, + polled_video_models, + routed_video_models, +) +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, # pyright: ignore[reportPrivateUsage, reportUnknownVariableType] # house cached-client factory has no public alias and its stub leaves params untyped + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped +) +from litellm.llms.volcengine.common_utils import get_volcengine_base_url +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import ModelInfo +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id +from litellm.utils import get_model_info + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging + + +_ParsedT: Final = TypeVar("_ParsedT") + + +class VolcEngineVideoError(BaseLLMException): + pass + + +class _TaskError(BaseModel, frozen=True): + code: str | None = None + message: str | None = None + + +class _TaskContent(BaseModel, frozen=True): + video_url: str | None = None + + +class _TaskUsage(BaseModel, frozen=True): + total_tokens: int | None = None + + +class _Task(BaseModel, frozen=True): + id: str + status: str + model: str | None = None + error: _TaskError | None = None + content: _TaskContent | None = None + usage: _TaskUsage | None = None + created_at: int | None = None + updated_at: int | None = None + resolution: str | None = None + ratio: str | None = None + duration: int | float | None = None + + +class _TaskCreateResponse(BaseModel, frozen=True): + id: str | None = None + error: _TaskError | None = None + + +class _TaskList(BaseModel, frozen=True): + items: Sequence[_Task] = () + total: int | None = None + + +class _ContentItem(BaseModel, frozen=True): + type: str | None = None + role: str | None = None + + +class _KnownFields(BaseModel, frozen=True): + resolution: StrictStr | None = None + ratio: StrictStr | None = None + seed: StrictInt | None = None + camera_fixed: StrictBool | None = None + watermark: StrictBool | None = None + generate_audio: StrictBool | None = None + + +VOLCENGINE_VIDEO_TASKS_PATH: Final = "/api/v3/contents/generations/tasks" +VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS: Final = 4 +VOLCENGINE_VIDEO_DEFAULT_RESOLUTION: Final = "720p" +_VOLCENGINE_API_MARKER: Final = "/api/" +_VOLCENGINE_COMPAT_SUFFIXES: Final = ("/api", "/v1") +_FRAMES_FIELD: Final = "volcengine_video_frames" + +_EMPTY_PARAMS: Final[dict[str, object]] = {} # mutable-ok: BaseVideoConfig contract empty params + +_TASK_ADAPTER: Final = TypeAdapter(_Task) +_TASK_CREATE_ADAPTER: Final = TypeAdapter(_TaskCreateResponse) +_TASK_LIST_ADAPTER: Final = TypeAdapter(_TaskList) +_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object]) +_CONTENT_ITEMS_ADAPTER: Final = TypeAdapter(tuple[dict[str, object], ...]) +_FRAMES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]]) +_EMPTY_FRAMES: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType({}) + +_STATUS_MAP: Final = MappingProxyType( + { + "queued": "queued", + "running": "in_progress", + "succeeded": "completed", + "failed": "failed", + "cancelled": "cancelled", + "expired": "failed", + } +) +_PENDING_STATUSES: Final = frozenset({"queued", "running"}) +_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "cancelled"}) + +_CREATE_PASSTHROUGH_KEYS: Final = ( + "resolution", + "ratio", + "duration", + "seed", + "camera_fixed", + "watermark", + "generate_audio", + "service_tier", + "priority", + "output_format", + "execution_expires_after", + "callback_url", + "safety_identifier", + "tools", + "omni_reference_task_type", +) + +_REQUEST_METADATA: Final = frozenset({"model", "prompt", "user", "extra_headers", "extra_query", "extra_body"}) +_UNSUPPORTED_BY_VOLCENGINE: Final = frozenset({"characters", "image"}) +_MAPPED_ELSEWHERE: Final = frozenset({*_REQUEST_METADATA, *_UNSUPPORTED_BY_VOLCENGINE, "seconds", "size", "parameters"}) +_READ_FROM_THE_REQUEST: Final = _MAPPED_ELSEWHERE - _UNSUPPORTED_BY_VOLCENGINE +_LITELLM_CONTROLS: Final = frozenset({"drop_params", "additional_drop_params", "organization"}) +_DROP_KEYS_ADAPTER: Final = TypeAdapter(frozenset[str]) + +# litellm/videos/utils.py merges the caller's raw ``extra_body`` back over the mapped params, which returns the keys +# this adapter maps itself a second time, as fields of their own. They are already accounted for, so they count as +# handled rather than unknown. +_MAPPED_EXTRA_BODY_COPIES: Final = frozenset({"size", "seconds", "user"}) +_CREATE_HANDLED_KEYS: Final = frozenset( + { + *_CREATE_PASSTHROUGH_KEYS, + "content", + "input_reference", + "frames", + *_MAPPED_EXTRA_BODY_COPIES, + *_LITELLM_CONTROLS, + } +) + +_VIDEO_INPUT_UNSUPPORTED: Final = ( + "{operation} is not supported through litellm: Ark bills a task with a video input at a different rate per " + "token, and litellm bills the video when the task is created, before that cost is known" +) +_INPUT_VIDEO_ROLE: Final = "reference_video" +_INPUT_VIDEO_TYPE: Final = "video_url" + +_RATIO_SET: Final = ("16:9", "9:16", "4:3", "3:4", "1:1", "21:9") +_RESOLUTION_SHORTEST_SIDE: Final = MappingProxyType({"480p": 480, "720p": 720, "1080p": 1080, "4k": 2160}) +_TIER_FRAME_ASPECT: Final = 16 / 9 + + +def _video_error(raw_response: httpx.Response, status_code: int, message: str) -> VolcEngineVideoError: + """The provider's own error status wins, and a body that failed on a 2xx answers the caller's status instead, + since a 2xx error has the proxy retry a call the provider already rejected.""" + return VolcEngineVideoError( + status_code=raw_response.status_code if raw_response.status_code >= 400 else status_code, + message=message, + headers=raw_response.headers, + ) + + +def _parse(adapter: TypeAdapter[_ParsedT], raw_response: httpx.Response) -> _ParsedT: + try: + return adapter.validate_python(raw_response.json()) + except (json.JSONDecodeError, ValidationError) as e: + raise _video_error(raw_response, 502, f"Unexpected Volcengine video response: {e}") from e + + +def _unsupported(model: str, message: str) -> UnsupportedParamsError: + return UnsupportedParamsError(message=message, model=model, llm_provider="volcengine") + + +def _droppable(model: str, message: str) -> UnsupportedParamsError: + return _unsupported(model, f"{message}, set drop_params=True to ignore it") + + +def _normalized_api_base(api_base: str) -> str: + """The Ark credential is shared with the chat, responses and Anthropic surfaces, so its api_base may carry any of + their paths, and the video paths carry their own version segment.""" + trimmed: Final = api_base.rstrip("/") + marker_index: Final = trimmed.rfind(_VOLCENGINE_API_MARKER) + if marker_index != -1: + return trimmed[:marker_index] + suffix: Final = next((suffix for suffix in _VOLCENGINE_COMPAT_SUFFIXES if trimmed.endswith(suffix)), None) + return trimmed[: -len(suffix)] if suffix is not None else trimmed + + +def _size_dimensions(size: str) -> tuple[int, int] | None: + width_str, separator, height_str = size.partition("x") + if not separator or not (width_str.isdigit() and height_str.isdigit()): + return None + if not int(width_str) or not int(height_str): + return None + return int(width_str), int(height_str) + + +def _ratio_value(ratio: str) -> float: + width_str, _, height_str = ratio.partition(":") + return int(width_str) / int(height_str) + + +def _ratio_from_size(size: str) -> str | None: + """Ark only takes its named ratios, and its own tiers do not reduce onto them (854x480 is 427:240), so the + closest ratio wins.""" + dimensions: Final = _size_dimensions(size) + if dimensions is None: + return None + width, height = dimensions + return min(_RATIO_SET, key=lambda candidate: abs(log(width / height / _ratio_value(candidate)))) + + +def _resolution_from_size(size: str) -> str | None: + """Ark keeps a tier's pixel count across its ratios (960x960 and 1280x720 are both 720p), so the tier whose 16:9 + frame has the closest pixel count wins.""" + dimensions: Final = _size_dimensions(size) + if dimensions is None: + return None + width, height = dimensions + return min( + _RESOLUTION_SHORTEST_SIDE, + key=lambda tier: abs(log(width * height / (_RESOLUTION_SHORTEST_SIDE[tier] ** 2 * _TIER_FRAME_ASPECT))), + ) + + +def _duration_param(seconds: object) -> int | None: + if isinstance(seconds, bool): + return None + if isinstance(seconds, int): + return seconds + if not isinstance(seconds, (str, float)): + return None + try: + parsed: Final = float(seconds) + except ValueError: + return None + return int(parsed) if parsed.is_integer() else None + + +def _duration_fields(seconds: object, model: str, drop_params: bool) -> Mapping[str, object]: + if seconds is None: + return _EMPTY_PARAMS + duration: Final = _duration_param(seconds) + if duration is not None: + return MappingProxyType({"duration": duration}) + if drop_params: + return _EMPTY_PARAMS + raise _droppable(model, f"seconds must be a whole number of seconds, got {seconds!r}") + + +def _size_fields(size: object, model: str, drop_params: bool) -> Mapping[str, object]: + if size is None: + return _EMPTY_PARAMS + ratio: Final = _ratio_from_size(size) if isinstance(size, str) else None + resolution: Final = _resolution_from_size(size) if isinstance(size, str) else None + if ratio is not None and resolution is not None: + return MappingProxyType({"ratio": ratio, "resolution": resolution}) + if drop_params: + return _EMPTY_PARAMS + raise _droppable(model, f"size must be WIDTHxHEIGHT, got {size!r}") + + +def _parameters_fields(parameters: object, model: str, drop_params: bool) -> Mapping[str, object]: + if parameters is None: + return _EMPTY_PARAMS + try: + fields: Final = _FIELDS_ADAPTER.validate_python(parameters) + except ValidationError: + if drop_params: + return _EMPTY_PARAMS + raise _droppable( + model, f"parameters must be an object of Ark request fields, got {type(parameters).__name__}" + ) from None + misplaced: Final = tuple(sorted(key for key in fields if key in _READ_FROM_THE_REQUEST)) + if misplaced and not drop_params: + raise _droppable(model, f"send {', '.join(misplaced)} as top-level request fields, not inside parameters") + return MappingProxyType({key: value for key, value in fields.items() if key not in _READ_FROM_THE_REQUEST}) + + +def _user_fields(user: object, model: str, drop_params: bool) -> Mapping[str, object]: + if isinstance(user, str) and user: + return MappingProxyType({"safety_identifier": user}) + if user is None or isinstance(user, str) or drop_params: + return _EMPTY_PARAMS + raise _droppable(model, f"user must be a string, got {type(user).__name__}") + + +def _additional_drop_keys(params: Mapping[str, object]) -> frozenset[str]: + try: + return _DROP_KEYS_ADAPTER.validate_python(params.get("additional_drop_params") or ()) + except ValidationError: + return frozenset() + + +def _without(params: Mapping[str, object], keys: frozenset[str]) -> Mapping[str, object]: + return MappingProxyType({key: value for key, value in params.items() if key not in keys}) + + +def _malformed_fields(fields: Mapping[str, object]) -> tuple[tuple[str, str], ...]: + try: + _ = _KnownFields.model_validate(dict(fields)) + except ValidationError as e: + return tuple((str(error["loc"][0]), error["msg"]) for error in e.errors() if error["loc"]) + return () + + +def _drops_params(litellm_params: GenericLiteLLMParams) -> bool: + return litellm_params.drop_params is True or litellm.drop_params is True + + +def _model_info(model: str) -> ModelInfo | None: + try: + return get_model_info(model=model, custom_llm_provider="volcengine") + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + return None + + +def _pricing_model(model: str, litellm_params: GenericLiteLLMParams) -> str: + """A deployment priced through ``model_info.base_model``, like an ``ep-`` endpoint id, bills at that model's + tiers.""" + model_info: Final = _FIELDS_ADAPTER.validate_python(litellm_params["model_info"] or _EMPTY_PARAMS) + base_model: Final = model_info.get("base_model") + return base_model if isinstance(base_model, str) and base_model else model + + +def _priced_tiers(model: str) -> tuple[str, ...] | None: + """A tier the model has no price for is one Ark does not offer it at, and litellm would bill it at the base rate. + A model with no tier price listed, missing from the cost map or registered unpriced by the router, has nothing + to check against.""" + info: Final = _model_info(model) + if info is None: + return None + return ( + tuple(tier for tier in _RESOLUTION_SHORTEST_SIDE if info.get(f"output_cost_per_second_{tier}") is not None) + or None + ) + + +def _delivered_frames(model: str | None) -> Mapping[str, Mapping[str, str]]: + info: Final = _model_info(model) if model else None + provider_specific: Final[object] = info.get("provider_specific_entry") if info is not None else None + frames: Final[object] = provider_specific.get(_FRAMES_FIELD) if isinstance(provider_specific, Mapping) else None + try: + return _FRAMES_ADAPTER.validate_python(frames) if frames is not None else _EMPTY_FRAMES + except ValidationError: + return _EMPTY_FRAMES + + +def _size_from_geometry(resolution: str | None, ratio: str | None, model: str | None) -> str | None: + """Ark echoes a tier and a ratio rather than pixels, and its rendered frame does not always follow the tier's + nominal size, so the frame listed for the model wins over scaling the tier's shortest side up the ratio.""" + if not resolution or not ratio or ":" not in ratio: + return None + tier: Final = resolution.strip().lower() + normalized_ratio: Final = ratio.strip() + listed: Final = _delivered_frames(model).get(tier, MappingProxyType({})).get(normalized_ratio) + if listed is not None: + return listed + shortest_side: Final = _RESOLUTION_SHORTEST_SIDE.get(tier) + width_str, _, height_str = normalized_ratio.partition(":") + if shortest_side is None or not (width_str.isdigit() and height_str.isdigit()): + return None + ratio_width: Final = int(width_str) + ratio_height: Final = int(height_str) + if not ratio_width or not ratio_height: + return None + shortest_ratio_side: Final = min(ratio_width, ratio_height) + return f"{-(-shortest_side * ratio_width // shortest_ratio_side)}x{-(-shortest_side * ratio_height // shortest_ratio_side)}" + + +def _is_input_video(item: _ContentItem) -> bool: + return item.type == _INPUT_VIDEO_TYPE or item.role == _INPUT_VIDEO_ROLE + + +def _supplied_items(supplied: object, model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]: + try: + items: Final = _CONTENT_ITEMS_ADAPTER.validate_python(supplied) + entries: Final = tuple(_ContentItem.model_validate(item) for item in items) + except ValidationError: + if drop_params: + return () + raise _droppable(model, "content must be an array of Ark content items") from None + if any(entry.type == "text" for entry in entries) and not drop_params: + raise _droppable( + model, + "A text item in content is not supported, put the text in prompt since it is the only text input " + "guardrails scan before the request reaches Ark", + ) + if any(_is_input_video(entry) for entry in entries) and not drop_params: + raise _droppable(model, _VIDEO_INPUT_UNSUPPORTED.format(operation="Video input (a reference_video item)")) + kept: Final = (item for item, entry in zip(items, entries) if entry.type != "text" and not _is_input_video(entry)) + return tuple(MappingProxyType(item) for item in kept) + + +def _reference_items(params: Mapping[str, object], model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]: + """A caller-supplied ``content`` array wins over ``input_reference``, since its roles are the only way to express + reference-to-video.""" + supplied: Final = params.get("content") + if supplied is not None: + return _supplied_items(supplied, model, drop_params) + reference: Final = params.get("input_reference") + if reference is None: + return () + try: + url: Final = input_reference_url(reference) + except ValueError as e: + if drop_params: + return () + raise _droppable(model, str(e)) from None + return (MappingProxyType({"type": "image_url", "image_url": {"url": url}, "role": "first_frame"}),) + + +def _sent_content( + prompt: str, params: Mapping[str, object], model: str, drop_params: bool +) -> tuple[Mapping[str, object], ...]: + """The prompt always leads, as it is the only text the input guardrails scan.""" + return (MappingProxyType({"type": "text", "text": prompt}), *_reference_items(params, model, drop_params)) + + +def _seconds_label(seconds: object) -> str | None: + if isinstance(seconds, float) and seconds.is_integer(): + return str(int(seconds)) + return None if seconds is None else str(seconds) + + +def _requested_usage(request_data: Mapping[str, object]) -> Mapping[str, object]: + """The create call is the only billed one and Ark's create response carries no usage, so the requested duration + and tier are what the video bills at.""" + duration: Final = _duration_param(request_data.get("duration")) + resolution: Final = request_data.get("resolution") + return MappingProxyType( + { + key: value + for key, value in ( + ("duration_seconds", float(duration) if duration is not None and duration > 0 else None), + ("video_resolution", resolution.strip().lower() if isinstance(resolution, str) else None), + ) + if value is not None + } + ) + + +def _task_usage(task: _Task) -> Mapping[str, object]: + resolution: Final = task.resolution.strip().lower() if task.resolution else None + return MappingProxyType( + { + key: value + for key, value in ( + ("duration_seconds", float(task.duration) if task.duration is not None else None), + ("video_resolution", resolution), + ("total_tokens", task.usage.total_tokens if task.usage is not None else None), + ) + if value is not None + } + ) + + +def _task_status(task: _Task, raw_response: httpx.Response) -> str: + status: Final = _STATUS_MAP.get(task.status.strip().lower()) + if status is None: # defaulting it would hand back a job the caller polls forever + raise _video_error(raw_response, 502, f"Unknown Volcengine task status: {task.status}") + return status + + +def _video_object_from_task(task: _Task, raw_response: httpx.Response, model: str | None) -> VideoObject: + status: Final = _task_status(task, raw_response) + usage: Final = _task_usage(task) + seconds: Final = usage.get("duration_seconds") + error: Final[dict[str, object] | None] = ( # mutable-ok: VideoObject.error is a dict field + {key: value for key, value in (("code", task.error.code), ("message", task.error.message)) if value is not None} + if task.error is not None + else None + ) + return VideoObject( + id=task.id, + object="video", + status=status, + created_at=task.created_at, + completed_at=task.updated_at if status in _TERMINAL_STATUSES else None, + error=error, + seconds=_seconds_label(seconds), + size=_size_from_geometry(task.resolution, task.ratio, task.model), + model=model or task.model, + usage=dict(usage) if usage else None, + ) + + +def _video_url_from_task(task: _Task, raw_response: httpx.Response) -> str: + """A download before the video is ready is the caller polling too early, not a fault of the deployment, so it + answers 400, which litellm neither retries nor counts toward a deployment cooldown. A 404 would cool the + deployment down, and a 409 or a 5xx would be retried against a job that is simply not finished.""" + if task.content is not None and task.content.video_url: + return task.content.video_url + status: Final = task.status.strip().lower() + if status in _PENDING_STATUSES: + raise _video_error( + raw_response, 400, f"Video is still processing (status: {status}). Please wait and try again." + ) + if task.error is not None and (task.error.message or task.error.code): + raise _video_error(raw_response, 400, f"Video generation failed: {task.error.message or task.error.code}") + if status == "expired": + raise _video_error(raw_response, 400, "Task expired. Ark keeps task records for 7 days after creation.") + if status in ("failed", "cancelled"): + raise _video_error(raw_response, 400, f"Video generation ended as {status} with no video URL.") + raise _video_error(raw_response, 502, "Video URL not found in task response.") + + +class VolcEngineVideoConfig(BaseVideoConfig): + """Volcengine Ark (Doubao-Seedance) on Ark's async task API: create, poll, download from the signed + ``content.video_url``, and list.""" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseVideoConfig contract returns list + return [ + "model", + "prompt", + "input_reference", + "seconds", + "size", + "user", + "extra_headers", + "content", + "parameters", + *_CREATE_PASSTHROUGH_KEYS, + ] + + def map_openai_params( + self, + video_create_optional_params: VideoCreateOptionalRequestParams, + model: str, + drop_params: bool, + ) -> dict[str, object]: # mutable-ok: BaseVideoConfig contract + requested: Final = _without(video_create_optional_params, _additional_drop_keys(video_create_optional_params)) + unsupported: Final = next( + (key for key, value in requested.items() if key in _UNSUPPORTED_BY_VOLCENGINE and value is not None), + None, + ) + if unsupported is not None and not drop_params: + raise _droppable(model, f"'{unsupported}' is not supported by Volcengine video generation") + passthrough: Final = MappingProxyType( + {key: value for key, value in requested.items() if value is not None and key not in _MAPPED_ELSEWHERE} + ) + return { + **_user_fields(requested.get("user"), model, drop_params), + **_size_fields(requested.get("size"), model, drop_params), + **_duration_fields(requested.get("seconds"), model, drop_params), + **passthrough, + **_parameters_fields(requested.get("parameters"), model, drop_params), + } + + def _resolve_api_key(self, api_key: str | None) -> str: + resolved_api_key: Final = api_key or get_secret_str("ARK_API_KEY") or get_secret_str("VOLCENGINE_API_KEY") + if resolved_api_key is None: + raise ValueError( + "Volcengine API key is required. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass " + "api_key parameter." + ) + return resolved_api_key + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract; handler expects a mutable headers dict + model: str, + api_key: str | None = None, + litellm_params: GenericLiteLLMParams | None = None, + ) -> dict[str, str]: # mutable-ok: BaseVideoConfig contract + resolved_api_key: Final = self._resolve_api_key( + api_key + or (litellm_params.api_key if litellm_params is not None and litellm_params.api_key else None) + or litellm.api_key + ) + headers.update({"Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}) + return headers + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: BaseVideoConfig contract + ) -> str: + return _normalized_api_base( + get_volcengine_base_url(api_base or get_secret_str("VOLCENGINE_API_BASE") or get_secret_str("ARK_API_BASE")) + ) + + def transform_video_create_request( + self, + model: str, + prompt: str, + api_base: str, + video_create_optional_request_params: dict[str, object], # mutable-ok: BaseVideoConfig contract + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + ) -> tuple[dict[str, object], RequestFiles, str]: # mutable-ok: BaseVideoConfig contract + """ + Ark bills a video when the task finishes, while litellm bills it here at create, so every request has to + carry a length and a tier whose cost is known now. Ark-picked length (``duration: -1``) and a video input + (edits, extensions and reference videos, billed at a different rate per token) are rejected for that reason, + and an omitted length is sent as OpenAI's documented default of 4 seconds, since the 2.5 family's own default + is ``-1``. + """ + drop_params: Final = _drops_params(litellm_params) + dropped_by_config: Final = _additional_drop_keys(video_create_optional_request_params) + kept: Final = _without(video_create_optional_request_params, dropped_by_config) + params: Final = _without( + { + **{key: value for key, value in kept.items() if key != "parameters"}, + **_parameters_fields(kept.get("parameters"), model, drop_params), + }, + dropped_by_config, + ) + content: Final = _sent_content(prompt, params, model, drop_params) + if params.get("frames") is not None and not drop_params: + raise _droppable(model, "frames is not supported through litellm, pass seconds instead") + requested_duration: Final = params.get("duration") + parsed_duration: Final = None if requested_duration is None else _duration_param(requested_duration) + if requested_duration is not None and (parsed_duration is None or parsed_duration <= 0) and not drop_params: + raise _droppable( + model, + f"duration must be a positive whole number of seconds, got {requested_duration!r}. Ark-picked length " + "(-1) is not supported through litellm, since the video is billed when the task is created, before " + "Ark has picked its length", + ) + duration: Final = ( + parsed_duration + if parsed_duration is not None and parsed_duration > 0 + else VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS + ) + + unknown_fields: Final = tuple( + sorted(key for key, value in params.items() if value is not None and key not in _CREATE_HANDLED_KEYS) + ) + if unknown_fields and not drop_params: + raise _droppable(model, f"litellm does not send {', '.join(unknown_fields)} to Ark video generation") + requested_fields: Final = MappingProxyType( + {key: params[key] for key in _CREATE_PASSTHROUGH_KEYS if params.get(key) is not None} + ) + malformed: Final = _malformed_fields(requested_fields) + if malformed and not drop_params: + raise _droppable(model, "; ".join(f"invalid {field}: {message}" for field, message in malformed)) + resolution: Final = requested_fields.get("resolution") + priced_tiers: Final = _priced_tiers(_pricing_model(model, litellm_params)) + drops_resolution: Final = ( + priced_tiers is not None and isinstance(resolution, str) and resolution.strip().lower() not in priced_tiers + ) + if drops_resolution and not drop_params: + raise _droppable( + model, f"{model} is not offered at resolution {resolution!r}, only at {', '.join(priced_tiers or ())}" + ) + dropped: Final = frozenset({*(field for field, _ in malformed), *(("resolution",) if drops_resolution else ())}) + kept_fields: Final = MappingProxyType( + {key: value for key, value in requested_fields.items() if key not in dropped} + ) + default_tier: Final = ( + MappingProxyType({"resolution": VOLCENGINE_VIDEO_DEFAULT_RESOLUTION}) + if "resolution" not in kept_fields + and priced_tiers is not None + and VOLCENGINE_VIDEO_DEFAULT_RESOLUTION in priced_tiers + else _EMPTY_PARAMS + ) + + request_data: Final[dict[str, object]] = { # mutable-ok: request body dict, JSON-serialized by the handler + "model": model, + "content": [dict(item) for item in content], + **default_tier, + **kept_fields, + "duration": duration, + } + return request_data, (), f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}" + + def transform_video_create_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + request_data: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> VideoObject: + task: Final = _parse(_TASK_CREATE_ADAPTER, raw_response) + if not task.id: + message: Final = task.error.message if task.error is not None and task.error.message else "no task id" + raise _video_error(raw_response, 400, f"Volcengine did not return a video task id: {message}") + requested: Final = request_data or _EMPTY_PARAMS + usage: Final = _requested_usage(requested) + seconds: Final = usage.get("duration_seconds") + requested_model, served_model = routed_video_models(logging_obj) or (model, None) + resolution: Final = requested.get("resolution") + ratio: Final = requested.get("ratio") + return VideoObject( + id=encode_video_id_with_provider(task.id, custom_llm_provider, requested_model, served_model) + if custom_llm_provider + else task.id, + object="video", + status="queued", + model=model, + seconds=_seconds_label(seconds), + size=_size_from_geometry( + resolution if isinstance(resolution, str) else None, ratio if isinstance(ratio, str) else None, model + ), + usage=dict(usage) if usage else None, + ) + + @staticmethod + def _task_url(video_id: str, api_base: str) -> str: + original_task_id: Final = extract_original_video_id(video_id) + encoded_task_id: Final = encode_url_path_segment(original_task_id, field_name="video_id") + return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}/{encoded_task_id}" + + def transform_video_status_retrieve_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + return self._task_url(video_id, api_base), _EMPTY_PARAMS + + def transform_video_status_retrieve_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + client: "HTTPHandler | None" = None, + ) -> VideoObject: + task: Final = _parse(_TASK_ADAPTER, raw_response) + model, served_model = polled_video_models(logging_obj) + video_obj: Final = _video_object_from_task(task, raw_response, model) + if custom_llm_provider: + video_obj.id = encode_video_id_with_provider(task.id, custom_llm_provider, model, served_model) + return video_obj + + def transform_video_content_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + variant: str | None = None, + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + return self._task_url(video_id, api_base), _EMPTY_PARAMS + + def transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + client: "HTTPHandler | None" = None, + ) -> bytes: + video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response) + httpx_client: Final[HTTPHandler] = client if client is not None else _get_httpx_client() + video_response: Final = httpx_client.get(video_url) # pyright: ignore[reportUnknownMemberType] # HTTPHandler.get stub leaves params/headers untyped + video_response.raise_for_status() + return video_response.content + + async def async_transform_video_content_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + client: "AsyncHTTPHandler | None" = None, + ) -> bytes: + video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response) + async_httpx_client: Final[AsyncHTTPHandler] = ( + client if client is not None else get_async_httpx_client(llm_provider=litellm.LlmProviders.VOLCENGINE) + ) + video_response: Final = await async_httpx_client.get(video_url) # pyright: ignore[reportUnknownMemberType] # HTTPHandler.get stub leaves params/headers untyped + video_response.raise_for_status() + return video_response.content + + def transform_video_remix_request( + self, + video_id: str, + prompt: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + extra_body: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video remix")) + + def transform_video_remix_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + ) -> VideoObject: + raise NotImplementedError("Video remix is not supported by Volcengine.") + + def transform_video_edit_request( + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + video_file: object | None = None, + extra_body: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + prefetched_source_data: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, Mapping[str, object], RequestFiles | None]: + raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video edit")) + + def transform_video_extension_request( + self, + prompt: str, + video_id: str, + seconds: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + extra_body: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video extension")) + + def transform_video_list_request( + self, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + after: str | None = None, + limit: int | None = None, + order: str | None = None, + extra_query: dict[str, object] | None = None, # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + """Ark pages by page_num/page_size with filters and has no cursor or order.""" + unsupported: Final = tuple(name for name, value in (("after", after), ("order", order)) if value is not None) + if unsupported and not _drops_params(litellm_params): + raise _droppable( + "", f"Ark's task list pages by page_num in extra_query and has no {' or '.join(unsupported)}" + ) + return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}", { + **({"page_size": str(limit)} if limit is not None else {}), + **(extra_query or {}), + } + + def transform_video_list_response( # pyright: ignore[reportIncompatibleMethodOverride] # base declares dict[str, str] but the payload is a heterogeneous list body + self, + raw_response: httpx.Response, + logging_obj: "Logging", + custom_llm_provider: str | None = None, + ) -> dict[str, object]: # mutable-ok: OpenAI list body served as JSON by the proxy + list_payload: Final = _parse(_TASK_LIST_ADAPTER, raw_response) + videos: Final = tuple(_video_object_from_task(task, raw_response, None) for task in list_payload.items) + data: Final = [ + video.model_dump() + | ( + {"id": encode_video_id_with_provider(video.id, custom_llm_provider, video.model)} + if custom_llm_provider + else {} + ) + for video in videos + ] + return { + "object": "list", + "data": data, + "total": list_payload.total if list_payload.total is not None else len(data), + "has_more": False, + **({"first_id": data[0]["id"], "last_id": data[-1]["id"]} if data else {}), + } + + def transform_video_delete_request( + self, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: BaseVideoConfig contract + ) -> tuple[str, dict[str, object]]: # mutable-ok: BaseVideoConfig contract + raise NotImplementedError("Video delete is not supported by Volcengine.") + + def transform_video_delete_response( + self, + raw_response: httpx.Response, + logging_obj: "Logging", + ) -> VideoObject: + raise NotImplementedError("Video delete is not supported by Volcengine.") + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseVideoConfig contract + ) -> BaseLLMException: + return VolcEngineVideoError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5fff760d85a..49d2ce90af4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -60467,6 +60467,61 @@ } ] }, + "volcengine/doubao-seedance-2-5-260628": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.216, + "output_cost_per_second_480p": 0.096075, + "output_cost_per_second_720p": 0.216, + "output_cost_per_second_1080p": 0.5346, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.141942857, + "output_cost_per_second_480p": 0.066003429, + "output_cost_per_second_720p": 0.141942857, + "output_cost_per_second_1080p": 0.354085714, + "output_cost_per_second_4k": 0.722057143, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-fast-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.114171429, + "output_cost_per_second_480p": 0.053089714, + "output_cost_per_second_720p": 0.114171429, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-mini-260615": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.070971429, + "output_cost_per_second_480p": 0.033001714, + "output_cost_per_second_720p": 0.070971429, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, diff --git a/litellm/utils.py b/litellm/utils.py index ea36a68ad5a..d2ca9faef41 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9947,6 +9947,10 @@ class ProviderConfigManager: return MinimaxVideoConfig() elif LlmProviders.EDENAI == provider: return litellm.EdenAIVideoConfig() + elif LlmProviders.VOLCENGINE == provider: + from litellm.llms.volcengine.videos.transformation import VolcEngineVideoConfig + + return VolcEngineVideoConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5fff760d85a..49d2ce90af4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -60467,6 +60467,61 @@ } ] }, + "volcengine/doubao-seedance-2-5-260628": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.216, + "output_cost_per_second_480p": 0.096075, + "output_cost_per_second_720p": 0.216, + "output_cost_per_second_1080p": 0.5346, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.141942857, + "output_cost_per_second_480p": 0.066003429, + "output_cost_per_second_720p": 0.141942857, + "output_cost_per_second_1080p": 0.354085714, + "output_cost_per_second_4k": 0.722057143, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-fast-260128": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.114171429, + "output_cost_per_second_480p": 0.053089714, + "output_cost_per_second_720p": 0.114171429, + "supported_modalities": ["text", "image", "video", "audio"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, + "volcengine/doubao-seedance-2-0-mini-260615": { + "litellm_provider": "volcengine", + "mode": "video_generation", + "source": "https://www.volcengine.com/docs/82379/1544106", + "output_cost_per_second": 0.070971429, + "output_cost_per_second_480p": 0.033001714, + "output_cost_per_second_720p": 0.070971429, + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["video"], + "supported_endpoints": ["/v1/videos"], + "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}}, + "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."} + }, "bedrock/us-east-1/zai.glm-5": { "input_cost_per_token": 1e-06, "output_cost_per_token": 3.2e-06, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 45104d40018..0325fe1f796 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2766,7 +2766,8 @@ "batches": false, "rerank": false, "a2a": true, - "interactions": true + "interactions": true, + "video_generations": true } }, "voyage": { diff --git a/tests/unit/llms/dashscope/test_dashscope_video_transformation.py b/tests/unit/llms/dashscope/test_dashscope_video_transformation.py index 65eefcab1af..15b77aaee3d 100644 --- a/tests/unit/llms/dashscope/test_dashscope_video_transformation.py +++ b/tests/unit/llms/dashscope/test_dashscope_video_transformation.py @@ -194,8 +194,9 @@ class TestDashScopeVideoInputReference: lambda tmp_path: ("first.png", PNG_BYTES, "image/png"), lambda tmp_path: ("first.png", io.BytesIO(PNG_BYTES)), lambda tmp_path: _write_png(tmp_path), + lambda tmp_path: ("first.png", ("inner.png", PNG_BYTES)), ], - ids=["filename-bytes-content-type", "filename-file", "path"], + ids=["filename-bytes-content-type", "filename-file", "path", "nested-tuple"], ) def test_file_tuple_and_path_references_are_encoded(self, make_reference, tmp_path): """Regression: tuples and paths are valid FileTypes but raised before reaching DashScope.""" diff --git a/tests/unit/llms/volcengine/test_volcengine_video_transformation.py b/tests/unit/llms/volcengine/test_volcengine_video_transformation.py new file mode 100644 index 00000000000..1fcf49eeb13 --- /dev/null +++ b/tests/unit/llms/volcengine/test_volcengine_video_transformation.py @@ -0,0 +1,1131 @@ +import base64 +import io +import json +from pathlib import Path +from typing import Final +from unittest.mock import Mock + +import httpx +import pytest +import respx + +import litellm +from litellm import Router +from litellm.exceptions import UnsupportedParamsError +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.openai.cost_calculation import video_generation_cost +from litellm.llms.volcengine.videos.transformation import VolcEngineVideoConfig, VolcEngineVideoError +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.utils import decode_video_id_with_provider, encode_video_id_with_provider +from litellm.utils import ProviderConfigManager +from litellm.videos.main import avideo_content, avideo_generation + +PNG_BYTES: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) +PNG_DATA_URL: Final = f"data:image/png;base64,{base64.b64encode(PNG_BYTES).decode()}" + +API_BASE: Final = "https://ark.cn-beijing.volces.com" +TASKS_URL: Final = f"{API_BASE}/api/v3/contents/generations/tasks" +SEEDANCE_2_5: Final = "doubao-seedance-2-5-260628" +SEEDANCE_2_0: Final = "doubao-seedance-2-0-260128" +SEEDANCE_2_0_FAST: Final = "doubao-seedance-2-0-fast-260128" +TASK_ID: Final = "cgt-2026-abc" + + +def _response(payload: object, status_code: int = 200) -> httpx.Response: + return httpx.Response(status_code, json=payload, request=httpx.Request("GET", TASKS_URL)) + + +def _text_response(text: str, status_code: int = 200) -> httpx.Response: + return httpx.Response(status_code, text=text, request=httpx.Request("GET", TASKS_URL)) + + +def _create_body( + params: dict[str, object], model: str = SEEDANCE_2_5, drop_params: bool | None = None +) -> dict[str, object]: + body, files, url = VolcEngineVideoConfig().transform_video_create_request( + model=model, + prompt="a cat on a roof", + api_base=API_BASE, + video_create_optional_request_params=params, + litellm_params=GenericLiteLLMParams(drop_params=drop_params), + headers={}, + ) + assert (files, url) == ((), TASKS_URL) + return body + + +def _created(request_data: dict[str, object], model: str = SEEDANCE_2_5, logging_obj: object = None): + return VolcEngineVideoConfig().transform_video_create_response( + model=model, + raw_response=_response({"id": TASK_ID}), + logging_obj=logging_obj, # pyright: ignore[reportArgumentType] # the transform only reads litellm_params off it + custom_llm_provider="volcengine", + request_data=request_data, + ) + + +def _polled(payload: object, logging_obj: object = None): + return VolcEngineVideoConfig().transform_video_status_retrieve_response( + raw_response=_response(payload), + logging_obj=logging_obj, # pyright: ignore[reportArgumentType] # the transform only reads litellm_params off it + custom_llm_provider="volcengine", + ) + + +def _task(**fields: object) -> dict[str, object]: + return {"id": TASK_ID, "model": SEEDANCE_2_5, "status": "succeeded", **fields} + + +@pytest.mark.parametrize( + "input_reference", + [ + PNG_BYTES, + io.BytesIO(PNG_BYTES), + ("frame.png", PNG_BYTES, "image/png"), + ("frame.png", io.BytesIO(PNG_BYTES)), + ], + ids=["bytes", "file", "tuple", "tuple-with-file"], +) +def test_a_file_input_reference_becomes_a_data_uri_first_frame(input_reference: object): + body = _create_body({"input_reference": input_reference, "duration": 4}) + + assert body == { + "model": SEEDANCE_2_5, + "content": [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": PNG_DATA_URL}, "role": "first_frame"}, + ], + "resolution": "720p", + "duration": 4, + } + + +def test_a_path_input_reference_is_read_from_disk(tmp_path: Path): + frame = tmp_path / "frame.png" + frame.write_bytes(PNG_BYTES) + + body = _create_body({"input_reference": frame, "duration": 4}) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": PNG_DATA_URL}, "role": "first_frame"}, + ] + + +def test_a_url_input_reference_is_passed_as_is_and_never_opened_as_a_path(): + body = _create_body({"input_reference": "https://x/first.png", "duration": 4}) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": "https://x/first.png"}, "role": "first_frame"}, + ] + + +def test_openais_object_input_reference_sends_its_image_url(): + body = _create_body({"input_reference": {"image_url": "https://x/first.png"}, "duration": 4}) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": "https://x/first.png"}, "role": "first_frame"}, + ] + + +def test_a_file_id_input_reference_is_a_bad_request_unless_drop_params(): + with pytest.raises(UnsupportedParamsError, match="file_id is not supported"): + _ = _create_body({"input_reference": {"file_id": "file-abc"}, "duration": 4}) + + body = _create_body({"input_reference": {"file_id": "file-abc"}, "duration": 4}, drop_params=True) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}] + + +def test_a_caller_content_array_wins_over_input_reference_and_the_prompt_leads(): + body = _create_body( + { + "content": [{"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}], + "input_reference": "https://x/ignored.png", + "duration": 4, + } + ) + + assert body["content"] == [ + {"type": "text", "text": "a cat on a roof"}, + {"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}, + ] + + +IMAGE_ITEM: Final = {"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"} + + +def test_a_text_item_in_content_is_rejected_because_input_guardrails_only_scan_the_prompt(): + with pytest.raises(UnsupportedParamsError, match="put the text in prompt"): + _ = _create_body({"content": [{"type": "text", "text": "text the guardrail never saw"}], "duration": 4}) + + +def test_a_text_item_in_content_is_dropped_under_drop_params_so_ark_never_sees_unscanned_text(): + body = _create_body( + {"content": [{"type": "text", "text": "text the guardrail never saw"}, IMAGE_ITEM], "duration": 4}, + drop_params=True, + ) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}, IMAGE_ITEM] + + +@pytest.mark.parametrize( + "item", + [ + {"type": "video_url", "video_url": {"url": "https://x/v.mp4"}, "role": "reference_video"}, + {"type": "video_url", "video_url": {"url": "https://x/v.mp4"}}, + {"type": "image_url", "image_url": {"url": "https://x/v.png"}, "role": "reference_video"}, + ], + ids=["reference-video", "untagged-video", "mistagged-role"], +) +def test_a_video_input_is_rejected_because_its_cost_is_not_known_at_create(item: dict[str, object]): + with pytest.raises(UnsupportedParamsError, match="Video input"): + _ = _create_body({"content": [item], "duration": 4}) + + body = _create_body({"content": [item, IMAGE_ITEM], "duration": 4}, drop_params=True) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}, IMAGE_ITEM] + + +@pytest.mark.parametrize( + "content", + ["a string", [{"type": "image_url", "role": 7}], [{"type": 7}]], + ids=["not-an-array", "non-string-role", "non-string-type"], +) +def test_a_content_value_that_is_not_an_array_of_items_is_rejected_unless_drop_params(content: object): + with pytest.raises(UnsupportedParamsError, match="content must be an array"): + _ = _create_body({"content": content, "duration": 4}) + + body = _create_body({"content": content, "duration": 4}, drop_params=True) + + assert body["content"] == [{"type": "text", "text": "a cat on a roof"}] + + +def test_an_omitted_duration_and_tier_are_sent_as_openais_four_seconds_at_720p_so_ark_cannot_pick_what_is_billed(): + body = _create_body({}) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "720p", + "duration": 4, + } + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_an_endpoint_id_priced_through_base_model_gets_the_base_models_default_tier(): + body, _, _ = VolcEngineVideoConfig().transform_video_create_request( + model="ep-20261015-abc", + prompt="a cat on a roof", + api_base=API_BASE, + video_create_optional_request_params={"duration": 4}, + litellm_params=GenericLiteLLMParams(model_info={"base_model": f"volcengine/{SEEDANCE_2_5}"}), + headers={}, + ) + + assert (body["model"], body["resolution"]) == ("ep-20261015-abc", "720p") + + +@pytest.mark.parametrize("field", ["return_last_frame", "draft", "frame_count"]) +def test_a_field_litellm_does_not_send_to_ark_is_rejected(field: str): + with pytest.raises(UnsupportedParamsError, match=f"does not send {field} to Ark"): + _ = _create_body({"duration": 4, field: True}) + + +def test_a_field_litellm_does_not_send_to_ark_is_dropped_under_drop_params(): + body = _create_body({"duration": 4, "return_last_frame": True, "seed": 7}, drop_params=True) + + assert "return_last_frame" not in body + assert body["seed"] == 7 + + +@pytest.mark.parametrize("duration", [-1, "-1", 0, "abc", 4.5]) +def test_a_duration_ark_would_pick_or_that_is_not_whole_seconds_is_rejected(duration: object): + with pytest.raises(UnsupportedParamsError, match="duration must be a positive whole number"): + _ = _create_body({"duration": duration}) + + assert _create_body({"duration": duration}, drop_params=True)["duration"] == 4 + + +def test_frames_is_rejected_because_it_bypasses_the_billed_duration(): + with pytest.raises(UnsupportedParamsError, match="frames"): + _ = _create_body({"frames": 121, "duration": 6}) + + body = _create_body({"frames": 121, "duration": 6}, drop_params=True) + + assert ("frames" not in body, body["duration"]) == (True, 6) + + +def test_ark_fields_reach_the_body_and_false_flags_are_kept(): + body = _create_body( + { + "duration": 4, + "resolution": "480p", + "ratio": "9:16", + "seed": 7, + "watermark": False, + "generate_audio": False, + "service_tier": "flex", + } + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "480p", + "ratio": "9:16", + "seed": 7, + "watermark": False, + "generate_audio": False, + "service_tier": "flex", + "duration": 4, + } + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_resolution_the_model_has_no_price_for_is_rejected(): + with pytest.raises(UnsupportedParamsError, match="not offered at resolution '1080p', only at 480p, 720p"): + _ = _create_body({"duration": 4, "resolution": "1080p"}, model=SEEDANCE_2_0_FAST) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_an_unpriced_resolution_is_dropped_under_drop_params(): + body = _create_body( + {"duration": 4, "resolution": "1080p", "ratio": "16:9"}, model=SEEDANCE_2_0_FAST, drop_params=True + ) + + assert body == { + "model": SEEDANCE_2_0_FAST, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "720p", + "ratio": "16:9", + "duration": 4, + } + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_model_the_router_registered_without_prices_keeps_its_resolution(): + endpoint_id: Final = "ep-20261015-unpriced" + _ = Router( + model_list=[ + {"model_name": "ark", "litellm_params": {"model": f"volcengine/{endpoint_id}", "api_key": "sk-test"}} + ] + ) + + body = _create_body({"duration": 4, "resolution": "480p"}, model=endpoint_id) + body_without_a_tier = _create_body({"duration": 4}, model=endpoint_id) + + assert body["resolution"] == "480p" + assert "resolution" not in body_without_a_tier + + +def test_a_mistyped_ark_field_is_rejected(): + with pytest.raises(UnsupportedParamsError, match="invalid seed"): + _ = _create_body({"duration": 4, "seed": "abc", "watermark": False}) + + +def test_a_mistyped_ark_field_is_dropped_under_drop_params(): + body = _create_body({"duration": 4, "seed": "abc", "watermark": False}, drop_params=True) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "a cat on a roof"}], + "resolution": "720p", + "watermark": False, + "duration": 4, + } + + +@pytest.mark.parametrize( + "size,expected", + [ + ("1920x1080", {"ratio": "16:9", "resolution": "1080p"}), + ("854x480", {"ratio": "16:9", "resolution": "480p"}), + ("864x496", {"ratio": "16:9", "resolution": "480p"}), + ("720x1280", {"ratio": "9:16", "resolution": "720p"}), + ("1024x768", {"ratio": "4:3", "resolution": "720p"}), + ("640x640", {"ratio": "1:1", "resolution": "480p"}), + ("960x960", {"ratio": "1:1", "resolution": "720p"}), + ("1440x1440", {"ratio": "1:1", "resolution": "1080p"}), + ("1112x834", {"ratio": "4:3", "resolution": "720p"}), + ("1470x630", {"ratio": "21:9", "resolution": "720p"}), + ("3840x2160", {"ratio": "16:9", "resolution": "4k"}), + ], +) +def test_size_maps_onto_an_ark_ratio_and_tier(size: str, expected: dict[str, str]): + assert VolcEngineVideoConfig().map_openai_params({"size": size}, SEEDANCE_2_5, False) == expected + + +def test_explicit_ratio_and_resolution_beat_the_size_derived_ones(): + mapped = VolcEngineVideoConfig().map_openai_params( + {"size": "1280x720", "seconds": "4", "ratio": "1:1", "resolution": "480p", "parameters": {"seed": 3}}, + SEEDANCE_2_5, + False, + ) + + assert mapped == {"ratio": "1:1", "resolution": "480p", "duration": 4, "seed": 3} + + +@pytest.mark.parametrize( + "params,message", + [ + ({"seconds": "4.5"}, "seconds must be a whole number"), + ({"size": "big"}, "size must be WIDTHxHEIGHT"), + ({"size": "16:9"}, "size must be WIDTHxHEIGHT"), + ({"parameters": "seed=3"}, "parameters must be an object"), + ({"characters": [{"id": "c"}]}, "'characters' is not supported"), + ({"image": "https://x/a.png"}, "'image' is not supported"), + ], +) +def test_a_value_ark_cannot_take_raises_unless_drop_params(params: dict[str, object], message: str): + with pytest.raises(UnsupportedParamsError, match=message): + _ = VolcEngineVideoConfig().map_openai_params(params, SEEDANCE_2_5, False) + + assert VolcEngineVideoConfig().map_openai_params(params, SEEDANCE_2_5, True) == {} + + +def test_request_metadata_is_not_mapped_into_the_body_and_user_becomes_the_safety_identifier(): + mapped = VolcEngineVideoConfig().map_openai_params( + {"user": "u", "extra_headers": {"x": "y"}, "seconds": 4}, SEEDANCE_2_5, False + ) + + assert mapped == {"safety_identifier": "u", "duration": 4} + + +def test_an_explicit_safety_identifier_wins_over_user(): + mapped = VolcEngineVideoConfig().map_openai_params( + {"user": "u", "safety_identifier": "s", "seconds": 4}, SEEDANCE_2_5, False + ) + + assert mapped["safety_identifier"] == "s" + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + "model,request_data,seconds,size,usage", + [ + ( + SEEDANCE_2_5, + {"duration": 4, "resolution": "480p", "ratio": "16:9"}, + "4", + "854x480", + {"duration_seconds": 4.0, "video_resolution": "480p"}, + ), + ( + SEEDANCE_2_0, + {"duration": 5, "resolution": "480p", "ratio": "16:9"}, + "5", + "864x496", + {"duration_seconds": 5.0, "video_resolution": "480p"}, + ), + (SEEDANCE_2_5, {"duration": 5}, "5", None, {"duration_seconds": 5.0}), + ], + ids=["2.5-480p", "2.0-480p-renders-its-own-frame", "no-geometry"], +) +def test_the_create_response_reports_what_the_task_bills( + model: str, request_data: dict[str, object], seconds: str, size: str | None, usage: dict[str, object] +): + video = _created(request_data, model=model) + + assert video.model_dump(include={"status", "seconds", "size", "usage", "model"}) == { + "status": "queued", + "seconds": seconds, + "size": size, + "usage": usage, + "model": model, + } + + +@pytest.mark.parametrize( + "usage,expected_cost", + [ + ({"duration_seconds": 4.0, "video_resolution": "480p"}, 4 * 0.1), + ({"duration_seconds": 5.0, "video_resolution": "720p"}, 5 * 0.2), + ({"duration_seconds": 5.0}, 5 * 0.2), + ], +) +def test_the_create_usage_bills_its_duration_at_its_tier_rate(usage: dict[str, object], expected_cost: float): + model_info = { + "output_cost_per_second": 0.2, + "output_cost_per_second_480p": 0.1, + "output_cost_per_second_720p": 0.2, + } + video = _created({"duration": int(usage["duration_seconds"]), "resolution": usage.get("video_resolution")}) + + cost = video_generation_cost( + model=SEEDANCE_2_5, + duration_seconds=video.usage["duration_seconds"], + model_info=model_info, # pyright: ignore[reportArgumentType] # a hand-written rate card stands in for ModelInfo + video_resolution=video.usage.get("video_resolution"), + ) + + assert cost == pytest.approx(expected_cost) + + +def test_the_created_id_carries_the_requested_and_served_model_groups(): + logging_obj = Mock( + litellm_params={"litellm_metadata": {"model_group": "seedance", "deployment_model_name": "seedance-account-b"}} + ) + + video = _created({"duration": 5}, logging_obj=logging_obj) + + assert decode_video_id_with_provider(video.id) == { + "custom_llm_provider": "volcengine", + "model_id": "seedance", + "video_id": TASK_ID, + "served_model": "seedance-account-b", + } + + +def test_a_create_without_router_metadata_carries_the_model(): + video = _created({"duration": 5}, logging_obj=Mock(litellm_params={})) + + assert decode_video_id_with_provider(video.id) == { + "custom_llm_provider": "volcengine", + "model_id": SEEDANCE_2_5, + "video_id": TASK_ID, + } + + +@pytest.mark.parametrize( + "raw_response,status_code,message", + [ + (_response({"error": {"code": "InvalidParameter", "message": "bad ratio"}}), 400, "bad ratio"), + (_response({"error": {"code": "Forbidden", "message": "no access"}}, 403), 403, "no access"), + (_text_response("bad gateway"), 502, "Unexpected Volcengine video response"), + ], + ids=["error-on-200", "error-status-kept", "not-json"], +) +def test_a_create_response_without_a_task_id_raises_instead_of_queueing_an_empty_video( + raw_response: httpx.Response, status_code: int, message: str +): + with pytest.raises(VolcEngineVideoError, match=message) as raised: + _ = VolcEngineVideoConfig().transform_video_create_response( + model=SEEDANCE_2_5, + raw_response=raw_response, + logging_obj=None, # pyright: ignore[reportArgumentType] # the transform only reads litellm_params off it + custom_llm_provider="volcengine", + request_data={"duration": 5}, + ) + + assert raised.value.status_code == status_code + + +@pytest.mark.parametrize( + "ark_status,expected", + [ + ("queued", "queued"), + ("running", "in_progress"), + ("succeeded", "completed"), + ("failed", "failed"), + ("cancelled", "cancelled"), + ("expired", "failed"), + ], +) +def test_the_ark_status_maps_onto_the_openai_vocabulary(ark_status: str, expected: str): + assert _polled(_task(status=ark_status)).status == expected + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_a_completed_task_reports_its_delivered_geometry_duration_and_tokens(): + video = _polled( + _task( + created_at=1791989938, + updated_at=1791990031, + resolution="480p", + ratio="16:9", + duration=4, + usage={"completion_tokens": 40594, "total_tokens": 40594}, + model=SEEDANCE_2_0, + content={"video_url": "https://tos/v.mp4"}, + ) + ) + + assert video.model_dump(exclude={"id"}) | {"id": decode_video_id_with_provider(video.id)["video_id"]} == { + "id": TASK_ID, + "object": "video", + "status": "completed", + "created_at": 1791989938, + "completed_at": 1791990031, + "expires_at": None, + "error": None, + "progress": None, + "remixed_from_video_id": None, + "seconds": "4", + "size": "864x496", + "model": SEEDANCE_2_0, + "usage": {"duration_seconds": 4.0, "video_resolution": "480p", "total_tokens": 40594}, + } + + +@pytest.mark.parametrize( + "resolution,ratio,expected", + [ + ("720p", "9:16", "720x1280"), + ("1080p", "16:9", "1920x1080"), + ("480p", "adaptive", None), + ], +) +def test_a_frame_the_cost_map_does_not_list_is_scaled_from_the_tier(resolution: str, ratio: str, expected: str | None): + assert _polled(_task(resolution=resolution, ratio=ratio, model="doubao-seedance-unlisted")).size == expected + + +def test_a_failed_task_carries_its_error_block(): + video = _polled(_task(status="failed", error={"code": "OutputVideoSensitiveContentDetected", "message": "flagged"})) + + assert (video.status, video.error, video.completed_at) == ( + "failed", + {"code": "OutputVideoSensitiveContentDetected", "message": "flagged"}, + None, + ) + + +@pytest.mark.parametrize( + "payload,message", + [ + (_task(status="paused"), "Unknown Volcengine task status: paused"), + ({"status": "running"}, "Unexpected Volcengine video response"), + ({"id": TASK_ID}, "Unexpected Volcengine video response"), + ], + ids=["unmapped-status", "no-id", "no-status"], +) +def test_a_poll_litellm_cannot_read_raises_instead_of_returning_a_job_to_poll_forever(payload: object, message: str): + with pytest.raises(VolcEngineVideoError, match=message) as raised: + _ = _polled(payload) + + assert raised.value.status_code == 502 + + +def test_a_poll_keeps_the_ids_requested_and_served_model_when_the_router_served_it(): + video_id = encode_video_id_with_provider(TASK_ID, "volcengine", "seedance", served_model="seedance-account-b") + logging_obj = Mock( + litellm_params={ + "video_id": video_id, + "litellm_metadata": {"model_group": "seedance", "deployment_model_name": "seedance-account-b"}, + } + ) + + video = _polled(_task(status="running"), logging_obj=logging_obj) + + assert video.id == video_id + assert video.model == "seedance" + + +def test_a_poll_drops_a_served_model_the_router_did_not_route_to(): + edited = encode_video_id_with_provider( + TASK_ID, "volcengine", "seedance-account-a", served_model="seedance-account-b" + ) + logging_obj = Mock(litellm_params={"video_id": edited, "litellm_metadata": {"model_group": "seedance-account-a"}}) + + video = _polled(_task(status="running"), logging_obj=logging_obj) + + assert video.id == encode_video_id_with_provider(TASK_ID, "volcengine", "seedance-account-a") + + +async def _download(task_payload: dict[str, object], video_bytes: bytes = b"mp4") -> bytes: + def respond(request: httpx.Request) -> httpx.Response: + if request.url.host == "tos.example": + return httpx.Response(200, content=video_bytes) + return httpx.Response(200, json=task_payload) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + return await avideo_content( + video_id=TASK_ID, + custom_llm_provider="volcengine", + model=f"volcengine/{SEEDANCE_2_5}", + api_key="sk-test", + api_base=API_BASE, + client=handler, + ) + + +@pytest.mark.asyncio +async def test_the_download_fetches_the_signed_url_through_the_callers_client(): + content = await _download(_task(content={"video_url": "https://tos.example/v.mp4"}), video_bytes=b"the-video") + + assert content == b"the-video" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "payload,message", + [ + (_task(status="queued"), "still processing"), + (_task(status="running"), "still processing"), + (_task(status="failed", error={"message": "flagged"}), "flagged"), + (_task(status="failed"), "ended as failed"), + (_task(status="cancelled"), "ended as cancelled"), + (_task(status="expired"), "Task expired"), + ], +) +async def test_a_download_with_no_video_yet_is_a_bad_request_so_it_is_neither_retried_nor_cools_the_deployment( + payload: dict[str, object], message: str +): + with pytest.raises(litellm.BadRequestError, match=message) as raised: + _ = await _download(payload) + + assert raised.value.status_code == 400 + + +def test_the_list_maps_ark_paging_and_filters(): + url, params = VolcEngineVideoConfig().transform_video_list_request( + api_base=API_BASE, + litellm_params=GenericLiteLLMParams(), + headers={}, + limit=20, + extra_query={"filter.status": "succeeded"}, + ) + + assert (url, params) == (TASKS_URL, {"page_size": "20", "filter.status": "succeeded"}) + + +@pytest.mark.parametrize("paging", [{"after": "video_abc"}, {"order": "asc"}]) +def test_a_list_cursor_or_order_is_rejected_unless_drop_params(paging: dict[str, str]): + config = VolcEngineVideoConfig() + + with pytest.raises(UnsupportedParamsError, match="pages by page_num"): + _ = config.transform_video_list_request( + api_base=API_BASE, litellm_params=GenericLiteLLMParams(), headers={}, **paging + ) + _, params = config.transform_video_list_request( + api_base=API_BASE, litellm_params=GenericLiteLLMParams(drop_params=True), headers={}, limit=5, **paging + ) + + assert params == {"page_size": "5"} + + +def test_the_list_response_is_an_openai_list_body(): + response = VolcEngineVideoConfig().transform_video_list_response( + raw_response=_response({"total": 1, "items": [_task(status="running")]}), + logging_obj=Mock(), # pyright: ignore[reportArgumentType] # unused by the transform + custom_llm_provider="volcengine", + ) + + encoded = encode_video_id_with_provider(TASK_ID, "volcengine", SEEDANCE_2_5) + assert {key: value for key, value in response.items() if key != "data"} == { + "object": "list", + "total": 1, + "has_more": False, + "first_id": encoded, + "last_id": encoded, + } + assert [(item["id"], item["status"]) for item in response["data"]] == [(encoded, "in_progress")] + + +def test_an_empty_list_still_carries_the_object_type(): + response = VolcEngineVideoConfig().transform_video_list_response( + raw_response=_response({"total": 0, "items": []}), + logging_obj=Mock(), # pyright: ignore[reportArgumentType] # unused by the transform + ) + + assert response == {"object": "list", "data": [], "total": 0, "has_more": False} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call", ["avideo_edit", "avideo_extension", "avideo_remix"]) +async def test_edit_extension_and_remix_are_a_bad_request_because_a_video_input_cannot_be_billed_at_create(call: str): + with pytest.raises(litellm.BadRequestError, match="Ark bills a task with a video input") as raised: + _ = await getattr(litellm, call)( + video_id="https://x/source.mp4", + prompt="make it brighter", + seconds="5", + model=f"volcengine/{SEEDANCE_2_5}", + custom_llm_provider="volcengine", + api_key="sk-test", + api_base=API_BASE, + ) + + assert raised.value.status_code == 400 + + +@pytest.mark.parametrize( + "api_base", + [ + None, + API_BASE, + f"{API_BASE}/", + f"{API_BASE}/api/v3", + f"{API_BASE}/api/compatible", + f"{API_BASE}/api/v3/responses", + f"{API_BASE}/v1", + ], +) +def test_any_ark_surface_on_the_shared_credential_normalizes_to_the_host_root( + api_base: str | None, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.delenv("VOLCENGINE_API_BASE", raising=False) + monkeypatch.delenv("ARK_API_BASE", raising=False) + + assert VolcEngineVideoConfig().get_complete_url("m", api_base, {}) == API_BASE + + +@pytest.mark.parametrize("env_var", ["VOLCENGINE_API_BASE", "ARK_API_BASE"]) +def test_an_api_base_env_var_routes_a_deployment_without_api_base(env_var: str, monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("VOLCENGINE_API_BASE", raising=False) + monkeypatch.delenv("ARK_API_BASE", raising=False) + monkeypatch.setenv(env_var, "https://ark.ap-southeast.bytepluses.com/api/v3") + + assert VolcEngineVideoConfig().get_complete_url("m", None, {}) == "https://ark.ap-southeast.bytepluses.com" + assert VolcEngineVideoConfig().get_complete_url("m", f"{API_BASE}/api/v3", {}) == API_BASE + + +@pytest.mark.parametrize( + "api_base,expected", + [ + ("https://gw.internal/ark/api/v3", "https://gw.internal/ark"), + ("https://gw.internal/api/ark/api/v3", "https://gw.internal/api/ark"), + ], +) +def test_a_path_prefix_before_the_api_marker_survives(api_base: str, expected: str): + assert VolcEngineVideoConfig().get_complete_url("m", api_base, {}) == expected + + +def test_a_missing_api_key_raises(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("ARK_API_KEY", raising=False) + monkeypatch.delenv("VOLCENGINE_API_KEY", raising=False) + + with pytest.raises(ValueError, match="Volcengine API key is required"): + _ = VolcEngineVideoConfig().validate_environment({}, SEEDANCE_2_5) + + +def test_volcengine_resolves_to_the_video_config(): + config = ProviderConfigManager.get_provider_video_config( + model=SEEDANCE_2_5, provider=litellm.LlmProviders.VOLCENGINE + ) + + assert type(config) is VolcEngineVideoConfig + + +ARK_VIDEO_TIERS: Final = { + SEEDANCE_2_5: {"480p": ((854, 480), 70), "720p": ((1280, 720), 70), "1080p": ((1920, 1080), 77)}, + SEEDANCE_2_0: {"480p": ((864, 496), 46), "720p": ((1280, 720), 46), "1080p": ((1920, 1080), 51), "4k": ((3840, 2160), 26)}, + SEEDANCE_2_0_FAST: {"480p": ((864, 496), 37), "720p": ((1280, 720), 37)}, + "doubao-seedance-2-0-mini-260615": {"480p": ((864, 496), 23), "720p": ((1280, 720), 23)}, +} +CNY_PER_USD: Final = 7.0 + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("model", sorted(ARK_VIDEO_TIERS)) +def test_each_tier_rate_is_its_rendered_pixels_at_arks_price_for_that_tier(model: str): + """Ark prices 1080p and 4k per token above and below the 480p/720p rate, so a tier is not the base + rate scaled by pixels. Prices and frames are Ark's published per-resolution table, read 2026-10-16 + from https://www.volcengine.com/docs/82379/1544106, whose own 2.0 examples agree: 1080p 16:9 5s bills + 2.48 CNY/s and 4k bills 5.05 CNY/s.""" + info = litellm.get_model_info(model=model, custom_llm_provider="volcengine") + frames = info["provider_specific_entry"]["volcengine_video_frames"] + + assert {tier: info[f"output_cost_per_second_{tier}"] for tier in ARK_VIDEO_TIERS[model]} == { + tier: pytest.approx(width * height * 24 / 1024 * cny_per_million / 1e6 / CNY_PER_USD, rel=1e-5) + for tier, ((width, height), cny_per_million) in ARK_VIDEO_TIERS[model].items() + } + + assert { + tier: tuple(int(side) for side in frames[tier]["16:9"].split("x")) + for tier in ARK_VIDEO_TIERS[model] + if tier in frames + } == {tier: frame for tier, (frame, _) in ARK_VIDEO_TIERS[model].items() if tier in frames} + + +async def _wire_body(**kwargs: object) -> dict[str, object]: + sent: Final[list[httpx.Request]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + sent.append(request) + return httpx.Response(200, json={"id": TASK_ID}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + handler: Final = AsyncHTTPHandler() + await handler.close() + handler.client = http_client + _ = await avideo_generation(api_key="sk-test", api_base=API_BASE, client=handler, **kwargs) + + (request,) = sent + assert (str(request.url), request.headers["authorization"]) == (TASKS_URL, "Bearer sk-test") + return json.loads(request.content) + + +@pytest.mark.asyncio +async def test_seconds_size_and_extra_body_content_reach_the_wire(): + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="use these", + seconds="4", + size="854x480", + extra_body={ + "content": [{"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}] + }, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [ + {"type": "text", "text": "use these"}, + {"type": "image_url", "image_url": {"url": "https://x/a.png"}, "role": "reference_image"}, + ], + "ratio": "16:9", + "resolution": "480p", + "duration": 4, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fields", + [{"parameters": {"seed": 3}}, {"extra_body": {"parameters": {"seed": 3}}}], + ids=["top_level", "extra_body"], +) +async def test_parameters_reach_the_wire_unpacked_whether_sent_top_level_or_in_extra_body(fields: dict[str, object]): + body = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", seconds="4", **fields) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + "seed": 3, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra_body,expected", + [ + ({"size": "1920x1080"}, {"resolution": "1080p"}), + ({"seconds": "8"}, {"duration": 8}), + ({"user": "someone"}, {"safety_identifier": "someone"}), + ], + ids=["size", "seconds", "user"], +) +async def test_an_extra_body_field_litellm_maps_itself_is_honoured_and_its_leftover_copy_ignored( + extra_body: dict[str, object], expected: dict[str, object] +): + """The shared request builder copies extra_body onto the top-level params, where this adapter maps it, and then + merges the raw extra_body back over the mapped params, returning the same key a second time as a field of its + own. The second copy has to be ignored rather than rejected as a field litellm never sends.""" + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="p", + seconds="4", + size="1280x720", + extra_body=extra_body, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "ratio": "16:9", + "resolution": "720p", + "duration": 4, + **expected, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "fields,misplaced", + [ + ({"parameters": {"seconds": "8"}}, "seconds"), + ({"extra_body": {"parameters": {"seconds": "8"}}}, "seconds"), + ({"extra_body": {"parameters": {"size": "1920x1080"}}}, "size"), + ({"extra_body": {"parameters": {"user": "someone"}}}, "user"), + ({"extra_body": {"parameters": {"prompt": "a dog"}}}, "prompt"), + ({"extra_body": {"parameters": {"parameters": {"seed": 3}}}}, "parameters"), + ], + ids=["top_level_seconds", "seconds", "size", "user", "prompt", "nested_parameters"], +) +async def test_a_request_field_litellm_maps_itself_is_rejected_inside_parameters( + fields: dict[str, object], misplaced: str +): + with pytest.raises(UnsupportedParamsError, match=f"send {misplaced} as top-level request fields"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", **fields) + + +@pytest.mark.asyncio +async def test_a_request_field_inside_parameters_is_dropped_under_drop_params_and_the_rest_is_sent(): + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="p", + drop_params=True, + extra_body={"parameters": {"seconds": "8", "seed": 3}}, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + "seed": 3, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra_body,field", + [ + ({"prompt": "a dog"}, "prompt"), + ({"model": SEEDANCE_2_0_FAST}, "model"), + ({"extra_headers": {"x-a": "1"}}, "extra_headers"), + ], + ids=["prompt", "model", "extra_headers"], +) +async def test_an_extra_body_copy_of_a_request_field_litellm_does_not_map_is_rejected( + extra_body: dict[str, object], field: str +): + with pytest.raises(UnsupportedParamsError, match=f"does not send {field} to Ark"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", extra_body=extra_body) + + +@pytest.mark.asyncio +async def test_a_user_that_is_not_a_string_is_rejected_rather_than_ignored(): + with pytest.raises(UnsupportedParamsError, match="user must be a string, got int"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", extra_body={"user": 5}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "control", + [{"drop_params": False}, {"organization": "org-1"}], + ids=["drop_params", "organization"], +) +async def test_a_litellm_control_field_is_not_mistaken_for_an_ark_field(control: dict[str, object]): + body = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", seconds="4", **control) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + } + + +@pytest.mark.asyncio +async def test_additional_drop_params_removes_the_named_ark_fields_from_the_wire(): + body = await _wire_body( + model=f"volcengine/{SEEDANCE_2_5}", + prompt="p", + seconds="4", + additional_drop_params=["seed"], + extra_body={"seed": 3, "watermark": False}, + ) + + assert body == { + "model": SEEDANCE_2_5, + "content": [{"type": "text", "text": "p"}], + "resolution": "720p", + "duration": 4, + "watermark": False, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("dropped", "request_fields"), + [ + pytest.param("user", {"user": "u-1"}, id="user"), + pytest.param("seconds", {"seconds": "8"}, id="seconds"), + pytest.param("size", {"size": "1920x1080"}, id="size"), + pytest.param("parameters", {"parameters": {"seed": 3}}, id="parameters"), + pytest.param("parameters", {"extra_body": {"parameters": {"seed": 3}}}, id="extra_body_parameters"), + pytest.param("image", {"image": {"gcsUri": "gs://b/a.png"}}, id="unsupported_image"), + ], +) +async def test_additional_drop_params_naming_a_request_field_sends_what_omitting_it_sends( + dropped: str, request_fields: dict[str, object] +): + request: Final = {"model": f"volcengine/{SEEDANCE_2_5}", "prompt": "p"} + + with_drop = await _wire_body(**request, **request_fields, additional_drop_params=[dropped]) + + assert with_drop == await _wire_body(**request) + + +@pytest.mark.asyncio +async def test_an_extra_body_duration_of_minus_one_is_rejected_before_anything_is_sent(): + with pytest.raises(UnsupportedParamsError, match="Ark-picked length"): + _ = await _wire_body(model=f"volcengine/{SEEDANCE_2_5}", prompt="p", extra_body={"duration": -1}) + + +ACCOUNT_A_HOST: Final = "https://ark-a.example" +ACCOUNT_B_HOST: Final = "https://ark-b.example" + + +def _two_account_router(**router_kwargs: object) -> Router: + return Router( + model_list=[ + { + "model_name": "seedance-a", + "litellm_params": { + "model": f"volcengine/{SEEDANCE_2_5}", + "api_key": "key-a", + "api_base": ACCOUNT_A_HOST, + }, + }, + { + "model_name": "seedance-b", + "litellm_params": { + "model": f"volcengine/{SEEDANCE_2_5}", + "api_key": "key-b", + "api_base": ACCOUNT_B_HOST, + }, + }, + ], + **router_kwargs, # pyright: ignore[reportArgumentType] # forwarded Router options + ) + + +def _proxy_poll_model(router: Router, video_id: str) -> str | None: + decoded: Final = decode_video_id_with_provider(video_id) + requested: Final = router.resolve_model_name_from_model_id( + decoded.get("model_id"), decoded.get("custom_llm_provider") + ) + served: Final = decoded.get("served_model") + return router.resolve_served_model_name(requested, served) if requested and served else requested + + +@pytest.mark.parametrize("caller_metadata", [None, {"tags": ["proxy-request"]}]) +@respx.mock +def test_a_video_is_polled_on_the_ark_account_that_created_it(caller_metadata: dict[str, object] | None): + router = _two_account_router() + _ = respx.post(f"{ACCOUNT_B_HOST}/api/v3/contents/generations/tasks").mock( + return_value=httpx.Response(200, json={"id": TASK_ID}) + ) + task_get = respx.get(f"{ACCOUNT_B_HOST}/api/v3/contents/generations/tasks/{TASK_ID}").mock( + return_value=httpx.Response(200, json=_task(status="running")) + ) + + created = router.video_generation(model="seedance-b", prompt="a cat", seconds="4", metadata=caller_metadata) + _ = router.video_status(model=_proxy_poll_model(router, created.id), video_id=created.id) + + assert [call.request.headers["authorization"] for call in task_get.calls] == ["Bearer key-b"] + + +@respx.mock +def test_a_video_created_through_a_routing_group_is_polled_on_the_member_that_created_it(): + router = _two_account_router( + routing_groups=[ + {"group_name": "seedance", "models": ["seedance-a", "seedance-b"], "routing_strategy": "simple-shuffle"} + ] + ) + for host in (ACCOUNT_A_HOST, ACCOUNT_B_HOST): + _ = respx.post(f"{host}/api/v3/contents/generations/tasks").mock( + return_value=httpx.Response(200, json={"id": TASK_ID}) + ) + _ = respx.get(f"{host}/api/v3/contents/generations/tasks/{TASK_ID}").mock( + return_value=httpx.Response(200, json=_task(status="running")) + ) + + for _ in range(8): + created = router.video_generation(model="seedance", prompt="a cat", seconds="4") + polled = router.video_status(model=_proxy_poll_model(router, created.id), video_id=created.id) + assert polled.id == created.id + + hosts = [call.request.url.host for call in respx.calls] + assert hosts[1::2] == hosts[0::2]