From e7c89b0dd46cd052100233a1efea11640caebc55 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Mon, 16 Nov 2026 14:43:03 -0800 Subject: [PATCH] fix(proxy): preserve terminal evidence in disconnect billing (#52088) * fix(proxy): preserve terminal evidence in disconnect billing * fix(proxy): retain held finish reasons through stream wrappers * refactor(proxy): use immutable finish evidence results * fix(proxy): type disconnect evidence boundaries and cover wrappers * fix(proxy): preserve object type before chunk narrowing * fix(proxy): retain disconnect finish evidence per choice * fix(proxy): preserve genuine Gemini finish evidence * fix(proxy): use typed finish evidence storage * test(proxy): exercise real router stream attempts * fix(proxy): keep unspecified Gemini evidence unknown --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/litellm_core_utils/core_helpers.py | 18 + .../streaming_chunk_builder_utils.py | 12 + .../litellm_core_utils/streaming_handler.py | 52 +- .../messages/adapters/streaming_iterator.py | 19 +- .../messages/native/response_cache.py | 5 + .../vertex_and_google_ai_studio_gemini.py | 20 + litellm/proxy/common_request_processing.py | 46 ++ litellm/router_rust/invoke/streaming.py | 13 +- .../proxy/test_common_request_processing.py | 617 ++++++++++++++++++ 10 files changed, 800 insertions(+), 3 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 4d239472542..c1c5305fa79 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -708,6 +708,7 @@ STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8") STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",) STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",) +PROVIDER_FINISH_REASONS_HIDDEN_PARAM: Final = "provider_finish_reasons" VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS ### SPEND TRACKING ### DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 656fadc483d..ec0e92afa0e 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -6,6 +6,7 @@ import math import re import sys from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence +from dataclasses import dataclass from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable @@ -191,6 +192,23 @@ def attribute_or_none(source: object, name: str) -> object: _Shape: Final = TypeVar("_Shape") +@dataclass(frozen=True, slots=True) +class FinishEvidence: + """Finish reasons a stream received from the provider. ``unindexed`` is a received reason whose + choice the stream could not tell, so it is only evidence for a single-choice response.""" + + by_choice: Mapping[int, str] + unindexed: str | None + + +NO_FINISH_EVIDENCE: Final = FinishEvidence(by_choice=MappingProxyType({}), unindexed=None) + + +def finish_evidence_of(stream: object) -> FinishEvidence: + evidence: Final[object] = getattr(stream, "finish_evidence", None) + return evidence if isinstance(evidence, FinishEvidence) else NO_FINISH_EVIDENCE + + def has_attributes( value: object, shape: type[_Shape], diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 79c1c82356d..0e4a3240543 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1124,6 +1124,18 @@ class ChunkProcessor: return chunk.get("choices", ()) return getattr(chunk, "choices", ()) + @staticmethod + def choice_indices_with_observed_finish_reason(chunks: Iterable[object]) -> frozenset[int]: + choices: Final = tuple(chain.from_iterable(object_items(_field_of(chunk, "choices")) for chunk in chunks)) + finished: Final = frozenset( + _stream_choice_index(choice) + for choice in choices + if ChunkProcessor._finish_reason_of_choice(choice) is not None + ) + if len(frozenset(map(_stream_choice_index, choices))) > 1: + return finished + return frozenset({0}) if finished else frozenset() + @staticmethod def _saw_finish_reason(chunks: Sequence["_UsageBearingChunk | ModelResponse"]) -> bool: return any( diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 47c1489ff7c..f10683fd14b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -23,10 +23,15 @@ from typing_extensions import NotRequired, TypedDict import litellm from litellm import verbose_logger from litellm._uuid import uuid -from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS +from litellm.constants import ( + PROVIDER_FINISH_REASONS_HIDDEN_PARAM, + STREAM_CHUNK_CHOICE_FIELDS, + STREAM_FINISH_CHOICE_FIELDS, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import ( UPSTREAM_USAGE_COST, + FinishEvidence, as_str_mapping, attribute_or_none, chunk_usage, @@ -171,6 +176,35 @@ def _prepend_upstream_choice_extras( _NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) _NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({}) +_NO_FINISH_REASONS_BY_INDEX: Final[Mapping[int, str]] = MappingProxyType({}) + + +def _indexed_finish_reason(choice: object) -> tuple[int, str] | None: + reason: Final[object] = getattr(choice, "finish_reason", None) + index: Final[object] = getattr(choice, "index", None) + if not isinstance(reason, str) or not reason or not isinstance(index, int): + return None + return index, reason + + +_PROVIDER_FINISH_REASONS: Final = TypeAdapter(tuple[tuple[int, str], ...]) + + +def _provider_finish_reasons(evidence: object) -> Mapping[int, str]: + if evidence is None: + return _NO_FINISH_REASONS_BY_INDEX + try: + return MappingProxyType(dict(_PROVIDER_FINISH_REASONS.validate_python(evidence, strict=True))) + except ValidationError: + return _NO_FINISH_REASONS_BY_INDEX + + +def _finish_reasons_by_index(upstream_chunk: object) -> Mapping[int, str]: + hidden_params: Final = get_hidden_params(upstream_chunk) + if hidden_params is not None and hidden_params.get("is_finished") is False: + return _provider_finish_reasons(hidden_params.get(PROVIDER_FINISH_REASONS_HIDDEN_PARAM)) + choices: Final[object] = getattr(upstream_chunk, "choices", None) + return MappingProxyType(dict(filter(None, map(_indexed_finish_reason, object_items(choices))))) def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None: @@ -509,6 +543,7 @@ class CustomStreamWrapper: self._upstream_provider: str | None = None self.received_finish_reason: str | None = None self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX + self.received_finish_reasons: Mapping[int, str] = _NO_FINISH_REASONS_BY_INDEX self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason @@ -1729,6 +1764,7 @@ class CustomStreamWrapper: response_obj = self.handle_openai_text_completion_chunk(chunk) completion_obj["content"] = response_obj["text"] print_verbose(f"completion obj content: {completion_obj['content']}") + self._record_finish_reasons(chunk) if response_obj["is_finished"]: self._record_finish_choice_fields(chunk) if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None: @@ -1805,6 +1841,7 @@ class CustomStreamWrapper: self.response_id = cached_chunk.id if hasattr(cached_chunk, "system_fingerprint"): self.system_fingerprint = cached_chunk.system_fingerprint + self._record_finish_reasons(cached_chunk) if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] self._record_finish_choice_fields(cached_chunk) @@ -1840,6 +1877,7 @@ class CustomStreamWrapper: ) self.received_finish_reason = response_obj["finish_reason"] self._record_finish_choice_fields(chunk) + self._record_finish_reasons(chunk) if response_obj.get("original_chunk", None) is not None: if hasattr(response_obj["original_chunk"], "id"): model_response = self.set_model_id(response_obj["original_chunk"].id, model_response) @@ -2190,6 +2228,18 @@ class CustomStreamWrapper: if self.logging_obj.is_sync_litellm_request(litellm_params): self.logging_obj.success_handler(processed_chunk, None, None, cache_hit) + @property + def finish_evidence(self) -> FinishEvidence: + return FinishEvidence( + by_choice=self.received_finish_reasons, + unindexed=None if self.received_finish_reasons else self.received_finish_reason, + ) + + def _record_finish_reasons(self, upstream_chunk: object) -> None: + received: Final = _finish_reasons_by_index(upstream_chunk) + if received: + self.received_finish_reasons = MappingProxyType({**self.received_finish_reasons, **received}) + def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str: if finish_reason == "stop" and self.tool_call: return "tool_calls" diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index cf96c1c5fda..343ce85dcd1 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -27,7 +27,12 @@ import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items +from litellm.litellm_core_utils.core_helpers import ( + FinishEvidence, + as_str_mapping, + attribute_items, + finish_evidence_of, +) from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.messages.utils import anthropic_sse_frame @@ -283,6 +288,10 @@ class _CombinedChunkSplitter: "list[ModelResponseStream] | None", getattr(self._stream, "chunks", None) ) + @property + def finish_evidence(self) -> FinishEvidence: + return finish_evidence_of(self._stream) + @property def messages(self) -> "list[AllMessageValues] | None": return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream @@ -711,6 +720,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): "list[ModelResponseStream] | None", getattr(self.completion_stream, "chunks", None) ) + @property + def finish_evidence(self) -> FinishEvidence: + return self._combined_chunk_splitter.finish_evidence + @property def messages(self) -> "list[AllMessageValues] | None": return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream @@ -1645,6 +1658,10 @@ class AnthropicSSEStream(AsyncIterator[bytes]): def chunks(self) -> "list[ModelResponseStream] | None": return self._anthropic_wrapper.chunks + @property + def finish_evidence(self) -> FinishEvidence: + return self._anthropic_wrapper.finish_evidence + @property def messages(self) -> "list[AllMessageValues] | None": return self._anthropic_wrapper.messages diff --git a/litellm/llms/anthropic/messages/native/response_cache.py b/litellm/llms/anthropic/messages/native/response_cache.py index 11c0b501966..ef2266ba8d9 100644 --- a/litellm/llms/anthropic/messages/native/response_cache.py +++ b/litellm/llms/anthropic/messages/native/response_cache.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final, cast import litellm from litellm._logging import verbose_logger from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output +from litellm.litellm_core_utils.core_helpers import FinishEvidence, finish_evidence_of from litellm.llms.anthropic.messages.native.streaming_iterator import ( AnthropicMessagesStreamingResponse, BaseAnthropicMessagesStreamingIterator, @@ -59,6 +60,10 @@ class AnthropicMessagesStreamCacheWriter: "list[ModelResponseStream] | None", getattr(self.stream, "chunks", None) ) + @property + def finish_evidence(self) -> FinishEvidence: + return finish_evidence_of(self.stream) + @property def messages(self) -> "list[AllMessageValues] | None": return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 8fa75c3b638..08615944541 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -29,10 +29,12 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH, DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE, DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO, + PROVIDER_FINISH_REASONS_HIDDEN_PARAM, ) from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.hidden_params import set_hidden_param from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature @@ -4213,6 +4215,21 @@ def _transform_long_decode_hops( return merged if marked is merged_usage else merged.model_copy(update={"usage": marked}) +def _provider_finish_reasons( + candidates: Sequence[Candidates], choices: Sequence["StreamingChoices"] +) -> tuple[tuple[int, str], ...]: + """Every streamed choice carries a finish_reason, "stop" when the candidate sent none or Google's + FINISH_REASON_UNSPECIFIED, so only a candidate that named how it ended is evidence of it.""" + finished: Final = frozenset( + candidate.get("index", position) + for position, candidate in enumerate(candidates) + if (candidate.get("finishReason") or "FINISH_REASON_UNSPECIFIED") != "FINISH_REASON_UNSPECIFIED" + ) + return tuple( + (choice.index, choice.finish_reason) for choice in choices if choice.index in finished and choice.finish_reason + ) + + class ModelResponseIterator: def __init__( self, @@ -4459,6 +4476,9 @@ class ModelResponseIterator: setattr(model_response, "usage", usage) self._plan_resume(processed_chunk, model_response) + provider_finish_reasons: Final = _provider_finish_reasons(_candidates or (), model_response.choices) + if provider_finish_reasons: + set_hidden_param(model_response, PROVIDER_FINISH_REASONS_HIDDEN_PARAM, provider_finish_reasons) model_response.hidden_params["is_finished"] = False return model_response diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d3ecdb8a8f1..f90d9d3c3ac 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -87,11 +87,14 @@ from litellm.litellm_core_utils.cloud_storage_security import ( s3_media_uris, ) from litellm.litellm_core_utils.core_helpers import ( + FinishEvidence, + finish_evidence_of, finite_cost, get_or_create_metadata_bucket, has_attributes, independent_snapshot, is_expected_client_error, + map_finish_reason, proxy_request_headers, redact_nested_match_and_regex_keys, require_attributes, @@ -108,6 +111,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import ( guardrail_entries, guardrail_information_cost, + object_items, object_mapping, ) from litellm.litellm_core_utils.llm_response_utils.get_headers import ( @@ -119,6 +123,7 @@ from litellm.litellm_core_utils.served_output_texts import ( record_served_output_texts, served_output_texts, ) +from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor from litellm.litellm_core_utils.streaming_handler import ( backfill_missing_cache_usage_fields, ) @@ -365,6 +370,7 @@ from litellm.proxy.litellm_pre_call_utils import ( from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.utils import ( + Choices, LLMResponseTypes, ModelResponse, ModelResponseStream, @@ -588,6 +594,7 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons ) return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event) chunks: Final[object] = getattr(response, "chunks", None) + chunk_source: Final[object] = chunks if not isinstance(chunks, list) or not chunks: return False verbose_proxy_logger.debug( @@ -616,9 +623,48 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons partial_usage: Final = getattr(partial_response, "usage", None) if isinstance(partial_usage, Usage): backfill_missing_cache_usage_fields(partial_usage) + if isinstance(partial_response, ModelResponse): + partial_response.choices = list( + _choices_without_unobserved_finish_reasons( + partial_response.choices, + ChunkProcessor.choice_indices_with_observed_finish_reason(object_items(chunk_source)), + finish_evidence_of(response), + ) + ) return await _dispatch_disconnect_billing_event(logging_obj, partial_response) +def _choices_without_unobserved_finish_reasons( + choices: Sequence[Choices], observed_indices: frozenset[int], evidence: FinishEvidence +) -> tuple[Choices, ...]: + """Finish reasons the stream recorded per choice off the provider's frames decide every choice: the + terminal chunk the stream emits can carry another choice's index, so the assembled chunks cannot.""" + if evidence.by_choice: + return tuple( + _with_received_finish_reason(choice, _indexed_finish_reason(choice.index, len(choices), evidence.by_choice)) + for choice in choices + ) + single_choice_reason: Final = evidence.unindexed if len(choices) == 1 else None + return tuple( + choice if choice.index in observed_indices else _with_received_finish_reason(choice, single_choice_reason) + for choice in choices + ) + + +def _indexed_finish_reason(choice_index: int, choice_count: int, by_choice: Mapping[int, str]) -> str | None: + if choice_count == 1 and len(by_choice) == 1: + return next(iter(by_choice.values())) + return by_choice.get(choice_index) + + +def _with_received_finish_reason(choice: Choices, received_finish_reason: str | None) -> Choices: + if received_finish_reason is None: + return choice.model_copy(update={"finish_reason": "finish_reason_unspecified"}) + finish_reason: Final = map_finish_reason(received_finish_reason) + ends_a_tool_call_turn: Final = finish_reason == "stop" and bool(choice.message.tool_calls) + return choice.model_copy(update={"finish_reason": "tool_calls" if ends_a_tool_call_turn else finish_reason}) + + async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None: bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj")) if bill_in_flight_round is None: diff --git a/litellm/router_rust/invoke/streaming.py b/litellm/router_rust/invoke/streaming.py index 710511626d6..bbbd3a87a3d 100644 --- a/litellm/router_rust/invoke/streaming.py +++ b/litellm/router_rust/invoke/streaming.py @@ -63,7 +63,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import ( get_hidden_params_dict, replace_complexity_router_headers, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, without +from litellm.litellm_core_utils.core_helpers import FinishEvidence, as_str_mapping, finish_evidence_of, without from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.custom_httpx.stream_close import aclose_if_supported @@ -862,6 +862,11 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper): fallback takes over.""" fallback_headers_adopted: bool = False + _source_stream: CustomStreamWrapper + + @property + def finish_evidence(self) -> FinishEvidence: + return self._source_stream.finish_evidence def adopt_fallback_response_headers( self, @@ -911,6 +916,7 @@ def _chat_wrapper_init( class _AsyncChatStream(FallbackAwareStreamWrapper): def __init__(self, generator: AsyncGenerator[object, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it _chat_wrapper_init(self, generator, model_response) + self._source_stream = model_response self._async_generator: Final = generator def __aiter__(self) -> "_AsyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator @@ -923,6 +929,7 @@ class _AsyncChatStream(FallbackAwareStreamWrapper): class _SyncChatStream(FallbackAwareStreamWrapper): def __init__(self, generator: Generator[object, None, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it _chat_wrapper_init(self, generator, model_response) + self._source_stream = model_response self.chunks = [] # the sync twin keeps no inner chunk list hidden_params: Final = get_hidden_params(model_response) if hidden_params is not None: @@ -1133,6 +1140,10 @@ class FallbackAwareAnthropicMessagesStream: def chunks(self) -> object: return getattr(self._source_iterator, "chunks", None) + @property + def finish_evidence(self) -> FinishEvidence: + return finish_evidence_of(self._source_iterator) + @property def messages(self) -> object: return getattr(self._source_iterator, "messages", None) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 34e76bb05f5..9a96438851e 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9,6 +9,7 @@ import time import weakref from base64 import urlsafe_b64encode from collections.abc import AsyncIterable, Awaitable, Coroutine, Mapping +from itertools import chain from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence, TypeAlias, TypeVar from unittest.mock import AsyncMock, MagicMock, patch @@ -43,11 +44,13 @@ from litellm.litellm_core_utils.bug_report import ( bug_report_notice, build_bug_report, ) +from litellm.litellm_core_utils.core_helpers import FinishEvidence from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.total_deadline import current_total_deadline from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.vertex_ai.gemini.long_decode import AsyncStreamResume from litellm.proxy._types import Litellm_EntityType, LitellmUserRoles, ProxyErrorTypes, ProxyException from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck @@ -95,6 +98,10 @@ from litellm.proxy.hooks.model_max_budget_limiter import ( from litellm.proxy.utils import ProxyLogging from litellm.responses.mcp.parked_stream_dispatch import IN_FLIGHT_ROUND from litellm.router import Router +from litellm.router_config import build_router_config +from litellm.router_python_host import python_host +from litellm.router_rust.core import RustCore +from litellm.router_rust.invoke.streaming import StreamAttempt, open_attempt_stream from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic from litellm.types.utils import CachingDetails, Delta, ModelResponseStream, SpecialEnums, StreamingChoices from litellm.utils import ModelResponseListIterator @@ -11095,6 +11102,616 @@ class TestStreamingClientDisconnectBilling: litellm.token_counter(model=model, text=text, count_response_tokens=True), ) + @staticmethod + def _recorded_logging_obj( + model: str, + messages: list[dict[str, str]], + call_id: str, + recorder: _RecordingSuccessLogger, + call_type: str = "acompletion", + ) -> LiteLLMLoggingObj: + logging_obj: Final = LiteLLMLoggingObj( + model=model, + messages=messages, + stream=True, + call_type=call_type, + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + litellm_call_id=call_id, + function_id=call_id, + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": call_id} + ) + return logging_obj + + @pytest.mark.asyncio + async def test_disconnect_before_the_finish_frame_records_an_unspecified_finish_reason(self): + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}] + text: Final = "The quick brown fox jumps over the lazy dog. " * 20 + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = self._recorded_logging_obj(model, messages, "unfinished-disconnect-call", recorder) + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream(id="chatcmpl-unfinished", created=1741037890, model=model, choices=choices) + for choices in ( + [StreamingChoices(index=0, delta=Delta(role="assistant", content=text))], + [StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + ] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + first_chunk: Final = await response.__aiter__().__anext__() + assert first_chunk.choices[0].finish_reason is None + + billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, response) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"] + assert (partial_response.usage.prompt_tokens, partial_response.usage.completion_tokens) == ( + litellm.token_counter(model=model, messages=messages), + litellm.token_counter(model=model, text=text, count_response_tokens=True), + ) + + @pytest.mark.parametrize( + "provider_frames", + [ + pytest.param( + ( + [StreamingChoices(index=0, delta=Delta(role="assistant", content="partial answer"))], + [StreamingChoices(index=0, delta=Delta(content=""), finish_reason="length")], + ), + id="finish-on-its-own-frame", + ), + pytest.param( + ( + [ + StreamingChoices( + index=0, delta=Delta(role="assistant", content="partial answer"), finish_reason="length" + ) + ], + ), + id="finish-on-the-content-frame", + ), + ], + ) + @pytest.mark.asyncio + async def test_disconnect_after_the_provider_finished_keeps_the_received_finish_reason( + self, provider_frames: tuple[list[StreamingChoices], ...] + ): + model: Final = "gpt-4o" + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = self._recorded_logging_obj( + model, [{"role": "user", "content": "hi"}], "finished-disconnect-call", recorder + ) + response: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream(id="chatcmpl-finished", created=1741037890, model=model, choices=choices) + for choices in provider_frames + ] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + stream_iter: Final = response.__aiter__() + for _ in provider_frames: + await stream_iter.__anext__() + + billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, response) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert [choice.finish_reason for choice in partial_response.choices] == ["length"] + assert partial_response.usage.completion_tokens > 0 + + @pytest.fixture + def stream_attempt(self) -> Iterator[StreamAttempt]: + core: Final = RustCore.build(build_router_config(MappingProxyType({"model_list": []})), python_host) + yield StreamAttempt( + core=core, + op="acompletion", + request=MappingProxyType({}), + kwargs=MappingProxyType({}), + group="gpt-4o", + order=None, + depth=0, + level=0, + deployment_id=None, + attempted_retries=0, + ) + core.retire() + + async def _router_stream( + self, + recorder: _RecordingSuccessLogger, + frames: Sequence[Sequence[StreamingChoices]], + stream_attempt: StreamAttempt, + ) -> CustomStreamWrapper: + model: Final = "gpt-4o" + logging_obj: Final = self._recorded_logging_obj( + model, [{"role": "user", "content": "hi"}], "router-held-finish-call", recorder + ) + provider_stream: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator( + model_responses=[ + ModelResponseStream(id="chatcmpl-held", created=1741037890, model=model, choices=list(choices)) + for choices in frames + ] + ), + model=model, + custom_llm_provider="openai", + logging_obj=logging_obj, + stream_options=None, + ) + routed: Final = await open_attempt_stream(stream_attempt, provider_stream) + assert isinstance(routed, CustomStreamWrapper) + return routed + + async def _router_stream_holding_a_length_finish( + self, recorder: _RecordingSuccessLogger, stream_attempt: StreamAttempt + ) -> CustomStreamWrapper: + return await self._router_stream( + recorder, + [ + [ + StreamingChoices( + index=0, delta=Delta(role="assistant", content="partial answer"), finish_reason="length" + ) + ] + ], + stream_attempt, + ) + + async def _assert_disconnect_bills_length( + self, recorder: _RecordingSuccessLogger, logging_obj: LiteLLMLoggingObj, streamed: object + ) -> None: + billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert [choice.finish_reason for choice in partial_response.choices] == ["length"] + assert partial_response.usage.completion_tokens > 0 + + @pytest.mark.asyncio + async def test_router_chat_stream_keeps_the_finish_reason_held_on_the_last_content_frame( + self, stream_attempt: StreamAttempt + ): + recorder: Final = _RecordingSuccessLogger() + routed: Final = await self._router_stream_holding_a_length_finish(recorder, stream_attempt) + + delivered: Final = await routed.__aiter__().__anext__() + assert (delivered.choices[0].delta.content, delivered.choices[0].finish_reason) == ("partial answer", None) + + await self._assert_disconnect_bills_length(recorder, routed.logging_obj, routed) + + @pytest.mark.parametrize("through_response_cache", [False, True], ids=["uncached", "response-cache-writer"]) + @pytest.mark.asyncio + async def test_anthropic_messages_stream_keeps_the_finish_reason_held_on_the_last_content_frame( + self, through_response_cache: bool, stream_attempt: StreamAttempt + ): + from litellm.caching.caching_handler import LLMCachingHandler + from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream + from litellm.llms.anthropic.messages.adapters.transformation import AnthropicAdapter + from litellm.llms.anthropic.messages.native import handler + from litellm.llms.anthropic.messages.native.response_cache import AnthropicMessagesStreamCacheWriter + from litellm.router_rust.invoke.streaming import FallbackAwareAnthropicMessagesStream + + recorder: Final = _RecordingSuccessLogger() + routed: Final = await self._router_stream_holding_a_length_finish(recorder, stream_attempt) + adapted: Final = AnthropicAdapter().translate_completion_output_params_streaming( + routed, model="gpt-4o", is_async=True, litellm_logging_obj=routed.logging_obj + ) + assert isinstance(adapted, AnthropicSSEStream) + source_iterator: Final = ( + AnthropicMessagesStreamCacheWriter( + stream=adapted, + caching_handler=LLMCachingHandler( + original_function=handler.anthropic_messages, + request_kwargs={}, + start_time=datetime.datetime(2025, 3, 3, 21, 38, 10), + ), + ) + if through_response_cache + else adapted + ) + async for frame in source_iterator: + if isinstance(frame, bytes) and b"partial answer" in frame: + break + + async def _sse_frames() -> AsyncGenerator[bytes, None]: + yield b"event: message_start\n\n" + + await self._assert_disconnect_bills_length( + recorder, + routed.logging_obj, + prepare_response_for_header_attachment( + FallbackAwareAnthropicMessagesStream(_sse_frames(), source_iterator, stream_attempt) + ), + ) + + @pytest.mark.asyncio + async def test_disconnect_bills_a_partial_text_completion_without_inventing_a_finish_reason(self): + from litellm.types.utils import TextChoices, TextCompletionResponse + + model: Final = "gpt-3.5-turbo-instruct" + messages: Final = [{"role": "user", "content": "Say something long"}] + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = self._recorded_logging_obj( + model, messages, "text-completion-disconnect-call", recorder, call_type="atext_completion" + ) + collected_stream: Final = SimpleNamespace( + chunks=[ + TextCompletionResponse( + id="cmpl-partial", created=1741037890, model=model, choices=[TextChoices(index=0, text=text)] + ) + for text in ("partial ", "answer") + ], + messages=messages, + model=model, + ) + + billed: Final = await _bill_partial_streamed_spend_on_disconnect( + {"litellm_logging_obj": logging_obj}, collected_stream + ) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert [(choice.text, choice.finish_reason) for choice in partial_response.choices] == [ + ("partial answer", None) + ] + assert partial_response.usage.completion_tokens == litellm.token_counter( + model=model, text="partial answer", count_response_tokens=True + ) + + @staticmethod + def _content_choice(index: int, text: str, finish_reason: str | None = None) -> StreamingChoices: + return StreamingChoices(index=index, delta=Delta(role="assistant", content=text), finish_reason=finish_reason) + + @pytest.mark.parametrize( + ("frames", "expected"), + [ + pytest.param( + (((0, "zero ", None),), ((1, "one ", None),), ((0, "done", "length"),)), + {0: "length", 1: "finish_reason_unspecified"}, + id="choice-0-finished", + ), + pytest.param( + (((0, "zero ", None),), ((1, "one ", None),), ((1, "done", "length"),)), + {0: "finish_reason_unspecified", 1: "length"}, + id="choice-1-finished", + ), + pytest.param( + (((1, "one", "stop"),), ((0, "zero", "length"),)), + {0: "length", 1: "stop"}, + id="both-finished-out-of-order", + ), + pytest.param( + (((2, "two", "length"),), ((0, "zero", "stop"),)), + {0: "stop", 2: "length"}, + id="sparse-indexes", + ), + pytest.param( + (((2, "partial answer", "length"),),), + {0: "length"}, + id="single-choice-nonzero-index", + ), + pytest.param( + (((0, "zero", "length"), (1, "one", "stop")),), + {0: "length", 1: "stop"}, + id="one-frame-two-finishes", + ), + ], + ) + @pytest.mark.asyncio + async def test_router_stream_keeps_each_choice_finish_reason_held_on_its_content_frame( + self, + frames: tuple[tuple[tuple[int, str, str | None], ...], ...], + expected: Mapping[int, str], + stream_attempt: StreamAttempt, + ): + recorder: Final = _RecordingSuccessLogger() + routed: Final = await self._router_stream( + recorder, + [tuple(self._content_choice(*choice) for choice in frame) for frame in frames], + stream_attempt, + ) + stream_iter: Final = routed.__aiter__() + delivered: Final = [await stream_iter.__anext__() for _ in frames] + delivered_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in delivered)) + assert [choice.finish_reason for choice in delivered_choices] == [None] * sum(map(len, frames)) + + billed: Final = await _bill_partial_streamed_spend_on_disconnect( + {"litellm_logging_obj": routed.logging_obj}, routed + ) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert {choice.index: choice.finish_reason for choice in partial_response.choices} == expected + assert partial_response.usage.completion_tokens > 0 + + @pytest.mark.parametrize( + ("frames", "expected"), + [ + pytest.param( + (((0, "zero ", None),), ((1, "one ", None),), ((1, "", "length"),)), + {0: "finish_reason_unspecified", 1: "length"}, + id="choice-1-terminal-frame", + ), + pytest.param( + (((0, "zero ", None), (1, "one ", None)), ((0, "", "length"), (1, "", "stop"))), + {0: "length", 1: "stop"}, + id="one-terminal-frame-two-finishes", + ), + ], + ) + @pytest.mark.asyncio + async def test_disconnect_after_a_terminal_frame_bills_each_choice_its_own_finish_reason( + self, + frames: tuple[tuple[tuple[int, str, str | None], ...], ...], + expected: Mapping[int, str], + stream_attempt: StreamAttempt, + ): + recorder: Final = _RecordingSuccessLogger() + routed: Final = await self._router_stream( + recorder, + [tuple(self._content_choice(*choice) for choice in frame) for frame in frames], + stream_attempt, + ) + stream_iter: Final = routed.__aiter__() + for _ in frames: + await stream_iter.__anext__() + + billed: Final = await _bill_partial_streamed_spend_on_disconnect( + {"litellm_logging_obj": routed.logging_obj}, routed + ) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert {choice.index: choice.finish_reason for choice in partial_response.choices} == expected + assert partial_response.usage.completion_tokens > 0 + + @pytest.mark.parametrize( + ("candidate", "expected"), + [ + pytest.param( + {"content": {"parts": [{"text": "partial answer"}], "role": "model"}, "index": 0}, + "finish_reason_unspecified", + id="no-finish-reason", + ), + pytest.param( + { + "content": {"parts": [{"text": "partial answer"}], "role": "model"}, + "finishReason": "FINISH_REASON_UNSPECIFIED", + }, + "finish_reason_unspecified", + id="finish-reason-unspecified-sentinel", + ), + pytest.param( + {"content": {"parts": [{"text": "partial answer"}], "role": "model"}, "finishReason": "MAX_TOKENS"}, + "length", + id="max-tokens", + ), + pytest.param( + {"content": {"parts": [{"text": "full answer"}], "role": "model"}, "finishReason": "STOP"}, + "stop", + id="stop", + ), + pytest.param( + { + "content": {"parts": [{"functionCall": {"name": "lookup", "args": {"q": "fox"}}}], "role": "model"}, + "finishReason": "STOP", + }, + "tool_calls", + id="stop-after-a-tool-call", + ), + ], + ) + @pytest.mark.asyncio + async def test_gemini_stream_bills_only_the_finish_reason_the_candidate_sent( + self, candidate: Mapping[str, object], expected: str, stream_attempt: StreamAttempt + ): + partial_response: Final = await self._billed_after_the_first_gemini_frame( + stream_attempt, {"candidates": [candidate]} + ) + + assert [choice.finish_reason for choice in partial_response.choices] == [expected] + assert partial_response.usage.completion_tokens > 0 + + @pytest.mark.asyncio + async def test_gemini_frame_that_plans_a_continuation_hop_is_not_billed_as_finished( + self, stream_attempt: StreamAttempt + ): + from litellm.types.llms.vertex_ai import GenerationConfig, RequestBody + + async def _hop_must_not_open(request_body: RequestBody) -> httpx.Response: + pytest.fail(f"the client disconnected before the hop, yet it opened with {request_body}") + + partial_response: Final = await self._billed_after_the_first_gemini_frame( + stream_attempt, + { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "hello"}]}, + "finishReason": "CONTINUATION", + "continuationToken": "dG9rLTE=", + } + ], + "usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 30, "totalTokenCount": 40}, + }, + AsyncStreamResume( + request_body=RequestBody( + contents=[{"role": "user", "parts": [{"text": "Prove it"}]}], + generationConfig=GenerationConfig(max_output_tokens=100), + ), + open_hop=_hop_must_not_open, + ), + ) + + assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"] + assert partial_response.usage.completion_tokens > 0 + + @pytest.mark.parametrize("evidence", ["length", ((0, None),), ((0, "length", "stop"),)]) + @pytest.mark.asyncio + async def test_unreadable_provider_finish_evidence_bills_unspecified_instead_of_the_synthesized_stop( + self, evidence: object, stream_attempt: StreamAttempt + ): + from litellm.constants import PROVIDER_FINISH_REASONS_HIDDEN_PARAM + from litellm.litellm_core_utils.hidden_params import set_hidden_param + + frame: Final = ModelResponseStream( + id="chatcmpl-unreadable", + created=1741037890, + model="gemini-3.8-flash", + choices=[self._content_choice(0, "partial answer", "stop")], + ) + set_hidden_param(frame, "is_finished", False) + set_hidden_param(frame, PROVIDER_FINISH_REASONS_HIDDEN_PARAM, evidence) + + partial_response: Final = await self._billed_after_the_first_frame( + stream_attempt, lambda _: ModelResponseListIterator(model_responses=[frame]) + ) + + assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"] + + async def _billed_after_the_first_gemini_frame( + self, stream_attempt: StreamAttempt, frame: Mapping[str, object], resume: AsyncStreamResume | None = None + ) -> litellm.ModelResponse: + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ModelResponseIterator + + async def _gemini_sse() -> AsyncGenerator[str, None]: + yield f"data: {json.dumps(frame)}" + + return await self._billed_after_the_first_frame( + stream_attempt, + lambda logging_obj: ModelResponseIterator( + streaming_response=_gemini_sse(), sync_stream=False, logging_obj=logging_obj, resume=resume + ), + ) + + async def _billed_after_the_first_frame( + self, stream_attempt: StreamAttempt, completion_stream: Callable[[LiteLLMLoggingObj], object] + ) -> litellm.ModelResponse: + model: Final = "gemini-3.8-flash" + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = self._recorded_logging_obj( + model, [{"role": "user", "content": "hi"}], "first-frame-disconnect-call", recorder + ) + provider_stream: Final = CustomStreamWrapper( + completion_stream=completion_stream(logging_obj), + model=model, + custom_llm_provider="gemini", + logging_obj=logging_obj, + stream_options=None, + ) + routed: Final = await open_attempt_stream(stream_attempt, provider_stream) + assert isinstance(routed, CustomStreamWrapper) + delivered: Final = await routed.__aiter__().__anext__() + assert delivered.choices[0].finish_reason is None + + billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, routed) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + return recorder.success_events[0]["response_obj"] + + @pytest.mark.asyncio + async def test_disconnect_uses_a_finish_reason_received_without_a_choice_index_for_a_single_choice(self): + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "hi"}] + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = self._recorded_logging_obj(model, messages, "unindexed-finish-call", recorder) + collected_stream: Final = SimpleNamespace( + chunks=[ + ModelResponseStream( + id="chatcmpl-unindexed", + created=1741037890, + model=model, + choices=[self._content_choice(0, "partial answer")], + ) + ], + messages=messages, + model=model, + finish_evidence=FinishEvidence(by_choice=MappingProxyType({}), unindexed="length"), + ) + + billed: Final = await _bill_partial_streamed_spend_on_disconnect( + {"litellm_logging_obj": logging_obj}, collected_stream + ) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + partial_response: Final = recorder.success_events[0]["response_obj"] + assert [choice.finish_reason for choice in partial_response.choices] == ["length"] + + @pytest.mark.asyncio + async def test_disconnect_marks_only_the_unfinished_choice_unspecified(self): + model: Final = "gpt-4o" + messages: Final = [{"role": "user", "content": "look both up"}] + recorder: Final = _RecordingSuccessLogger() + logging_obj: Final = self._recorded_logging_obj(model, messages, "two-choice-disconnect-call", recorder) + + def tool_call_delta(call_id: str) -> Delta: + return Delta( + role="assistant", + tool_calls=[ + {"id": call_id, "type": "function", "index": 0, "function": {"name": "lookup", "arguments": "{}"}} + ], + ) + + collected_stream: Final = SimpleNamespace( + chunks=[ + ModelResponseStream(id="chatcmpl-two", created=1741037890, model=model, choices=choices) + for choices in ( + [ + StreamingChoices(index=0, delta=tool_call_delta("call_a")), + StreamingChoices(index=1, delta=tool_call_delta("call_b")), + ], + [StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")], + ) + ], + messages=messages, + model=model, + finish_evidence=FinishEvidence(by_choice=MappingProxyType({}), unindexed="tool_calls"), + ) + + billed: Final = await _bill_partial_streamed_spend_on_disconnect( + {"litellm_logging_obj": logging_obj}, collected_stream + ) + await asyncio.wait_for(recorder.logged.wait(), timeout=5) + + assert billed is True + assert len(recorder.success_events) == 1 + partial_response: Final = recorder.success_events[0]["response_obj"] + assert {choice.index: choice.finish_reason for choice in partial_response.choices} == { + 0: "tool_calls", + 1: "finish_reason_unspecified", + } + assert [len(choice.message.tool_calls) for choice in partial_response.choices] == [1, 1] + assert partial_response.usage.total_tokens > 0 + @pytest.mark.asyncio async def test_completed_stream_does_not_double_bill_on_late_disconnect(self): recorder = _RecordingSuccessLogger()