fix(proxy): preserve terminal evidence in disconnect billing (#52088)

* fix(proxy): preserve terminal evidence in disconnect billing

* fix(proxy): retain held finish reasons through stream wrappers

* refactor(proxy): use immutable finish evidence results

* fix(proxy): type disconnect evidence boundaries and cover wrappers

* fix(proxy): preserve object type before chunk narrowing

* fix(proxy): retain disconnect finish evidence per choice

* fix(proxy): preserve genuine Gemini finish evidence

* fix(proxy): use typed finish evidence storage

* test(proxy): exercise real router stream attempts

* fix(proxy): keep unspecified Gemini evidence unknown

---------

Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
joshua-berri 2026-11-16 14:43:03 -08:00 • committed by GitHub
parent 4af6fb70d8
commit e7c89b0dd4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 800 additions and 3 deletions

View file

@ -708,6 +708,7 @@ STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
PROVIDER_FINISH_REASONS_HIDDEN_PARAM: Final = "provider_finish_reasons"
VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(

View file

@ -6,6 +6,7 @@ import math
import re
import sys
from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
@ -191,6 +192,23 @@ def attribute_or_none(source: object, name: str) -> object:
_Shape: Final = TypeVar("_Shape")
@dataclass(frozen=True, slots=True)
class FinishEvidence:
"""Finish reasons a stream received from the provider. ``unindexed`` is a received reason whose
choice the stream could not tell, so it is only evidence for a single-choice response."""
by_choice: Mapping[int, str]
unindexed: str | None
NO_FINISH_EVIDENCE: Final = FinishEvidence(by_choice=MappingProxyType({}), unindexed=None)
def finish_evidence_of(stream: object) -> FinishEvidence:
evidence: Final[object] = getattr(stream, "finish_evidence", None)
return evidence if isinstance(evidence, FinishEvidence) else NO_FINISH_EVIDENCE
def has_attributes(
value: object,
shape: type[_Shape],

View file

@ -1124,6 +1124,18 @@ class ChunkProcessor:
return chunk.get("choices", ())
return getattr(chunk, "choices", ())
@staticmethod
def choice_indices_with_observed_finish_reason(chunks: Iterable[object]) -> frozenset[int]:
choices: Final = tuple(chain.from_iterable(object_items(_field_of(chunk, "choices")) for chunk in chunks))
finished: Final = frozenset(
_stream_choice_index(choice)
for choice in choices
if ChunkProcessor._finish_reason_of_choice(choice) is not None
)
if len(frozenset(map(_stream_choice_index, choices))) > 1:
return finished
return frozenset({0}) if finished else frozenset()
@staticmethod
def _saw_finish_reason(chunks: Sequence["_UsageBearingChunk | ModelResponse"]) -> bool:
return any(

View file

@ -23,10 +23,15 @@ from typing_extensions import NotRequired, TypedDict
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS
from litellm.constants import (
PROVIDER_FINISH_REASONS_HIDDEN_PARAM,
STREAM_CHUNK_CHOICE_FIELDS,
STREAM_FINISH_CHOICE_FIELDS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
FinishEvidence,
as_str_mapping,
attribute_or_none,
chunk_usage,
@ -171,6 +176,35 @@ def _prepend_upstream_choice_extras(
_NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
_NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({})
_NO_FINISH_REASONS_BY_INDEX: Final[Mapping[int, str]] = MappingProxyType({})
def _indexed_finish_reason(choice: object) -> tuple[int, str] | None:
reason: Final[object] = getattr(choice, "finish_reason", None)
index: Final[object] = getattr(choice, "index", None)
if not isinstance(reason, str) or not reason or not isinstance(index, int):
return None
return index, reason
_PROVIDER_FINISH_REASONS: Final = TypeAdapter(tuple[tuple[int, str], ...])
def _provider_finish_reasons(evidence: object) -> Mapping[int, str]:
if evidence is None:
return _NO_FINISH_REASONS_BY_INDEX
try:
return MappingProxyType(dict(_PROVIDER_FINISH_REASONS.validate_python(evidence, strict=True)))
except ValidationError:
return _NO_FINISH_REASONS_BY_INDEX
def _finish_reasons_by_index(upstream_chunk: object) -> Mapping[int, str]:
hidden_params: Final = get_hidden_params(upstream_chunk)
if hidden_params is not None and hidden_params.get("is_finished") is False:
return _provider_finish_reasons(hidden_params.get(PROVIDER_FINISH_REASONS_HIDDEN_PARAM))
choices: Final[object] = getattr(upstream_chunk, "choices", None)
return MappingProxyType(dict(filter(None, map(_indexed_finish_reason, object_items(choices)))))
def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None:
@ -509,6 +543,7 @@ class CustomStreamWrapper:
self._upstream_provider: str | None = None
self.received_finish_reason: str | None = None
self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
self.received_finish_reasons: Mapping[int, str] = _NO_FINISH_REASONS_BY_INDEX
self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
@ -1729,6 +1764,7 @@ class CustomStreamWrapper:
response_obj = self.handle_openai_text_completion_chunk(chunk)
completion_obj["content"] = response_obj["text"]
print_verbose(f"completion obj content: {completion_obj['content']}")
self._record_finish_reasons(chunk)
if response_obj["is_finished"]:
self._record_finish_choice_fields(chunk)
if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
@ -1805,6 +1841,7 @@ class CustomStreamWrapper:
self.response_id = cached_chunk.id
if hasattr(cached_chunk, "system_fingerprint"):
self.system_fingerprint = cached_chunk.system_fingerprint
self._record_finish_reasons(cached_chunk)
if response_obj["is_finished"]:
self.received_finish_reason = response_obj["finish_reason"]
self._record_finish_choice_fields(cached_chunk)
@ -1840,6 +1877,7 @@ class CustomStreamWrapper:
)
self.received_finish_reason = response_obj["finish_reason"]
self._record_finish_choice_fields(chunk)
self._record_finish_reasons(chunk)
if response_obj.get("original_chunk", None) is not None:
if hasattr(response_obj["original_chunk"], "id"):
model_response = self.set_model_id(response_obj["original_chunk"].id, model_response)
@ -2190,6 +2228,18 @@ class CustomStreamWrapper:
if self.logging_obj.is_sync_litellm_request(litellm_params):
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
@property
def finish_evidence(self) -> FinishEvidence:
return FinishEvidence(
by_choice=self.received_finish_reasons,
unindexed=None if self.received_finish_reasons else self.received_finish_reason,
)
def _record_finish_reasons(self, upstream_chunk: object) -> None:
received: Final = _finish_reasons_by_index(upstream_chunk)
if received:
self.received_finish_reasons = MappingProxyType({**self.received_finish_reasons, **received})
def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str:
if finish_reason == "stop" and self.tool_call:
return "tool_calls"

View file

@ -27,7 +27,12 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.core_helpers import (
FinishEvidence,
as_str_mapping,
attribute_items,
finish_evidence_of,
)
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
@ -283,6 +288,10 @@ class _CombinedChunkSplitter:
"list[ModelResponseStream] | None", getattr(self._stream, "chunks", None)
)
@property
def finish_evidence(self) -> FinishEvidence:
return finish_evidence_of(self._stream)
@property
def messages(self) -> "list[AllMessageValues] | None":
return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream
@ -711,6 +720,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
"list[ModelResponseStream] | None", getattr(self.completion_stream, "chunks", None)
)
@property
def finish_evidence(self) -> FinishEvidence:
return self._combined_chunk_splitter.finish_evidence
@property
def messages(self) -> "list[AllMessageValues] | None":
return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream
@ -1645,6 +1658,10 @@ class AnthropicSSEStream(AsyncIterator[bytes]):
def chunks(self) -> "list[ModelResponseStream] | None":
return self._anthropic_wrapper.chunks
@property
def finish_evidence(self) -> FinishEvidence:
return self._anthropic_wrapper.finish_evidence
@property
def messages(self) -> "list[AllMessageValues] | None":
return self._anthropic_wrapper.messages

View file

@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final, cast
import litellm
from litellm._logging import verbose_logger
from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
from litellm.litellm_core_utils.core_helpers import FinishEvidence, finish_evidence_of
from litellm.llms.anthropic.messages.native.streaming_iterator import (
AnthropicMessagesStreamingResponse,
BaseAnthropicMessagesStreamingIterator,
@ -59,6 +60,10 @@ class AnthropicMessagesStreamCacheWriter:
"list[ModelResponseStream] | None", getattr(self.stream, "chunks", None)
)
@property
def finish_evidence(self) -> FinishEvidence:
return finish_evidence_of(self.stream)
@property
def messages(self) -> "list[AllMessageValues] | None":
return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream

View file

@ -29,10 +29,12 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH,
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE,
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
PROVIDER_FINISH_REASONS_HIDDEN_PARAM,
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.hidden_params import set_hidden_param
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
@ -4213,6 +4215,21 @@ def _transform_long_decode_hops(
return merged if marked is merged_usage else merged.model_copy(update={"usage": marked})
def _provider_finish_reasons(
candidates: Sequence[Candidates], choices: Sequence["StreamingChoices"]
) -> tuple[tuple[int, str], ...]:
"""Every streamed choice carries a finish_reason, "stop" when the candidate sent none or Google's
FINISH_REASON_UNSPECIFIED, so only a candidate that named how it ended is evidence of it."""
finished: Final = frozenset(
candidate.get("index", position)
for position, candidate in enumerate(candidates)
if (candidate.get("finishReason") or "FINISH_REASON_UNSPECIFIED") != "FINISH_REASON_UNSPECIFIED"
)
return tuple(
(choice.index, choice.finish_reason) for choice in choices if choice.index in finished and choice.finish_reason
)
class ModelResponseIterator:
def __init__(
self,
@ -4459,6 +4476,9 @@ class ModelResponseIterator:
setattr(model_response, "usage", usage)
self._plan_resume(processed_chunk, model_response)
provider_finish_reasons: Final = _provider_finish_reasons(_candidates or (), model_response.choices)
if provider_finish_reasons:
set_hidden_param(model_response, PROVIDER_FINISH_REASONS_HIDDEN_PARAM, provider_finish_reasons)
model_response.hidden_params["is_finished"] = False
return model_response

View file

@ -87,11 +87,14 @@ from litellm.litellm_core_utils.cloud_storage_security import (
s3_media_uris,
)
from litellm.litellm_core_utils.core_helpers import (
FinishEvidence,
finish_evidence_of,
finite_cost,
get_or_create_metadata_bucket,
has_attributes,
independent_snapshot,
is_expected_client_error,
map_finish_reason,
proxy_request_headers,
redact_nested_match_and_regex_keys,
require_attributes,
@ -108,6 +111,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
guardrail_entries,
guardrail_information_cost,
object_items,
object_mapping,
)
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
@ -119,6 +123,7 @@ from litellm.litellm_core_utils.served_output_texts import (
record_served_output_texts,
served_output_texts,
)
from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor
from litellm.litellm_core_utils.streaming_handler import (
backfill_missing_cache_usage_fields,
)
@ -365,6 +370,7 @@ from litellm.proxy.litellm_pre_call_utils import (
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.utils import (
Choices,
LLMResponseTypes,
ModelResponse,
ModelResponseStream,
@ -588,6 +594,7 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
)
return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event)
chunks: Final[object] = getattr(response, "chunks", None)
chunk_source: Final[object] = chunks
if not isinstance(chunks, list) or not chunks:
return False
verbose_proxy_logger.debug(
@ -616,9 +623,48 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
partial_usage: Final = getattr(partial_response, "usage", None)
if isinstance(partial_usage, Usage):
backfill_missing_cache_usage_fields(partial_usage)
if isinstance(partial_response, ModelResponse):
partial_response.choices = list(
_choices_without_unobserved_finish_reasons(
partial_response.choices,
ChunkProcessor.choice_indices_with_observed_finish_reason(object_items(chunk_source)),
finish_evidence_of(response),
)
)
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
def _choices_without_unobserved_finish_reasons(
choices: Sequence[Choices], observed_indices: frozenset[int], evidence: FinishEvidence
) -> tuple[Choices, ...]:
"""Finish reasons the stream recorded per choice off the provider's frames decide every choice: the
terminal chunk the stream emits can carry another choice's index, so the assembled chunks cannot."""
if evidence.by_choice:
return tuple(
_with_received_finish_reason(choice, _indexed_finish_reason(choice.index, len(choices), evidence.by_choice))
for choice in choices
)
single_choice_reason: Final = evidence.unindexed if len(choices) == 1 else None
return tuple(
choice if choice.index in observed_indices else _with_received_finish_reason(choice, single_choice_reason)
for choice in choices
)
def _indexed_finish_reason(choice_index: int, choice_count: int, by_choice: Mapping[int, str]) -> str | None:
if choice_count == 1 and len(by_choice) == 1:
return next(iter(by_choice.values()))
return by_choice.get(choice_index)
def _with_received_finish_reason(choice: Choices, received_finish_reason: str | None) -> Choices:
if received_finish_reason is None:
return choice.model_copy(update={"finish_reason": "finish_reason_unspecified"})
finish_reason: Final = map_finish_reason(received_finish_reason)
ends_a_tool_call_turn: Final = finish_reason == "stop" and bool(choice.message.tool_calls)
return choice.model_copy(update={"finish_reason": "tool_calls" if ends_a_tool_call_turn else finish_reason})
async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None:
bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj"))
if bill_in_flight_round is None:

View file

@ -63,7 +63,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
get_hidden_params_dict,
replace_complexity_router_headers,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, without
from litellm.litellm_core_utils.core_helpers import FinishEvidence, as_str_mapping, finish_evidence_of, without
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.custom_httpx.stream_close import aclose_if_supported
@ -862,6 +862,11 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper):
fallback takes over."""
fallback_headers_adopted: bool = False
_source_stream: CustomStreamWrapper
@property
def finish_evidence(self) -> FinishEvidence:
return self._source_stream.finish_evidence
def adopt_fallback_response_headers(
self,
@ -911,6 +916,7 @@ def _chat_wrapper_init(
class _AsyncChatStream(FallbackAwareStreamWrapper):
def __init__(self, generator: AsyncGenerator[object, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it
_chat_wrapper_init(self, generator, model_response)
self._source_stream = model_response
self._async_generator: Final = generator
def __aiter__(self) -> "_AsyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator
@ -923,6 +929,7 @@ class _AsyncChatStream(FallbackAwareStreamWrapper):
class _SyncChatStream(FallbackAwareStreamWrapper):
def __init__(self, generator: Generator[object, None, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it
_chat_wrapper_init(self, generator, model_response)
self._source_stream = model_response
self.chunks = [] # the sync twin keeps no inner chunk list
hidden_params: Final = get_hidden_params(model_response)
if hidden_params is not None:
@ -1133,6 +1140,10 @@ class FallbackAwareAnthropicMessagesStream:
def chunks(self) -> object:
return getattr(self._source_iterator, "chunks", None)
@property
def finish_evidence(self) -> FinishEvidence:
return finish_evidence_of(self._source_iterator)
@property
def messages(self) -> object:
return getattr(self._source_iterator, "messages", None)

View file

@ -9,6 +9,7 @@ import time
import weakref
from base64 import urlsafe_b64encode
from collections.abc import AsyncIterable, Awaitable, Coroutine, Mapping
from itertools import chain
from types import MappingProxyType, SimpleNamespace
from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence, TypeAlias, TypeVar
from unittest.mock import AsyncMock, MagicMock, patch
@ -43,11 +44,13 @@ from litellm.litellm_core_utils.bug_report import (
bug_report_notice,
build_bug_report,
)
from litellm.litellm_core_utils.core_helpers import FinishEvidence
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.total_deadline import current_total_deadline
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.vertex_ai.gemini.long_decode import AsyncStreamResume
from litellm.proxy._types import Litellm_EntityType, LitellmUserRoles, ProxyErrorTypes, ProxyException
from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth
from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck
@ -95,6 +98,10 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
from litellm.proxy.utils import ProxyLogging
from litellm.responses.mcp.parked_stream_dispatch import IN_FLIGHT_ROUND
from litellm.router import Router
from litellm.router_config import build_router_config
from litellm.router_python_host import python_host
from litellm.router_rust.core import RustCore
from litellm.router_rust.invoke.streaming import StreamAttempt, open_attempt_stream
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
from litellm.types.utils import CachingDetails, Delta, ModelResponseStream, SpecialEnums, StreamingChoices
from litellm.utils import ModelResponseListIterator
@ -11095,6 +11102,616 @@ class TestStreamingClientDisconnectBilling:
litellm.token_counter(model=model, text=text, count_response_tokens=True),
)
@staticmethod
def _recorded_logging_obj(
model: str,
messages: list[dict[str, str]],
call_id: str,
recorder: _RecordingSuccessLogger,
call_type: str = "acompletion",
) -> LiteLLMLoggingObj:
logging_obj: Final = LiteLLMLoggingObj(
model=model,
messages=messages,
stream=True,
call_type=call_type,
start_time=datetime.datetime(2025, 3, 3, 21, 38, 10),
litellm_call_id=call_id,
function_id=call_id,
dynamic_async_success_callbacks=[recorder],
)
logging_obj.update_environment_variables(
model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": call_id}
)
return logging_obj
@pytest.mark.asyncio
async def test_disconnect_before_the_finish_frame_records_an_unspecified_finish_reason(self):
model: Final = "gpt-4o"
messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}]
text: Final = "The quick brown fox jumps over the lazy dog. " * 20
recorder: Final = _RecordingSuccessLogger()
logging_obj: Final = self._recorded_logging_obj(model, messages, "unfinished-disconnect-call", recorder)
response: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(
model_responses=[
ModelResponseStream(id="chatcmpl-unfinished", created=1741037890, model=model, choices=choices)
for choices in (
[StreamingChoices(index=0, delta=Delta(role="assistant", content=text))],
[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")],
)
]
),
model=model,
custom_llm_provider="openai",
logging_obj=logging_obj,
stream_options=None,
)
first_chunk: Final = await response.__aiter__().__anext__()
assert first_chunk.choices[0].finish_reason is None
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, response)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"]
assert (partial_response.usage.prompt_tokens, partial_response.usage.completion_tokens) == (
litellm.token_counter(model=model, messages=messages),
litellm.token_counter(model=model, text=text, count_response_tokens=True),
)
@pytest.mark.parametrize(
"provider_frames",
[
pytest.param(
(
[StreamingChoices(index=0, delta=Delta(role="assistant", content="partial answer"))],
[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="length")],
),
id="finish-on-its-own-frame",
),
pytest.param(
(
[
StreamingChoices(
index=0, delta=Delta(role="assistant", content="partial answer"), finish_reason="length"
)
],
),
id="finish-on-the-content-frame",
),
],
)
@pytest.mark.asyncio
async def test_disconnect_after_the_provider_finished_keeps_the_received_finish_reason(
self, provider_frames: tuple[list[StreamingChoices], ...]
):
model: Final = "gpt-4o"
recorder: Final = _RecordingSuccessLogger()
logging_obj: Final = self._recorded_logging_obj(
model, [{"role": "user", "content": "hi"}], "finished-disconnect-call", recorder
)
response: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(
model_responses=[
ModelResponseStream(id="chatcmpl-finished", created=1741037890, model=model, choices=choices)
for choices in provider_frames
]
),
model=model,
custom_llm_provider="openai",
logging_obj=logging_obj,
stream_options=None,
)
stream_iter: Final = response.__aiter__()
for _ in provider_frames:
await stream_iter.__anext__()
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, response)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert [choice.finish_reason for choice in partial_response.choices] == ["length"]
assert partial_response.usage.completion_tokens > 0
@pytest.fixture
def stream_attempt(self) -> Iterator[StreamAttempt]:
core: Final = RustCore.build(build_router_config(MappingProxyType({"model_list": []})), python_host)
yield StreamAttempt(
core=core,
op="acompletion",
request=MappingProxyType({}),
kwargs=MappingProxyType({}),
group="gpt-4o",
order=None,
depth=0,
level=0,
deployment_id=None,
attempted_retries=0,
)
core.retire()
async def _router_stream(
self,
recorder: _RecordingSuccessLogger,
frames: Sequence[Sequence[StreamingChoices]],
stream_attempt: StreamAttempt,
) -> CustomStreamWrapper:
model: Final = "gpt-4o"
logging_obj: Final = self._recorded_logging_obj(
model, [{"role": "user", "content": "hi"}], "router-held-finish-call", recorder
)
provider_stream: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(
model_responses=[
ModelResponseStream(id="chatcmpl-held", created=1741037890, model=model, choices=list(choices))
for choices in frames
]
),
model=model,
custom_llm_provider="openai",
logging_obj=logging_obj,
stream_options=None,
)
routed: Final = await open_attempt_stream(stream_attempt, provider_stream)
assert isinstance(routed, CustomStreamWrapper)
return routed
async def _router_stream_holding_a_length_finish(
self, recorder: _RecordingSuccessLogger, stream_attempt: StreamAttempt
) -> CustomStreamWrapper:
return await self._router_stream(
recorder,
[
[
StreamingChoices(
index=0, delta=Delta(role="assistant", content="partial answer"), finish_reason="length"
)
]
],
stream_attempt,
)
async def _assert_disconnect_bills_length(
self, recorder: _RecordingSuccessLogger, logging_obj: LiteLLMLoggingObj, streamed: object
) -> None:
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert [choice.finish_reason for choice in partial_response.choices] == ["length"]
assert partial_response.usage.completion_tokens > 0
@pytest.mark.asyncio
async def test_router_chat_stream_keeps_the_finish_reason_held_on_the_last_content_frame(
self, stream_attempt: StreamAttempt
):
recorder: Final = _RecordingSuccessLogger()
routed: Final = await self._router_stream_holding_a_length_finish(recorder, stream_attempt)
delivered: Final = await routed.__aiter__().__anext__()
assert (delivered.choices[0].delta.content, delivered.choices[0].finish_reason) == ("partial answer", None)
await self._assert_disconnect_bills_length(recorder, routed.logging_obj, routed)
@pytest.mark.parametrize("through_response_cache", [False, True], ids=["uncached", "response-cache-writer"])
@pytest.mark.asyncio
async def test_anthropic_messages_stream_keeps_the_finish_reason_held_on_the_last_content_frame(
self, through_response_cache: bool, stream_attempt: StreamAttempt
):
from litellm.caching.caching_handler import LLMCachingHandler
from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream
from litellm.llms.anthropic.messages.adapters.transformation import AnthropicAdapter
from litellm.llms.anthropic.messages.native import handler
from litellm.llms.anthropic.messages.native.response_cache import AnthropicMessagesStreamCacheWriter
from litellm.router_rust.invoke.streaming import FallbackAwareAnthropicMessagesStream
recorder: Final = _RecordingSuccessLogger()
routed: Final = await self._router_stream_holding_a_length_finish(recorder, stream_attempt)
adapted: Final = AnthropicAdapter().translate_completion_output_params_streaming(
routed, model="gpt-4o", is_async=True, litellm_logging_obj=routed.logging_obj
)
assert isinstance(adapted, AnthropicSSEStream)
source_iterator: Final = (
AnthropicMessagesStreamCacheWriter(
stream=adapted,
caching_handler=LLMCachingHandler(
original_function=handler.anthropic_messages,
request_kwargs={},
start_time=datetime.datetime(2025, 3, 3, 21, 38, 10),
),
)
if through_response_cache
else adapted
)
async for frame in source_iterator:
if isinstance(frame, bytes) and b"partial answer" in frame:
break
async def _sse_frames() -> AsyncGenerator[bytes, None]:
yield b"event: message_start\n\n"
await self._assert_disconnect_bills_length(
recorder,
routed.logging_obj,
prepare_response_for_header_attachment(
FallbackAwareAnthropicMessagesStream(_sse_frames(), source_iterator, stream_attempt)
),
)
@pytest.mark.asyncio
async def test_disconnect_bills_a_partial_text_completion_without_inventing_a_finish_reason(self):
from litellm.types.utils import TextChoices, TextCompletionResponse
model: Final = "gpt-3.5-turbo-instruct"
messages: Final = [{"role": "user", "content": "Say something long"}]
recorder: Final = _RecordingSuccessLogger()
logging_obj: Final = self._recorded_logging_obj(
model, messages, "text-completion-disconnect-call", recorder, call_type="atext_completion"
)
collected_stream: Final = SimpleNamespace(
chunks=[
TextCompletionResponse(
id="cmpl-partial", created=1741037890, model=model, choices=[TextChoices(index=0, text=text)]
)
for text in ("partial ", "answer")
],
messages=messages,
model=model,
)
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
{"litellm_logging_obj": logging_obj}, collected_stream
)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert [(choice.text, choice.finish_reason) for choice in partial_response.choices] == [
("partial answer", None)
]
assert partial_response.usage.completion_tokens == litellm.token_counter(
model=model, text="partial answer", count_response_tokens=True
)
@staticmethod
def _content_choice(index: int, text: str, finish_reason: str | None = None) -> StreamingChoices:
return StreamingChoices(index=index, delta=Delta(role="assistant", content=text), finish_reason=finish_reason)
@pytest.mark.parametrize(
("frames", "expected"),
[
pytest.param(
(((0, "zero ", None),), ((1, "one ", None),), ((0, "done", "length"),)),
{0: "length", 1: "finish_reason_unspecified"},
id="choice-0-finished",
),
pytest.param(
(((0, "zero ", None),), ((1, "one ", None),), ((1, "done", "length"),)),
{0: "finish_reason_unspecified", 1: "length"},
id="choice-1-finished",
),
pytest.param(
(((1, "one", "stop"),), ((0, "zero", "length"),)),
{0: "length", 1: "stop"},
id="both-finished-out-of-order",
),
pytest.param(
(((2, "two", "length"),), ((0, "zero", "stop"),)),
{0: "stop", 2: "length"},
id="sparse-indexes",
),
pytest.param(
(((2, "partial answer", "length"),),),
{0: "length"},
id="single-choice-nonzero-index",
),
pytest.param(
(((0, "zero", "length"), (1, "one", "stop")),),
{0: "length", 1: "stop"},
id="one-frame-two-finishes",
),
],
)
@pytest.mark.asyncio
async def test_router_stream_keeps_each_choice_finish_reason_held_on_its_content_frame(
self,
frames: tuple[tuple[tuple[int, str, str | None], ...], ...],
expected: Mapping[int, str],
stream_attempt: StreamAttempt,
):
recorder: Final = _RecordingSuccessLogger()
routed: Final = await self._router_stream(
recorder,
[tuple(self._content_choice(*choice) for choice in frame) for frame in frames],
stream_attempt,
)
stream_iter: Final = routed.__aiter__()
delivered: Final = [await stream_iter.__anext__() for _ in frames]
delivered_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in delivered))
assert [choice.finish_reason for choice in delivered_choices] == [None] * sum(map(len, frames))
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
{"litellm_logging_obj": routed.logging_obj}, routed
)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert {choice.index: choice.finish_reason for choice in partial_response.choices} == expected
assert partial_response.usage.completion_tokens > 0
@pytest.mark.parametrize(
("frames", "expected"),
[
pytest.param(
(((0, "zero ", None),), ((1, "one ", None),), ((1, "", "length"),)),
{0: "finish_reason_unspecified", 1: "length"},
id="choice-1-terminal-frame",
),
pytest.param(
(((0, "zero ", None), (1, "one ", None)), ((0, "", "length"), (1, "", "stop"))),
{0: "length", 1: "stop"},
id="one-terminal-frame-two-finishes",
),
],
)
@pytest.mark.asyncio
async def test_disconnect_after_a_terminal_frame_bills_each_choice_its_own_finish_reason(
self,
frames: tuple[tuple[tuple[int, str, str | None], ...], ...],
expected: Mapping[int, str],
stream_attempt: StreamAttempt,
):
recorder: Final = _RecordingSuccessLogger()
routed: Final = await self._router_stream(
recorder,
[tuple(self._content_choice(*choice) for choice in frame) for frame in frames],
stream_attempt,
)
stream_iter: Final = routed.__aiter__()
for _ in frames:
await stream_iter.__anext__()
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
{"litellm_logging_obj": routed.logging_obj}, routed
)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert {choice.index: choice.finish_reason for choice in partial_response.choices} == expected
assert partial_response.usage.completion_tokens > 0
@pytest.mark.parametrize(
("candidate", "expected"),
[
pytest.param(
{"content": {"parts": [{"text": "partial answer"}], "role": "model"}, "index": 0},
"finish_reason_unspecified",
id="no-finish-reason",
),
pytest.param(
{
"content": {"parts": [{"text": "partial answer"}], "role": "model"},
"finishReason": "FINISH_REASON_UNSPECIFIED",
},
"finish_reason_unspecified",
id="finish-reason-unspecified-sentinel",
),
pytest.param(
{"content": {"parts": [{"text": "partial answer"}], "role": "model"}, "finishReason": "MAX_TOKENS"},
"length",
id="max-tokens",
),
pytest.param(
{"content": {"parts": [{"text": "full answer"}], "role": "model"}, "finishReason": "STOP"},
"stop",
id="stop",
),
pytest.param(
{
"content": {"parts": [{"functionCall": {"name": "lookup", "args": {"q": "fox"}}}], "role": "model"},
"finishReason": "STOP",
},
"tool_calls",
id="stop-after-a-tool-call",
),
],
)
@pytest.mark.asyncio
async def test_gemini_stream_bills_only_the_finish_reason_the_candidate_sent(
self, candidate: Mapping[str, object], expected: str, stream_attempt: StreamAttempt
):
partial_response: Final = await self._billed_after_the_first_gemini_frame(
stream_attempt, {"candidates": [candidate]}
)
assert [choice.finish_reason for choice in partial_response.choices] == [expected]
assert partial_response.usage.completion_tokens > 0
@pytest.mark.asyncio
async def test_gemini_frame_that_plans_a_continuation_hop_is_not_billed_as_finished(
self, stream_attempt: StreamAttempt
):
from litellm.types.llms.vertex_ai import GenerationConfig, RequestBody
async def _hop_must_not_open(request_body: RequestBody) -> httpx.Response:
pytest.fail(f"the client disconnected before the hop, yet it opened with {request_body}")
partial_response: Final = await self._billed_after_the_first_gemini_frame(
stream_attempt,
{
"candidates": [
{
"content": {"role": "model", "parts": [{"text": "hello"}]},
"finishReason": "CONTINUATION",
"continuationToken": "dG9rLTE=",
}
],
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 30, "totalTokenCount": 40},
},
AsyncStreamResume(
request_body=RequestBody(
contents=[{"role": "user", "parts": [{"text": "Prove it"}]}],
generationConfig=GenerationConfig(max_output_tokens=100),
),
open_hop=_hop_must_not_open,
),
)
assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"]
assert partial_response.usage.completion_tokens > 0
@pytest.mark.parametrize("evidence", ["length", ((0, None),), ((0, "length", "stop"),)])
@pytest.mark.asyncio
async def test_unreadable_provider_finish_evidence_bills_unspecified_instead_of_the_synthesized_stop(
self, evidence: object, stream_attempt: StreamAttempt
):
from litellm.constants import PROVIDER_FINISH_REASONS_HIDDEN_PARAM
from litellm.litellm_core_utils.hidden_params import set_hidden_param
frame: Final = ModelResponseStream(
id="chatcmpl-unreadable",
created=1741037890,
model="gemini-3.8-flash",
choices=[self._content_choice(0, "partial answer", "stop")],
)
set_hidden_param(frame, "is_finished", False)
set_hidden_param(frame, PROVIDER_FINISH_REASONS_HIDDEN_PARAM, evidence)
partial_response: Final = await self._billed_after_the_first_frame(
stream_attempt, lambda _: ModelResponseListIterator(model_responses=[frame])
)
assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"]
async def _billed_after_the_first_gemini_frame(
self, stream_attempt: StreamAttempt, frame: Mapping[str, object], resume: AsyncStreamResume | None = None
) -> litellm.ModelResponse:
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ModelResponseIterator
async def _gemini_sse() -> AsyncGenerator[str, None]:
yield f"data: {json.dumps(frame)}"
return await self._billed_after_the_first_frame(
stream_attempt,
lambda logging_obj: ModelResponseIterator(
streaming_response=_gemini_sse(), sync_stream=False, logging_obj=logging_obj, resume=resume
),
)
async def _billed_after_the_first_frame(
self, stream_attempt: StreamAttempt, completion_stream: Callable[[LiteLLMLoggingObj], object]
) -> litellm.ModelResponse:
model: Final = "gemini-3.8-flash"
recorder: Final = _RecordingSuccessLogger()
logging_obj: Final = self._recorded_logging_obj(
model, [{"role": "user", "content": "hi"}], "first-frame-disconnect-call", recorder
)
provider_stream: Final = CustomStreamWrapper(
completion_stream=completion_stream(logging_obj),
model=model,
custom_llm_provider="gemini",
logging_obj=logging_obj,
stream_options=None,
)
routed: Final = await open_attempt_stream(stream_attempt, provider_stream)
assert isinstance(routed, CustomStreamWrapper)
delivered: Final = await routed.__aiter__().__anext__()
assert delivered.choices[0].finish_reason is None
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, routed)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
return recorder.success_events[0]["response_obj"]
@pytest.mark.asyncio
async def test_disconnect_uses_a_finish_reason_received_without_a_choice_index_for_a_single_choice(self):
model: Final = "gpt-4o"
messages: Final = [{"role": "user", "content": "hi"}]
recorder: Final = _RecordingSuccessLogger()
logging_obj: Final = self._recorded_logging_obj(model, messages, "unindexed-finish-call", recorder)
collected_stream: Final = SimpleNamespace(
chunks=[
ModelResponseStream(
id="chatcmpl-unindexed",
created=1741037890,
model=model,
choices=[self._content_choice(0, "partial answer")],
)
],
messages=messages,
model=model,
finish_evidence=FinishEvidence(by_choice=MappingProxyType({}), unindexed="length"),
)
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
{"litellm_logging_obj": logging_obj}, collected_stream
)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
partial_response: Final = recorder.success_events[0]["response_obj"]
assert [choice.finish_reason for choice in partial_response.choices] == ["length"]
@pytest.mark.asyncio
async def test_disconnect_marks_only_the_unfinished_choice_unspecified(self):
model: Final = "gpt-4o"
messages: Final = [{"role": "user", "content": "look both up"}]
recorder: Final = _RecordingSuccessLogger()
logging_obj: Final = self._recorded_logging_obj(model, messages, "two-choice-disconnect-call", recorder)
def tool_call_delta(call_id: str) -> Delta:
return Delta(
role="assistant",
tool_calls=[
{"id": call_id, "type": "function", "index": 0, "function": {"name": "lookup", "arguments": "{}"}}
],
)
collected_stream: Final = SimpleNamespace(
chunks=[
ModelResponseStream(id="chatcmpl-two", created=1741037890, model=model, choices=choices)
for choices in (
[
StreamingChoices(index=0, delta=tool_call_delta("call_a")),
StreamingChoices(index=1, delta=tool_call_delta("call_b")),
],
[StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")],
)
],
messages=messages,
model=model,
finish_evidence=FinishEvidence(by_choice=MappingProxyType({}), unindexed="tool_calls"),
)
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
{"litellm_logging_obj": logging_obj}, collected_stream
)
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
assert billed is True
assert len(recorder.success_events) == 1
partial_response: Final = recorder.success_events[0]["response_obj"]
assert {choice.index: choice.finish_reason for choice in partial_response.choices} == {
0: "tool_calls",
1: "finish_reason_unspecified",
}
assert [len(choice.message.tool_calls) for choice in partial_response.choices] == [1, 1]
assert partial_response.usage.total_tokens > 0
@pytest.mark.asyncio
async def test_completed_stream_does_not_double_bill_on_late_disconnect(self):
recorder = _RecordingSuccessLogger()