mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(proxy): preserve terminal evidence in disconnect billing (#52088)
* fix(proxy): preserve terminal evidence in disconnect billing * fix(proxy): retain held finish reasons through stream wrappers * refactor(proxy): use immutable finish evidence results * fix(proxy): type disconnect evidence boundaries and cover wrappers * fix(proxy): preserve object type before chunk narrowing * fix(proxy): retain disconnect finish evidence per choice * fix(proxy): preserve genuine Gemini finish evidence * fix(proxy): use typed finish evidence storage * test(proxy): exercise real router stream attempts * fix(proxy): keep unspecified Gemini evidence unknown --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
parent
4af6fb70d8
commit
e7c89b0dd4
10 changed files with 800 additions and 3 deletions
|
|
@ -708,6 +708,7 @@ STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping
|
|||
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
|
||||
STREAM_FINISH_CHOICE_FIELDS: Final = ("stop_reason",)
|
||||
STREAM_CHUNK_CHOICE_FIELDS: Final = ("token_ids",)
|
||||
PROVIDER_FINISH_REASONS_HIDDEN_PARAM: Final = "provider_finish_reasons"
|
||||
VLLM_CHOICE_FIELDS: Final = STREAM_FINISH_CHOICE_FIELDS + STREAM_CHUNK_CHOICE_FIELDS
|
||||
### SPEND TRACKING ###
|
||||
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import math
|
|||
import re
|
||||
import sys
|
||||
from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
|
||||
|
||||
|
|
@ -191,6 +192,23 @@ def attribute_or_none(source: object, name: str) -> object:
|
|||
_Shape: Final = TypeVar("_Shape")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FinishEvidence:
|
||||
"""Finish reasons a stream received from the provider. ``unindexed`` is a received reason whose
|
||||
choice the stream could not tell, so it is only evidence for a single-choice response."""
|
||||
|
||||
by_choice: Mapping[int, str]
|
||||
unindexed: str | None
|
||||
|
||||
|
||||
NO_FINISH_EVIDENCE: Final = FinishEvidence(by_choice=MappingProxyType({}), unindexed=None)
|
||||
|
||||
|
||||
def finish_evidence_of(stream: object) -> FinishEvidence:
|
||||
evidence: Final[object] = getattr(stream, "finish_evidence", None)
|
||||
return evidence if isinstance(evidence, FinishEvidence) else NO_FINISH_EVIDENCE
|
||||
|
||||
|
||||
def has_attributes(
|
||||
value: object,
|
||||
shape: type[_Shape],
|
||||
|
|
|
|||
|
|
@ -1124,6 +1124,18 @@ class ChunkProcessor:
|
|||
return chunk.get("choices", ())
|
||||
return getattr(chunk, "choices", ())
|
||||
|
||||
@staticmethod
|
||||
def choice_indices_with_observed_finish_reason(chunks: Iterable[object]) -> frozenset[int]:
|
||||
choices: Final = tuple(chain.from_iterable(object_items(_field_of(chunk, "choices")) for chunk in chunks))
|
||||
finished: Final = frozenset(
|
||||
_stream_choice_index(choice)
|
||||
for choice in choices
|
||||
if ChunkProcessor._finish_reason_of_choice(choice) is not None
|
||||
)
|
||||
if len(frozenset(map(_stream_choice_index, choices))) > 1:
|
||||
return finished
|
||||
return frozenset({0}) if finished else frozenset()
|
||||
|
||||
@staticmethod
|
||||
def _saw_finish_reason(chunks: Sequence["_UsageBearingChunk | ModelResponse"]) -> bool:
|
||||
return any(
|
||||
|
|
|
|||
|
|
@ -23,10 +23,15 @@ from typing_extensions import NotRequired, TypedDict
|
|||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_FIELDS
|
||||
from litellm.constants import (
|
||||
PROVIDER_FINISH_REASONS_HIDDEN_PARAM,
|
||||
STREAM_CHUNK_CHOICE_FIELDS,
|
||||
STREAM_FINISH_CHOICE_FIELDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
UPSTREAM_USAGE_COST,
|
||||
FinishEvidence,
|
||||
as_str_mapping,
|
||||
attribute_or_none,
|
||||
chunk_usage,
|
||||
|
|
@ -171,6 +176,35 @@ def _prepend_upstream_choice_extras(
|
|||
|
||||
_NO_UPSTREAM_CHOICE_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_NO_CHOICE_FIELDS_BY_INDEX: Final[Mapping[int, Mapping[str, object]]] = MappingProxyType({})
|
||||
_NO_FINISH_REASONS_BY_INDEX: Final[Mapping[int, str]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _indexed_finish_reason(choice: object) -> tuple[int, str] | None:
|
||||
reason: Final[object] = getattr(choice, "finish_reason", None)
|
||||
index: Final[object] = getattr(choice, "index", None)
|
||||
if not isinstance(reason, str) or not reason or not isinstance(index, int):
|
||||
return None
|
||||
return index, reason
|
||||
|
||||
|
||||
_PROVIDER_FINISH_REASONS: Final = TypeAdapter(tuple[tuple[int, str], ...])
|
||||
|
||||
|
||||
def _provider_finish_reasons(evidence: object) -> Mapping[int, str]:
|
||||
if evidence is None:
|
||||
return _NO_FINISH_REASONS_BY_INDEX
|
||||
try:
|
||||
return MappingProxyType(dict(_PROVIDER_FINISH_REASONS.validate_python(evidence, strict=True)))
|
||||
except ValidationError:
|
||||
return _NO_FINISH_REASONS_BY_INDEX
|
||||
|
||||
|
||||
def _finish_reasons_by_index(upstream_chunk: object) -> Mapping[int, str]:
|
||||
hidden_params: Final = get_hidden_params(upstream_chunk)
|
||||
if hidden_params is not None and hidden_params.get("is_finished") is False:
|
||||
return _provider_finish_reasons(hidden_params.get(PROVIDER_FINISH_REASONS_HIDDEN_PARAM))
|
||||
choices: Final[object] = getattr(upstream_chunk, "choices", None)
|
||||
return MappingProxyType(dict(filter(None, map(_indexed_finish_reason, object_items(choices)))))
|
||||
|
||||
|
||||
def _upstream_first_choice(upstream_chunk: object) -> BaseModel | None:
|
||||
|
|
@ -509,6 +543,7 @@ class CustomStreamWrapper:
|
|||
self._upstream_provider: str | None = None
|
||||
self.received_finish_reason: str | None = None
|
||||
self.received_finish_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
|
||||
self.received_finish_reasons: Mapping[int, str] = _NO_FINISH_REASONS_BY_INDEX
|
||||
self.carried_chunk_choice_fields: Mapping[int, Mapping[str, object]] = _NO_CHOICE_FIELDS_BY_INDEX
|
||||
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
|
||||
self.received_any_finish_reason = False # a later chunk can reset intermittent_finish_reason
|
||||
|
|
@ -1729,6 +1764,7 @@ class CustomStreamWrapper:
|
|||
response_obj = self.handle_openai_text_completion_chunk(chunk)
|
||||
completion_obj["content"] = response_obj["text"]
|
||||
print_verbose(f"completion obj content: {completion_obj['content']}")
|
||||
self._record_finish_reasons(chunk)
|
||||
if response_obj["is_finished"]:
|
||||
self._record_finish_choice_fields(chunk)
|
||||
if response_obj["is_finished"] and response_obj["sibling_finish_reason"] is None:
|
||||
|
|
@ -1805,6 +1841,7 @@ class CustomStreamWrapper:
|
|||
self.response_id = cached_chunk.id
|
||||
if hasattr(cached_chunk, "system_fingerprint"):
|
||||
self.system_fingerprint = cached_chunk.system_fingerprint
|
||||
self._record_finish_reasons(cached_chunk)
|
||||
if response_obj["is_finished"]:
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
self._record_finish_choice_fields(cached_chunk)
|
||||
|
|
@ -1840,6 +1877,7 @@ class CustomStreamWrapper:
|
|||
)
|
||||
self.received_finish_reason = response_obj["finish_reason"]
|
||||
self._record_finish_choice_fields(chunk)
|
||||
self._record_finish_reasons(chunk)
|
||||
if response_obj.get("original_chunk", None) is not None:
|
||||
if hasattr(response_obj["original_chunk"], "id"):
|
||||
model_response = self.set_model_id(response_obj["original_chunk"].id, model_response)
|
||||
|
|
@ -2190,6 +2228,18 @@ class CustomStreamWrapper:
|
|||
if self.logging_obj.is_sync_litellm_request(litellm_params):
|
||||
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return FinishEvidence(
|
||||
by_choice=self.received_finish_reasons,
|
||||
unindexed=None if self.received_finish_reasons else self.received_finish_reason,
|
||||
)
|
||||
|
||||
def _record_finish_reasons(self, upstream_chunk: object) -> None:
|
||||
received: Final = _finish_reasons_by_index(upstream_chunk)
|
||||
if received:
|
||||
self.received_finish_reasons = MappingProxyType({**self.received_finish_reasons, **received})
|
||||
|
||||
def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str:
|
||||
if finish_reason == "stop" and self.tool_call:
|
||||
return "tool_calls"
|
||||
|
|
|
|||
|
|
@ -27,7 +27,12 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
FinishEvidence,
|
||||
as_str_mapping,
|
||||
attribute_items,
|
||||
finish_evidence_of,
|
||||
)
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_params
|
||||
from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
|
||||
|
|
@ -283,6 +288,10 @@ class _CombinedChunkSplitter:
|
|||
"list[ModelResponseStream] | None", getattr(self._stream, "chunks", None)
|
||||
)
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return finish_evidence_of(self._stream)
|
||||
|
||||
@property
|
||||
def messages(self) -> "list[AllMessageValues] | None":
|
||||
return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream
|
||||
|
|
@ -711,6 +720,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
|
|||
"list[ModelResponseStream] | None", getattr(self.completion_stream, "chunks", None)
|
||||
)
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return self._combined_chunk_splitter.finish_evidence
|
||||
|
||||
@property
|
||||
def messages(self) -> "list[AllMessageValues] | None":
|
||||
return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream
|
||||
|
|
@ -1645,6 +1658,10 @@ class AnthropicSSEStream(AsyncIterator[bytes]):
|
|||
def chunks(self) -> "list[ModelResponseStream] | None":
|
||||
return self._anthropic_wrapper.chunks
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return self._anthropic_wrapper.finish_evidence
|
||||
|
||||
@property
|
||||
def messages(self) -> "list[AllMessageValues] | None":
|
||||
return self._anthropic_wrapper.messages
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
|
||||
from litellm.litellm_core_utils.core_helpers import FinishEvidence, finish_evidence_of
|
||||
from litellm.llms.anthropic.messages.native.streaming_iterator import (
|
||||
AnthropicMessagesStreamingResponse,
|
||||
BaseAnthropicMessagesStreamingIterator,
|
||||
|
|
@ -59,6 +60,10 @@ class AnthropicMessagesStreamCacheWriter:
|
|||
"list[ModelResponseStream] | None", getattr(self.stream, "chunks", None)
|
||||
)
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return finish_evidence_of(self.stream)
|
||||
|
||||
@property
|
||||
def messages(self) -> "list[AllMessageValues] | None":
|
||||
return cast( # cast-ok: messages is a list of AllMessageValues on the inner stream
|
||||
|
|
|
|||
|
|
@ -29,10 +29,12 @@ from litellm.constants import (
|
|||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH,
|
||||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE,
|
||||
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
|
||||
PROVIDER_FINISH_REASONS_HIDDEN_PARAM,
|
||||
)
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_param
|
||||
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
|
||||
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
|
||||
|
|
@ -4213,6 +4215,21 @@ def _transform_long_decode_hops(
|
|||
return merged if marked is merged_usage else merged.model_copy(update={"usage": marked})
|
||||
|
||||
|
||||
def _provider_finish_reasons(
|
||||
candidates: Sequence[Candidates], choices: Sequence["StreamingChoices"]
|
||||
) -> tuple[tuple[int, str], ...]:
|
||||
"""Every streamed choice carries a finish_reason, "stop" when the candidate sent none or Google's
|
||||
FINISH_REASON_UNSPECIFIED, so only a candidate that named how it ended is evidence of it."""
|
||||
finished: Final = frozenset(
|
||||
candidate.get("index", position)
|
||||
for position, candidate in enumerate(candidates)
|
||||
if (candidate.get("finishReason") or "FINISH_REASON_UNSPECIFIED") != "FINISH_REASON_UNSPECIFIED"
|
||||
)
|
||||
return tuple(
|
||||
(choice.index, choice.finish_reason) for choice in choices if choice.index in finished and choice.finish_reason
|
||||
)
|
||||
|
||||
|
||||
class ModelResponseIterator:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -4459,6 +4476,9 @@ class ModelResponseIterator:
|
|||
setattr(model_response, "usage", usage)
|
||||
self._plan_resume(processed_chunk, model_response)
|
||||
|
||||
provider_finish_reasons: Final = _provider_finish_reasons(_candidates or (), model_response.choices)
|
||||
if provider_finish_reasons:
|
||||
set_hidden_param(model_response, PROVIDER_FINISH_REASONS_HIDDEN_PARAM, provider_finish_reasons)
|
||||
model_response.hidden_params["is_finished"] = False
|
||||
return model_response
|
||||
|
||||
|
|
|
|||
|
|
@ -87,11 +87,14 @@ from litellm.litellm_core_utils.cloud_storage_security import (
|
|||
s3_media_uris,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
FinishEvidence,
|
||||
finish_evidence_of,
|
||||
finite_cost,
|
||||
get_or_create_metadata_bucket,
|
||||
has_attributes,
|
||||
independent_snapshot,
|
||||
is_expected_client_error,
|
||||
map_finish_reason,
|
||||
proxy_request_headers,
|
||||
redact_nested_match_and_regex_keys,
|
||||
require_attributes,
|
||||
|
|
@ -108,6 +111,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
|
||||
guardrail_entries,
|
||||
guardrail_information_cost,
|
||||
object_items,
|
||||
object_mapping,
|
||||
)
|
||||
from litellm.litellm_core_utils.llm_response_utils.get_headers import (
|
||||
|
|
@ -119,6 +123,7 @@ from litellm.litellm_core_utils.served_output_texts import (
|
|||
record_served_output_texts,
|
||||
served_output_texts,
|
||||
)
|
||||
from litellm.litellm_core_utils.streaming_chunk_builder_utils import ChunkProcessor
|
||||
from litellm.litellm_core_utils.streaming_handler import (
|
||||
backfill_missing_cache_usage_fields,
|
||||
)
|
||||
|
|
@ -365,6 +370,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.utils import (
|
||||
Choices,
|
||||
LLMResponseTypes,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
|
|
@ -588,6 +594,7 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
|
|||
)
|
||||
return await _dispatch_disconnect_billing_event(logging_obj, partial_responses_event)
|
||||
chunks: Final[object] = getattr(response, "chunks", None)
|
||||
chunk_source: Final[object] = chunks
|
||||
if not isinstance(chunks, list) or not chunks:
|
||||
return False
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -616,9 +623,48 @@ async def _bill_partial_streamed_spend_on_disconnect(request_data: dict, respons
|
|||
partial_usage: Final = getattr(partial_response, "usage", None)
|
||||
if isinstance(partial_usage, Usage):
|
||||
backfill_missing_cache_usage_fields(partial_usage)
|
||||
if isinstance(partial_response, ModelResponse):
|
||||
partial_response.choices = list(
|
||||
_choices_without_unobserved_finish_reasons(
|
||||
partial_response.choices,
|
||||
ChunkProcessor.choice_indices_with_observed_finish_reason(object_items(chunk_source)),
|
||||
finish_evidence_of(response),
|
||||
)
|
||||
)
|
||||
return await _dispatch_disconnect_billing_event(logging_obj, partial_response)
|
||||
|
||||
|
||||
def _choices_without_unobserved_finish_reasons(
|
||||
choices: Sequence[Choices], observed_indices: frozenset[int], evidence: FinishEvidence
|
||||
) -> tuple[Choices, ...]:
|
||||
"""Finish reasons the stream recorded per choice off the provider's frames decide every choice: the
|
||||
terminal chunk the stream emits can carry another choice's index, so the assembled chunks cannot."""
|
||||
if evidence.by_choice:
|
||||
return tuple(
|
||||
_with_received_finish_reason(choice, _indexed_finish_reason(choice.index, len(choices), evidence.by_choice))
|
||||
for choice in choices
|
||||
)
|
||||
single_choice_reason: Final = evidence.unindexed if len(choices) == 1 else None
|
||||
return tuple(
|
||||
choice if choice.index in observed_indices else _with_received_finish_reason(choice, single_choice_reason)
|
||||
for choice in choices
|
||||
)
|
||||
|
||||
|
||||
def _indexed_finish_reason(choice_index: int, choice_count: int, by_choice: Mapping[int, str]) -> str | None:
|
||||
if choice_count == 1 and len(by_choice) == 1:
|
||||
return next(iter(by_choice.values()))
|
||||
return by_choice.get(choice_index)
|
||||
|
||||
|
||||
def _with_received_finish_reason(choice: Choices, received_finish_reason: str | None) -> Choices:
|
||||
if received_finish_reason is None:
|
||||
return choice.model_copy(update={"finish_reason": "finish_reason_unspecified"})
|
||||
finish_reason: Final = map_finish_reason(received_finish_reason)
|
||||
ends_a_tool_call_turn: Final = finish_reason == "stop" and bool(choice.message.tool_calls)
|
||||
return choice.model_copy(update={"finish_reason": "tool_calls" if ends_a_tool_call_turn else finish_reason})
|
||||
|
||||
|
||||
async def _bill_in_flight_round_on_disconnect(request_data: Mapping[str, object]) -> None:
|
||||
bill_in_flight_round: Final = in_flight_round_biller(request_data.get("litellm_logging_obj"))
|
||||
if bill_in_flight_round is None:
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
|||
get_hidden_params_dict,
|
||||
replace_complexity_router_headers,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, without
|
||||
from litellm.litellm_core_utils.core_helpers import FinishEvidence, as_str_mapping, finish_evidence_of, without
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params, set_hidden_params
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.custom_httpx.stream_close import aclose_if_supported
|
||||
|
|
@ -862,6 +862,11 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper):
|
|||
fallback takes over."""
|
||||
|
||||
fallback_headers_adopted: bool = False
|
||||
_source_stream: CustomStreamWrapper
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return self._source_stream.finish_evidence
|
||||
|
||||
def adopt_fallback_response_headers(
|
||||
self,
|
||||
|
|
@ -911,6 +916,7 @@ def _chat_wrapper_init(
|
|||
class _AsyncChatStream(FallbackAwareStreamWrapper):
|
||||
def __init__(self, generator: AsyncGenerator[object, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it
|
||||
_chat_wrapper_init(self, generator, model_response)
|
||||
self._source_stream = model_response
|
||||
self._async_generator: Final = generator
|
||||
|
||||
def __aiter__(self) -> "_AsyncChatStream": # pyright: ignore[reportIncompatibleMethodOverride] # iterates the generator
|
||||
|
|
@ -923,6 +929,7 @@ class _AsyncChatStream(FallbackAwareStreamWrapper):
|
|||
class _SyncChatStream(FallbackAwareStreamWrapper):
|
||||
def __init__(self, generator: Generator[object, None, None], model_response: CustomStreamWrapper) -> None: # pyright: ignore[reportMissingSuperCall] # _chat_wrapper_init calls it
|
||||
_chat_wrapper_init(self, generator, model_response)
|
||||
self._source_stream = model_response
|
||||
self.chunks = [] # the sync twin keeps no inner chunk list
|
||||
hidden_params: Final = get_hidden_params(model_response)
|
||||
if hidden_params is not None:
|
||||
|
|
@ -1133,6 +1140,10 @@ class FallbackAwareAnthropicMessagesStream:
|
|||
def chunks(self) -> object:
|
||||
return getattr(self._source_iterator, "chunks", None)
|
||||
|
||||
@property
|
||||
def finish_evidence(self) -> FinishEvidence:
|
||||
return finish_evidence_of(self._source_iterator)
|
||||
|
||||
@property
|
||||
def messages(self) -> object:
|
||||
return getattr(self._source_iterator, "messages", None)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import time
|
|||
import weakref
|
||||
from base64 import urlsafe_b64encode
|
||||
from collections.abc import AsyncIterable, Awaitable, Coroutine, Mapping
|
||||
from itertools import chain
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence, TypeAlias, TypeVar
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -43,11 +44,13 @@ from litellm.litellm_core_utils.bug_report import (
|
|||
bug_report_notice,
|
||||
build_bug_report,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import FinishEvidence
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.total_deadline import current_total_deadline
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.vertex_ai.gemini.long_decode import AsyncStreamResume
|
||||
from litellm.proxy._types import Litellm_EntityType, LitellmUserRoles, ProxyErrorTypes, ProxyException
|
||||
from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth
|
||||
from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck
|
||||
|
|
@ -95,6 +98,10 @@ from litellm.proxy.hooks.model_max_budget_limiter import (
|
|||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.responses.mcp.parked_stream_dispatch import IN_FLIGHT_ROUND
|
||||
from litellm.router import Router
|
||||
from litellm.router_config import build_router_config
|
||||
from litellm.router_python_host import python_host
|
||||
from litellm.router_rust.core import RustCore
|
||||
from litellm.router_rust.invoke.streaming import StreamAttempt, open_attempt_stream
|
||||
from litellm.types.router import RouterRateLimitError, RouterRateLimitErrorBasic
|
||||
from litellm.types.utils import CachingDetails, Delta, ModelResponseStream, SpecialEnums, StreamingChoices
|
||||
from litellm.utils import ModelResponseListIterator
|
||||
|
|
@ -11095,6 +11102,616 @@ class TestStreamingClientDisconnectBilling:
|
|||
litellm.token_counter(model=model, text=text, count_response_tokens=True),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _recorded_logging_obj(
|
||||
model: str,
|
||||
messages: list[dict[str, str]],
|
||||
call_id: str,
|
||||
recorder: _RecordingSuccessLogger,
|
||||
call_type: str = "acompletion",
|
||||
) -> LiteLLMLoggingObj:
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model=model,
|
||||
messages=messages,
|
||||
stream=True,
|
||||
call_type=call_type,
|
||||
start_time=datetime.datetime(2025, 3, 3, 21, 38, 10),
|
||||
litellm_call_id=call_id,
|
||||
function_id=call_id,
|
||||
dynamic_async_success_callbacks=[recorder],
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": call_id}
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_before_the_finish_frame_records_an_unspecified_finish_reason(self):
|
||||
model: Final = "gpt-4o"
|
||||
messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}]
|
||||
text: Final = "The quick brown fox jumps over the lazy dog. " * 20
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
logging_obj: Final = self._recorded_logging_obj(model, messages, "unfinished-disconnect-call", recorder)
|
||||
response: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(
|
||||
model_responses=[
|
||||
ModelResponseStream(id="chatcmpl-unfinished", created=1741037890, model=model, choices=choices)
|
||||
for choices in (
|
||||
[StreamingChoices(index=0, delta=Delta(role="assistant", content=text))],
|
||||
[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")],
|
||||
)
|
||||
]
|
||||
),
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging_obj,
|
||||
stream_options=None,
|
||||
)
|
||||
first_chunk: Final = await response.__aiter__().__anext__()
|
||||
assert first_chunk.choices[0].finish_reason is None
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, response)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"]
|
||||
assert (partial_response.usage.prompt_tokens, partial_response.usage.completion_tokens) == (
|
||||
litellm.token_counter(model=model, messages=messages),
|
||||
litellm.token_counter(model=model, text=text, count_response_tokens=True),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_frames",
|
||||
[
|
||||
pytest.param(
|
||||
(
|
||||
[StreamingChoices(index=0, delta=Delta(role="assistant", content="partial answer"))],
|
||||
[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="length")],
|
||||
),
|
||||
id="finish-on-its-own-frame",
|
||||
),
|
||||
pytest.param(
|
||||
(
|
||||
[
|
||||
StreamingChoices(
|
||||
index=0, delta=Delta(role="assistant", content="partial answer"), finish_reason="length"
|
||||
)
|
||||
],
|
||||
),
|
||||
id="finish-on-the-content-frame",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_after_the_provider_finished_keeps_the_received_finish_reason(
|
||||
self, provider_frames: tuple[list[StreamingChoices], ...]
|
||||
):
|
||||
model: Final = "gpt-4o"
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
logging_obj: Final = self._recorded_logging_obj(
|
||||
model, [{"role": "user", "content": "hi"}], "finished-disconnect-call", recorder
|
||||
)
|
||||
response: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(
|
||||
model_responses=[
|
||||
ModelResponseStream(id="chatcmpl-finished", created=1741037890, model=model, choices=choices)
|
||||
for choices in provider_frames
|
||||
]
|
||||
),
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging_obj,
|
||||
stream_options=None,
|
||||
)
|
||||
stream_iter: Final = response.__aiter__()
|
||||
for _ in provider_frames:
|
||||
await stream_iter.__anext__()
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, response)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == ["length"]
|
||||
assert partial_response.usage.completion_tokens > 0
|
||||
|
||||
@pytest.fixture
|
||||
def stream_attempt(self) -> Iterator[StreamAttempt]:
|
||||
core: Final = RustCore.build(build_router_config(MappingProxyType({"model_list": []})), python_host)
|
||||
yield StreamAttempt(
|
||||
core=core,
|
||||
op="acompletion",
|
||||
request=MappingProxyType({}),
|
||||
kwargs=MappingProxyType({}),
|
||||
group="gpt-4o",
|
||||
order=None,
|
||||
depth=0,
|
||||
level=0,
|
||||
deployment_id=None,
|
||||
attempted_retries=0,
|
||||
)
|
||||
core.retire()
|
||||
|
||||
async def _router_stream(
|
||||
self,
|
||||
recorder: _RecordingSuccessLogger,
|
||||
frames: Sequence[Sequence[StreamingChoices]],
|
||||
stream_attempt: StreamAttempt,
|
||||
) -> CustomStreamWrapper:
|
||||
model: Final = "gpt-4o"
|
||||
logging_obj: Final = self._recorded_logging_obj(
|
||||
model, [{"role": "user", "content": "hi"}], "router-held-finish-call", recorder
|
||||
)
|
||||
provider_stream: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(
|
||||
model_responses=[
|
||||
ModelResponseStream(id="chatcmpl-held", created=1741037890, model=model, choices=list(choices))
|
||||
for choices in frames
|
||||
]
|
||||
),
|
||||
model=model,
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=logging_obj,
|
||||
stream_options=None,
|
||||
)
|
||||
routed: Final = await open_attempt_stream(stream_attempt, provider_stream)
|
||||
assert isinstance(routed, CustomStreamWrapper)
|
||||
return routed
|
||||
|
||||
async def _router_stream_holding_a_length_finish(
|
||||
self, recorder: _RecordingSuccessLogger, stream_attempt: StreamAttempt
|
||||
) -> CustomStreamWrapper:
|
||||
return await self._router_stream(
|
||||
recorder,
|
||||
[
|
||||
[
|
||||
StreamingChoices(
|
||||
index=0, delta=Delta(role="assistant", content="partial answer"), finish_reason="length"
|
||||
)
|
||||
]
|
||||
],
|
||||
stream_attempt,
|
||||
)
|
||||
|
||||
async def _assert_disconnect_bills_length(
|
||||
self, recorder: _RecordingSuccessLogger, logging_obj: LiteLLMLoggingObj, streamed: object
|
||||
) -> None:
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, streamed)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == ["length"]
|
||||
assert partial_response.usage.completion_tokens > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_chat_stream_keeps_the_finish_reason_held_on_the_last_content_frame(
|
||||
self, stream_attempt: StreamAttempt
|
||||
):
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
routed: Final = await self._router_stream_holding_a_length_finish(recorder, stream_attempt)
|
||||
|
||||
delivered: Final = await routed.__aiter__().__anext__()
|
||||
assert (delivered.choices[0].delta.content, delivered.choices[0].finish_reason) == ("partial answer", None)
|
||||
|
||||
await self._assert_disconnect_bills_length(recorder, routed.logging_obj, routed)
|
||||
|
||||
@pytest.mark.parametrize("through_response_cache", [False, True], ids=["uncached", "response-cache-writer"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_messages_stream_keeps_the_finish_reason_held_on_the_last_content_frame(
|
||||
self, through_response_cache: bool, stream_attempt: StreamAttempt
|
||||
):
|
||||
from litellm.caching.caching_handler import LLMCachingHandler
|
||||
from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicSSEStream
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import AnthropicAdapter
|
||||
from litellm.llms.anthropic.messages.native import handler
|
||||
from litellm.llms.anthropic.messages.native.response_cache import AnthropicMessagesStreamCacheWriter
|
||||
from litellm.router_rust.invoke.streaming import FallbackAwareAnthropicMessagesStream
|
||||
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
routed: Final = await self._router_stream_holding_a_length_finish(recorder, stream_attempt)
|
||||
adapted: Final = AnthropicAdapter().translate_completion_output_params_streaming(
|
||||
routed, model="gpt-4o", is_async=True, litellm_logging_obj=routed.logging_obj
|
||||
)
|
||||
assert isinstance(adapted, AnthropicSSEStream)
|
||||
source_iterator: Final = (
|
||||
AnthropicMessagesStreamCacheWriter(
|
||||
stream=adapted,
|
||||
caching_handler=LLMCachingHandler(
|
||||
original_function=handler.anthropic_messages,
|
||||
request_kwargs={},
|
||||
start_time=datetime.datetime(2025, 3, 3, 21, 38, 10),
|
||||
),
|
||||
)
|
||||
if through_response_cache
|
||||
else adapted
|
||||
)
|
||||
async for frame in source_iterator:
|
||||
if isinstance(frame, bytes) and b"partial answer" in frame:
|
||||
break
|
||||
|
||||
async def _sse_frames() -> AsyncGenerator[bytes, None]:
|
||||
yield b"event: message_start\n\n"
|
||||
|
||||
await self._assert_disconnect_bills_length(
|
||||
recorder,
|
||||
routed.logging_obj,
|
||||
prepare_response_for_header_attachment(
|
||||
FallbackAwareAnthropicMessagesStream(_sse_frames(), source_iterator, stream_attempt)
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_bills_a_partial_text_completion_without_inventing_a_finish_reason(self):
|
||||
from litellm.types.utils import TextChoices, TextCompletionResponse
|
||||
|
||||
model: Final = "gpt-3.5-turbo-instruct"
|
||||
messages: Final = [{"role": "user", "content": "Say something long"}]
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
logging_obj: Final = self._recorded_logging_obj(
|
||||
model, messages, "text-completion-disconnect-call", recorder, call_type="atext_completion"
|
||||
)
|
||||
collected_stream: Final = SimpleNamespace(
|
||||
chunks=[
|
||||
TextCompletionResponse(
|
||||
id="cmpl-partial", created=1741037890, model=model, choices=[TextChoices(index=0, text=text)]
|
||||
)
|
||||
for text in ("partial ", "answer")
|
||||
],
|
||||
messages=messages,
|
||||
model=model,
|
||||
)
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
|
||||
{"litellm_logging_obj": logging_obj}, collected_stream
|
||||
)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert [(choice.text, choice.finish_reason) for choice in partial_response.choices] == [
|
||||
("partial answer", None)
|
||||
]
|
||||
assert partial_response.usage.completion_tokens == litellm.token_counter(
|
||||
model=model, text="partial answer", count_response_tokens=True
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _content_choice(index: int, text: str, finish_reason: str | None = None) -> StreamingChoices:
|
||||
return StreamingChoices(index=index, delta=Delta(role="assistant", content=text), finish_reason=finish_reason)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("frames", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
(((0, "zero ", None),), ((1, "one ", None),), ((0, "done", "length"),)),
|
||||
{0: "length", 1: "finish_reason_unspecified"},
|
||||
id="choice-0-finished",
|
||||
),
|
||||
pytest.param(
|
||||
(((0, "zero ", None),), ((1, "one ", None),), ((1, "done", "length"),)),
|
||||
{0: "finish_reason_unspecified", 1: "length"},
|
||||
id="choice-1-finished",
|
||||
),
|
||||
pytest.param(
|
||||
(((1, "one", "stop"),), ((0, "zero", "length"),)),
|
||||
{0: "length", 1: "stop"},
|
||||
id="both-finished-out-of-order",
|
||||
),
|
||||
pytest.param(
|
||||
(((2, "two", "length"),), ((0, "zero", "stop"),)),
|
||||
{0: "stop", 2: "length"},
|
||||
id="sparse-indexes",
|
||||
),
|
||||
pytest.param(
|
||||
(((2, "partial answer", "length"),),),
|
||||
{0: "length"},
|
||||
id="single-choice-nonzero-index",
|
||||
),
|
||||
pytest.param(
|
||||
(((0, "zero", "length"), (1, "one", "stop")),),
|
||||
{0: "length", 1: "stop"},
|
||||
id="one-frame-two-finishes",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_stream_keeps_each_choice_finish_reason_held_on_its_content_frame(
|
||||
self,
|
||||
frames: tuple[tuple[tuple[int, str, str | None], ...], ...],
|
||||
expected: Mapping[int, str],
|
||||
stream_attempt: StreamAttempt,
|
||||
):
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
routed: Final = await self._router_stream(
|
||||
recorder,
|
||||
[tuple(self._content_choice(*choice) for choice in frame) for frame in frames],
|
||||
stream_attempt,
|
||||
)
|
||||
stream_iter: Final = routed.__aiter__()
|
||||
delivered: Final = [await stream_iter.__anext__() for _ in frames]
|
||||
delivered_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in delivered))
|
||||
assert [choice.finish_reason for choice in delivered_choices] == [None] * sum(map(len, frames))
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
|
||||
{"litellm_logging_obj": routed.logging_obj}, routed
|
||||
)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert {choice.index: choice.finish_reason for choice in partial_response.choices} == expected
|
||||
assert partial_response.usage.completion_tokens > 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("frames", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
(((0, "zero ", None),), ((1, "one ", None),), ((1, "", "length"),)),
|
||||
{0: "finish_reason_unspecified", 1: "length"},
|
||||
id="choice-1-terminal-frame",
|
||||
),
|
||||
pytest.param(
|
||||
(((0, "zero ", None), (1, "one ", None)), ((0, "", "length"), (1, "", "stop"))),
|
||||
{0: "length", 1: "stop"},
|
||||
id="one-terminal-frame-two-finishes",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_after_a_terminal_frame_bills_each_choice_its_own_finish_reason(
|
||||
self,
|
||||
frames: tuple[tuple[tuple[int, str, str | None], ...], ...],
|
||||
expected: Mapping[int, str],
|
||||
stream_attempt: StreamAttempt,
|
||||
):
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
routed: Final = await self._router_stream(
|
||||
recorder,
|
||||
[tuple(self._content_choice(*choice) for choice in frame) for frame in frames],
|
||||
stream_attempt,
|
||||
)
|
||||
stream_iter: Final = routed.__aiter__()
|
||||
for _ in frames:
|
||||
await stream_iter.__anext__()
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
|
||||
{"litellm_logging_obj": routed.logging_obj}, routed
|
||||
)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert {choice.index: choice.finish_reason for choice in partial_response.choices} == expected
|
||||
assert partial_response.usage.completion_tokens > 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("candidate", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
{"content": {"parts": [{"text": "partial answer"}], "role": "model"}, "index": 0},
|
||||
"finish_reason_unspecified",
|
||||
id="no-finish-reason",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"content": {"parts": [{"text": "partial answer"}], "role": "model"},
|
||||
"finishReason": "FINISH_REASON_UNSPECIFIED",
|
||||
},
|
||||
"finish_reason_unspecified",
|
||||
id="finish-reason-unspecified-sentinel",
|
||||
),
|
||||
pytest.param(
|
||||
{"content": {"parts": [{"text": "partial answer"}], "role": "model"}, "finishReason": "MAX_TOKENS"},
|
||||
"length",
|
||||
id="max-tokens",
|
||||
),
|
||||
pytest.param(
|
||||
{"content": {"parts": [{"text": "full answer"}], "role": "model"}, "finishReason": "STOP"},
|
||||
"stop",
|
||||
id="stop",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"content": {"parts": [{"functionCall": {"name": "lookup", "args": {"q": "fox"}}}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
},
|
||||
"tool_calls",
|
||||
id="stop-after-a-tool-call",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_stream_bills_only_the_finish_reason_the_candidate_sent(
|
||||
self, candidate: Mapping[str, object], expected: str, stream_attempt: StreamAttempt
|
||||
):
|
||||
partial_response: Final = await self._billed_after_the_first_gemini_frame(
|
||||
stream_attempt, {"candidates": [candidate]}
|
||||
)
|
||||
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == [expected]
|
||||
assert partial_response.usage.completion_tokens > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_frame_that_plans_a_continuation_hop_is_not_billed_as_finished(
|
||||
self, stream_attempt: StreamAttempt
|
||||
):
|
||||
from litellm.types.llms.vertex_ai import GenerationConfig, RequestBody
|
||||
|
||||
async def _hop_must_not_open(request_body: RequestBody) -> httpx.Response:
|
||||
pytest.fail(f"the client disconnected before the hop, yet it opened with {request_body}")
|
||||
|
||||
partial_response: Final = await self._billed_after_the_first_gemini_frame(
|
||||
stream_attempt,
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"role": "model", "parts": [{"text": "hello"}]},
|
||||
"finishReason": "CONTINUATION",
|
||||
"continuationToken": "dG9rLTE=",
|
||||
}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 30, "totalTokenCount": 40},
|
||||
},
|
||||
AsyncStreamResume(
|
||||
request_body=RequestBody(
|
||||
contents=[{"role": "user", "parts": [{"text": "Prove it"}]}],
|
||||
generationConfig=GenerationConfig(max_output_tokens=100),
|
||||
),
|
||||
open_hop=_hop_must_not_open,
|
||||
),
|
||||
)
|
||||
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"]
|
||||
assert partial_response.usage.completion_tokens > 0
|
||||
|
||||
@pytest.mark.parametrize("evidence", ["length", ((0, None),), ((0, "length", "stop"),)])
|
||||
@pytest.mark.asyncio
|
||||
async def test_unreadable_provider_finish_evidence_bills_unspecified_instead_of_the_synthesized_stop(
|
||||
self, evidence: object, stream_attempt: StreamAttempt
|
||||
):
|
||||
from litellm.constants import PROVIDER_FINISH_REASONS_HIDDEN_PARAM
|
||||
from litellm.litellm_core_utils.hidden_params import set_hidden_param
|
||||
|
||||
frame: Final = ModelResponseStream(
|
||||
id="chatcmpl-unreadable",
|
||||
created=1741037890,
|
||||
model="gemini-3.8-flash",
|
||||
choices=[self._content_choice(0, "partial answer", "stop")],
|
||||
)
|
||||
set_hidden_param(frame, "is_finished", False)
|
||||
set_hidden_param(frame, PROVIDER_FINISH_REASONS_HIDDEN_PARAM, evidence)
|
||||
|
||||
partial_response: Final = await self._billed_after_the_first_frame(
|
||||
stream_attempt, lambda _: ModelResponseListIterator(model_responses=[frame])
|
||||
)
|
||||
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == ["finish_reason_unspecified"]
|
||||
|
||||
async def _billed_after_the_first_gemini_frame(
|
||||
self, stream_attempt: StreamAttempt, frame: Mapping[str, object], resume: AsyncStreamResume | None = None
|
||||
) -> litellm.ModelResponse:
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ModelResponseIterator
|
||||
|
||||
async def _gemini_sse() -> AsyncGenerator[str, None]:
|
||||
yield f"data: {json.dumps(frame)}"
|
||||
|
||||
return await self._billed_after_the_first_frame(
|
||||
stream_attempt,
|
||||
lambda logging_obj: ModelResponseIterator(
|
||||
streaming_response=_gemini_sse(), sync_stream=False, logging_obj=logging_obj, resume=resume
|
||||
),
|
||||
)
|
||||
|
||||
async def _billed_after_the_first_frame(
|
||||
self, stream_attempt: StreamAttempt, completion_stream: Callable[[LiteLLMLoggingObj], object]
|
||||
) -> litellm.ModelResponse:
|
||||
model: Final = "gemini-3.8-flash"
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
logging_obj: Final = self._recorded_logging_obj(
|
||||
model, [{"role": "user", "content": "hi"}], "first-frame-disconnect-call", recorder
|
||||
)
|
||||
provider_stream: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream(logging_obj),
|
||||
model=model,
|
||||
custom_llm_provider="gemini",
|
||||
logging_obj=logging_obj,
|
||||
stream_options=None,
|
||||
)
|
||||
routed: Final = await open_attempt_stream(stream_attempt, provider_stream)
|
||||
assert isinstance(routed, CustomStreamWrapper)
|
||||
delivered: Final = await routed.__aiter__().__anext__()
|
||||
assert delivered.choices[0].finish_reason is None
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect({"litellm_logging_obj": logging_obj}, routed)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
return recorder.success_events[0]["response_obj"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_uses_a_finish_reason_received_without_a_choice_index_for_a_single_choice(self):
|
||||
model: Final = "gpt-4o"
|
||||
messages: Final = [{"role": "user", "content": "hi"}]
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
logging_obj: Final = self._recorded_logging_obj(model, messages, "unindexed-finish-call", recorder)
|
||||
collected_stream: Final = SimpleNamespace(
|
||||
chunks=[
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-unindexed",
|
||||
created=1741037890,
|
||||
model=model,
|
||||
choices=[self._content_choice(0, "partial answer")],
|
||||
)
|
||||
],
|
||||
messages=messages,
|
||||
model=model,
|
||||
finish_evidence=FinishEvidence(by_choice=MappingProxyType({}), unindexed="length"),
|
||||
)
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
|
||||
{"litellm_logging_obj": logging_obj}, collected_stream
|
||||
)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert [choice.finish_reason for choice in partial_response.choices] == ["length"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_marks_only_the_unfinished_choice_unspecified(self):
|
||||
model: Final = "gpt-4o"
|
||||
messages: Final = [{"role": "user", "content": "look both up"}]
|
||||
recorder: Final = _RecordingSuccessLogger()
|
||||
logging_obj: Final = self._recorded_logging_obj(model, messages, "two-choice-disconnect-call", recorder)
|
||||
|
||||
def tool_call_delta(call_id: str) -> Delta:
|
||||
return Delta(
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
{"id": call_id, "type": "function", "index": 0, "function": {"name": "lookup", "arguments": "{}"}}
|
||||
],
|
||||
)
|
||||
|
||||
collected_stream: Final = SimpleNamespace(
|
||||
chunks=[
|
||||
ModelResponseStream(id="chatcmpl-two", created=1741037890, model=model, choices=choices)
|
||||
for choices in (
|
||||
[
|
||||
StreamingChoices(index=0, delta=tool_call_delta("call_a")),
|
||||
StreamingChoices(index=1, delta=tool_call_delta("call_b")),
|
||||
],
|
||||
[StreamingChoices(index=0, delta=Delta(), finish_reason="tool_calls")],
|
||||
)
|
||||
],
|
||||
messages=messages,
|
||||
model=model,
|
||||
finish_evidence=FinishEvidence(by_choice=MappingProxyType({}), unindexed="tool_calls"),
|
||||
)
|
||||
|
||||
billed: Final = await _bill_partial_streamed_spend_on_disconnect(
|
||||
{"litellm_logging_obj": logging_obj}, collected_stream
|
||||
)
|
||||
await asyncio.wait_for(recorder.logged.wait(), timeout=5)
|
||||
|
||||
assert billed is True
|
||||
assert len(recorder.success_events) == 1
|
||||
partial_response: Final = recorder.success_events[0]["response_obj"]
|
||||
assert {choice.index: choice.finish_reason for choice in partial_response.choices} == {
|
||||
0: "tool_calls",
|
||||
1: "finish_reason_unspecified",
|
||||
}
|
||||
assert [len(choice.message.tool_calls) for choice in partial_response.choices] == [1, 1]
|
||||
assert partial_response.usage.total_tokens > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_stream_does_not_double_bill_on_late_disconnect(self):
|
||||
recorder = _RecordingSuccessLogger()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue