diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 321bfd546e7..0b4a226d159 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -498,6 +498,7 @@ class BaseResponsesAPIStreamingIterator: self.request_data.get("input"), self.request_data, self._generated_content + self._generated_tool_arguments, + use_default_image_token_count=False, ) if _estimate_wanted else None @@ -888,7 +889,11 @@ class BaseResponsesAPIStreamingIterator: if not generated_text or self.logging_obj.model_call_details.get("combined_usage_object") is not None: return estimate: Final = _estimate_usage_safely( - self.model or "", self.request_data.get("input"), self.request_data, generated_text + self.model or "", + self.request_data.get("input"), + self.request_data, + generated_text, + use_default_image_token_count=True, ) if estimate is None: return @@ -1558,13 +1563,15 @@ def _estimate_usage_from_text( request_input: object, responses_api_request: Mapping[str, object], generated_text: str, + *, + use_default_image_token_count: bool, ) -> ResponseAPIUsage: messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union responses_api_request=dict(responses_api_request), ) input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped - model=model, messages=messages + model=model, messages=messages, use_default_image_token_count=use_default_image_token_count ) output_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped model=model, text=generated_text, count_response_tokens=True @@ -1581,6 +1588,8 @@ def _estimate_usage_safely( request_input: object, responses_api_request: Mapping[str, object], generated_text: str, + *, + use_default_image_token_count: bool, ) -> ResponseAPIUsage | None: try: return _estimate_usage_from_text( @@ -1588,6 +1597,7 @@ def _estimate_usage_safely( request_input=request_input, responses_api_request=responses_api_request, generated_text=generated_text, + use_default_image_token_count=use_default_image_token_count, ) except Exception as e: verbose_logger.debug("Could not estimate usage from stream text, billing $0: %s", e) diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index c8d05bc0f57..47b9dca0dac 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -658,10 +658,6 @@ def _collect_with_logging(stream, logging_obj: LitellmLogging) -> list: class TestUpstreamFailureReachesTheProxyFailureHook: - """The ``error`` event alone leaves the proxy unaware the stream failed, so the wrapper also hands - the failure to the hook the proxy arms on the logging object, the one that writes the failure - spend row and releases the budget reservation.""" - def test_raised_upstream_failure_reaches_the_hook_unwrapped_before_the_error_event(self): rate_limit = litellm.RateLimitError(message="You have no credits remaining.", llm_provider="openai", model="m") wrapped = MidStreamFallbackError( diff --git a/tests/unit/responses/test_responses_streaming_iterator.py b/tests/unit/responses/test_responses_streaming_iterator.py index 02ce972268f..f6ea7dcfa9c 100644 --- a/tests/unit/responses/test_responses_streaming_iterator.py +++ b/tests/unit/responses/test_responses_streaming_iterator.py @@ -183,11 +183,24 @@ class _ChunkRecorder: self.chunks = (*self.chunks, chunk) +_SEA_DELTAS = ("The sea ", "is wide ", "and deep.") +_SEA_PROMPT = "Write a 300 word story about the sea." +_UNREACHABLE_IMAGE_INPUT = [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Describe this picture."}, + {"type": "input_image", "image_url": "http://images.invalid/photo.png", "detail": "high"}, + ], + } +] + + class _UpstreamThatTimesOutAfterThreeDeltas: headers: dict = {} async def aiter_bytes(self): - for sequence_number, delta in enumerate(("The sea ", "is wide ", "and deep.")): + for sequence_number, delta in enumerate(_SEA_DELTAS): event = { "type": "response.output_text.delta", "item_id": "msg_1", @@ -200,8 +213,7 @@ class _UpstreamThatTimesOutAfterThreeDeltas: raise httpx.ReadTimeout("Timeout on reading data from socket") -@pytest.mark.asyncio -async def test_mid_stream_read_timeout_logs_failure_with_partial_usage_and_no_success(monkeypatch): +async def _failure_payload_after_mid_stream_timeout(monkeypatch, request_input) -> tuple[FailureRecorder, dict]: recorder = FailureRecorder() monkeypatch.setattr(litellm, "failure_callback", [recorder]) monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) @@ -219,17 +231,50 @@ async def test_mid_stream_read_timeout_logs_failure_with_partial_usage_and_no_su responses_api_provider_config=OpenAIResponsesAPIConfig(), logging_obj=logging_obj, custom_llm_provider="openai", + request_data={"model": "gpt-5.4-nano", "input": request_input, "stream": True}, ) received = _ChunkRecorder() with pytest.raises(httpx.ReadTimeout, match="Timeout on reading data from socket"): await received.drain(iterator) - assert [chunk.delta for chunk in received.chunks] == ["The sea ", "is wide ", "and deep."] + assert [chunk.delta for chunk in received.chunks] == list(_SEA_DELTAS) await iterator._await_pending_logging() (payload,) = recorder.failure_payloads + return recorder, payload + + +@pytest.mark.asyncio +async def test_mid_stream_read_timeout_logs_failure_with_partial_usage_and_no_success(monkeypatch): + recorder, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _SEA_PROMPT) + assert payload["status"] == "failure" - assert payload["prompt_tokens"] > 0 - assert payload["completion_tokens"] > 0 + assert payload["prompt_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", messages=[{"role": "user", "content": _SEA_PROMPT}] + ) + assert payload["completion_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", text="".join(_SEA_DELTAS), count_response_tokens=True + ) assert payload["response_cost"] > 0 assert recorder.success_payloads == () + + +@pytest.mark.asyncio +async def test_mid_stream_failure_usage_estimate_never_fetches_image_dimensions(monkeypatch): + _, payload = await _failure_payload_after_mid_stream_timeout(monkeypatch, _UNREACHABLE_IMAGE_INPUT) + + assert payload["status"] == "failure" + assert payload["prompt_tokens"] == litellm.token_counter( + model="gpt-5.4-nano", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this picture."}, + {"type": "image_url", "image_url": {"url": "http://images.invalid/photo.png", "detail": "high"}}, + ], + } + ], + use_default_image_token_count=True, + ) + assert payload["completion_tokens"] > 0 diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 2f6dccb37f3..ef571973a3c 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -1141,7 +1141,9 @@ async def test_completed_event_survives_a_failing_usage_estimate(): falls back to usage None.""" malformed_input: Final = [{"type": "message", "role": "user", "content": 42}] with pytest.raises(ValueError, match="Invalid content type"): - _estimate_usage_from_text("gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world") + _estimate_usage_from_text( + "gpt-4o-mini", malformed_input, {"input": malformed_input}, "hello world", use_default_image_token_count=False + ) response = _responses_api_response_without_usage() iterator = _make_iterator(