From 1a1e9925ba5139e12d3de53c8b4f434964fac4e1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:45:00 -0700 Subject: [PATCH 1/2] fix(proxy): bill streamed /v1/completions without stream_options like chat --- .../litellm_core_utils/stream_usage_chunks.py | 20 ++ litellm/main.py | 2 + litellm/proxy/common_request_processing.py | 5 +- litellm/proxy/proxy_server.py | 15 +- litellm/utils.py | 23 ++- .../test_text_completion_stream_usage_cost.py | 195 ++++++++++++++++++ .../proxy/test_common_request_processing.py | 8 + tests/unit/test_main.py | 109 +++++++++- tests/unit/test_utils.py | 52 ++++- 9 files changed, 407 insertions(+), 22 deletions(-) create mode 100644 litellm/litellm_core_utils/stream_usage_chunks.py create mode 100644 tests/integration/spend/test_text_completion_stream_usage_cost.py diff --git a/litellm/litellm_core_utils/stream_usage_chunks.py b/litellm/litellm_core_utils/stream_usage_chunks.py new file mode 100644 index 00000000000..4db2f674278 --- /dev/null +++ b/litellm/litellm_core_utils/stream_usage_chunks.py @@ -0,0 +1,20 @@ +from typing import Final + +from litellm.types.utils import ModelResponseStream, StreamingChoices + + +def is_empty_streaming_choice(choice: StreamingChoices) -> bool: + if choice.finish_reason is not None: + return False + if getattr(choice, "logprobs", None) is not None: + return False + delta: Final = getattr(choice, "delta", None) + if delta is None: + return True + return all(value is None for value in delta.model_dump().values()) + + +def is_usage_only_chunk(chunk: ModelResponseStream) -> bool: + if getattr(chunk, "usage", None) is None: + return False + return all(is_empty_streaming_choice(choice) for choice in chunk.choices or []) diff --git a/litellm/main.py b/litellm/main.py index 37b290f51ba..c27cdafd804 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7458,6 +7458,7 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp model=model, custom_llm_provider=custom_llm_provider, stream_options=kwargs.get("stream_options"), + usage_injected_by_proxy=kwargs.get("_litellm_strip_stream_usage") is True, ) else: ## OpenAI / Azure Text Completion Returns here @@ -7703,6 +7704,7 @@ def text_completion( model=model, stream_options=stream_options, custom_llm_provider=custom_llm_provider, + usage_injected_by_proxy=kwargs.get("_litellm_strip_stream_usage") is True, ) return response elif isinstance(response, TextCompletionStreamWrapper): diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 317203cc2d5..1cb0f55cd4e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -522,6 +522,9 @@ def _model_deployments_support_stream_options( return all(_litellm_model_supports_stream_options(m) for m in candidate_models) +_STREAM_USAGE_TRACKED_ROUTES: Final = frozenset({"acompletion", "atext_completion"}) + + def _stream_usage_tracking_updates( data: Mapping[str, object], general_settings: Mapping[str, object], @@ -539,7 +542,7 @@ def _stream_usage_tracking_updates( if isinstance(stream_options, dict) and "include_usage" not in stream_options: return {**scrub, "stream_options": {**stream_options, "include_usage": True}} return scrub - if always_include is False or route_type != "acompletion": + if always_include is False or route_type not in _STREAM_USAGE_TRACKED_ROUTES: return scrub if isinstance(stream_options, dict) and stream_options.get("include_usage") is True: return scrub diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 000ffa9c305..d470ee08e76 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -163,7 +163,6 @@ from litellm.types.utils import ( PRICING_OVERRIDES_KEY, ModelResponse, ModelResponseStream, - StreamingChoices, TextCompletionResponse, TokenCountResponse, echoed_cost_map_pricing_fields, @@ -333,6 +332,7 @@ from litellm.litellm_core_utils.sensitive_data_masker import ( SensitiveDataMasker, mask_sensitive_keys, ) +from litellm.litellm_core_utils.stream_usage_chunks import is_empty_streaming_choice from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.openai_like.model_info import MODEL_INFO_REFRESH_SECONDS from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -9824,18 +9824,7 @@ def _is_injected_stream_usage_artifact(chunk: object) -> bool: return False if chunk.provider_specific_fields is not None: return False - return all(_is_empty_streaming_choice(choice) for choice in chunk.choices or []) - - -def _is_empty_streaming_choice(choice: StreamingChoices) -> bool: - if choice.finish_reason is not None: - return False - if getattr(choice, "logprobs", None) is not None: - return False - delta: Final = getattr(choice, "delta", None) - if delta is None: - return True - return all(value is None for value in delta.model_dump().values()) + return all(is_empty_streaming_choice(choice) for choice in chunk.choices or []) async def _apply_streaming_chunk_hooks( diff --git a/litellm/utils.py b/litellm/utils.py index 7a334f8974b..572bad4c3e8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -90,6 +90,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_fill_missing_generalizations, ) from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload +from litellm.litellm_core_utils.stream_usage_chunks import is_usage_only_chunk from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, strip_special_tokens from litellm.rust_bridge import tokenizer as tokenizer_dispatch from litellm.rust_bridge.catalog import decision @@ -7373,11 +7374,17 @@ class TextCompletionStreamWrapper: model, stream_options: dict | None = None, custom_llm_provider: str | None = None, + usage_injected_by_proxy: bool = False, ): self.completion_stream = completion_stream self.model = model self.stream_options = stream_options self.custom_llm_provider = custom_llm_provider + self.include_usage: Final = ( + not usage_injected_by_proxy + and stream_options is not None + and stream_options.get("include_usage", False) is True + ) def __iter__(self): return self @@ -7402,14 +7409,18 @@ class TextCompletionStreamWrapper: text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] response["choices"] = [text_choices] - # only pass usage when stream_options["include_usage"] is True - if self.stream_options and self.stream_options.get("include_usage", False) is True: + if self.include_usage: response["usage"] = chunk.get("usage", None) return response except Exception as e: raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") + def _client_frame(self, chunk: ModelResponse) -> TextCompletionResponse | None: + if not self.include_usage and isinstance(chunk, ModelResponseStream) and is_usage_only_chunk(chunk): + return None + return self.convert_to_text_completion_object(chunk=chunk) + def __next__(self): # model_response = ModelResponse(stream=True, model=self.model) TextCompletionResponse() @@ -7417,7 +7428,9 @@ class TextCompletionStreamWrapper: for chunk in self.completion_stream: if chunk == "None" or chunk is None: raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) + processed_chunk = self._client_frame(chunk) + if processed_chunk is None: + continue return processed_chunk raise StopIteration except StopIteration: @@ -7437,7 +7450,9 @@ class TextCompletionStreamWrapper: async for chunk in self.completion_stream: if chunk == "None" or chunk is None: raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) + processed_chunk = self._client_frame(chunk) + if processed_chunk is None: + continue return processed_chunk raise StopIteration except StopIteration: diff --git a/tests/integration/spend/test_text_completion_stream_usage_cost.py b/tests/integration/spend/test_text_completion_stream_usage_cost.py new file mode 100644 index 00000000000..8391020bd46 --- /dev/null +++ b/tests/integration/spend/test_text_completion_stream_usage_cost.py @@ -0,0 +1,195 @@ +"""A streamed /v1/completions bills its tokens without the client asking for stream_options.include_usage. + +The scripted OpenAI-compatible upstream sends a usage chunk only when the request asks for one, the way OpenAI +does, so the proxy has to ask on the client's behalf and keep the answer to itself: the spend row of a streamed +text completion with no stream_options equals the include_usage variant's and the non-streamed twin's, and the +client gets no usage frame it did not ask for +""" + +import json +from pathlib import Path +from typing import Final +from uuid import uuid4 + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +PROMPT_TOKENS: Final = 19 +COMPLETION_TOKENS: Final = 5 +INPUT_RATE: Final = 0.001 +OUTPUT_RATE: Final = 0.002 +EXPECTED_SPEND: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE +USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, +} +PROMPT: Final = "Say hi" + + +def _sse_frame(payload: dict[str, JsonValue]) -> bytes: + return f"data: {json.dumps(payload, separators=(',', ':'))}\n\n".encode() + + +def _chunk(request_id: str, model: str, delta: dict[str, JsonValue], finish_reason: str | None) -> dict[str, JsonValue]: + return { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +def _respond(request: Request) -> Reply: + if request.method == "GET" and request.target == "/v1/models": + return Reply(body=json.dumps({"object": "list", "data": [{"id": "gpt-5.4-mini", "object": "model"}]}).encode()) + assert request.method == "POST", request.method + assert request.target == "/v1/chat/completions", request.target + body: Final = json.loads(request.body) + assert body["messages"] == [{"role": "user", "content": PROMPT}], body + request_id: Final = f"chatcmpl-{uuid4().hex}" + model: Final = str(body["model"]) + if body.get("stream") is not True: + return Reply( + body=json.dumps( + { + "id": request_id, + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Hi there"}, "finish_reason": "length"} + ], + "usage": USAGE, + } + ).encode() + ) + stream_options: Final = body.get("stream_options") or {} + usage_frames: Final = ( + ( + _sse_frame( + { + "id": request_id, + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [], + "usage": USAGE, + } + ), + ) + if stream_options.get("include_usage") is True + else () + ) + return Reply( + content_type="text/event-stream", + chunks=( + _sse_frame(_chunk(request_id, model, {"role": "assistant", "content": "Hi"}, None)), + _sse_frame(_chunk(request_id, model, {"content": " there"}, None)), + _sse_frame(_chunk(request_id, model, {}, "length")), + *usage_frames, + b"data: [DONE]\n\n", + ), + ) + + +def _proxy_config(directory: Path, model: str, upstream_url: str) -> Path: + config: Final = directory / "text_completion_stream_usage_cost_config.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": model, + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_base": f"{upstream_url}/v1", + "api_key": "scripted-upstream", + "input_cost_per_token": INPUT_RATE, + "output_cost_per_token": OUTPUT_RATE, + }, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "store_model_in_db": True, + "disable_spend_logs": False, + "proxy_batch_write_at": 1, + "proxy_batch_polling_interval": 1, + }, + "router_settings": {"disable_cooldowns": True}, + } + ) + ) + return config + + +def _data_events(body: str) -> tuple[dict[str, JsonValue], ...]: + return tuple( + json.loads(line.removeprefix("data:")) + for line in body.splitlines() + if line.startswith("data:") and not line.endswith("[DONE]") + ) + + +def _carries_text_or_finish(frame: dict[str, JsonValue]) -> bool: + choices: Final = frame["choices"] + assert isinstance(choices, list) and len(choices) == 1, frame + choice: Final = choices[0] + assert isinstance(choice, dict), frame + return choice.get("text") is not None or choice.get("finish_reason") is not None + + +def _billed(row: dict[str, object]) -> tuple[object, object, float]: + return (row["prompt_tokens"], row["completion_tokens"], float(str(row["spend"]))) + + +@pytest.mark.timeout(180) +def test_text_completion_stream_without_stream_options_bills_like_its_twins(gateway: Gateway, tmp_path: Path) -> None: + model: Final = f"gpt-5.4-mini-{uuid4().hex}" + body: Final[dict[str, JsonValue]] = {"model": model, "prompt": PROMPT, "max_tokens": COMPLETION_TOKENS} + + with wire_server(_respond) as wire: + config: Final = _proxy_config(tmp_path, model, wire.url) + with owned_proxy(gateway, tmp_path, {}, config=config) as candidate: + silent: Final = candidate.request("POST", "/v1/completions", {**body, "stream": True}) + asked: Final = candidate.request( + "POST", "/v1/completions", {**body, "stream": True, "stream_options": {"include_usage": True}} + ) + whole: Final = candidate.request("POST", "/v1/completions", body) + assert (silent.status_code, asked.status_code, whole.status_code) == (200, 200, 200), ( + silent.text, + asked.text, + whole.text, + ) + silent_frames: Final = _data_events(silent.text) + asked_frames: Final = _data_events(asked.text) + request_ids: Final = (silent_frames[0]["id"], asked_frames[0]["id"], whole.json()["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE request_id = ANY(%s)", + (list(request_ids),), + ), + lambda values: len(values) == 3, + seconds=70, + ) + upstream_stream_options: Final = [ + json.loads(sent.body).get("stream_options") for sent in wire.drain() if sent.method == "POST" + ] + + assert upstream_stream_options == [{"include_usage": True}, {"include_usage": True}, None] + assert all(_carries_text_or_finish(frame) for frame in silent_frames), silent.text + assert all(frame.get("usage", {}) == {} or frame["usage"]["total_tokens"] == 0 for frame in silent_frames), ( + silent.text + ) + assert asked_frames[-1]["usage"]["total_tokens"] == PROMPT_TOKENS + COMPLETION_TOKENS, asked.text + billed: Final = {str(row["request_id"]): _billed(row) for row in rows} + assert billed[str(request_ids[0])] == billed[str(request_ids[1])] == billed[str(request_ids[2])], billed + assert billed[str(request_ids[0])] == (PROMPT_TOKENS, COMPLETION_TOKENS, pytest.approx(EXPECTED_SPEND)), billed diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 1aed60ee5e2..fc19bd4df5f 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -7829,6 +7829,14 @@ class TestApplyStreamUsageTracking: assert data["stream_options"] == {"include_usage": True} assert data["_litellm_strip_stream_usage"] is True + def test_default_injects_usage_and_marks_strip_for_text_completions(self): + data = {"stream": True, "model": "gpt-5.4-nano"} + + _apply_stream_usage_tracking(data=data, general_settings={}, route_type="atext_completion") + + assert data["stream_options"] == {"include_usage": True} + assert data["_litellm_strip_stream_usage"] is True + def test_default_preserves_other_client_stream_options_keys(self): data = {"stream": True, "stream_options": {"include_obfuscation": True}} diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index dd7073bb381..8406f4a9d18 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -18,7 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError +from openai import APITimeoutError, OpenAI from openai.types.chat.chat_completion import ChatCompletion import litellm @@ -37,7 +37,14 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.litellm_params import ControlOptions from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec -from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage +from litellm.types.utils import ( + Delta, + ModelResponseStream, + StandardCallbackDynamicParams, + StreamingChoices, + TextCompletionResponse, + Usage, +) @pytest.fixture(autouse=True) @@ -3162,6 +3169,103 @@ def test_mock_text_completion_stream_and_non_stream_report_the_same_zero_admissi assert non_stream.usage.prompt_tokens == 0 +def _carries_text_or_finish(frame: TextCompletionResponse) -> bool: + return frame.choices[0].text is not None or frame.choices[0].finish_reason is not None + + +def test_mock_text_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + frames: Final = list( + litellm.text_completion( + model="openai/gpt-5.4-mini", + prompt="hi", + mock_response="ok then", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + ) + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames) + assert all(frame.usage.total_tokens == 0 for frame in frames) + + +@pytest.mark.asyncio +async def test_mock_atext_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + response: Final = await litellm.atext_completion( + model="openai/gpt-5.4-mini", + prompt="hi", + mock_response="ok then", + api_key="mock", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + + frames: Final = [frame async for frame in response] + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames) + assert all(frame.usage.total_tokens == 0 for frame in frames) + + +def _native_completions_upstream(outbound: list[dict[str, object]]) -> OpenAI: + def respond(request: httpx.Request) -> httpx.Response: + outbound.append(json.loads(request.content)) + base: Final = {"id": "cmpl-native", "object": "text_completion", "created": 1, "model": "code-model"} + frames: Final = ( + {**base, "choices": [{"index": 0, "text": "def ", "finish_reason": None, "logprobs": None}]}, + {**base, "choices": [{"index": 0, "text": "main", "finish_reason": "length", "logprobs": None}]}, + {**base, "choices": [], "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}, + ) + content: Final = "".join(f"data: {json.dumps(frame)}\n\n" for frame in frames) + "data: [DONE]\n\n" + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=content.encode()) + + return OpenAI(api_key="transport-only", http_client=httpx.Client(transport=httpx.MockTransport(respond))) + + +def test_native_text_completion_stream_hides_the_usage_chunk_the_proxy_asked_for_on_the_clients_behalf(): + outbound: Final[list[dict[str, object]]] = [] + + frames: Final = list( + litellm.text_completion( + model="text-completion-openai/code-model", + prompt="hi", + client=_native_completions_upstream(outbound), + api_key="transport-only", + stream=True, + stream_options={"include_usage": True}, + _litellm_strip_stream_usage=True, + ) + ) + + assert [sent.get("stream_options") for sent in outbound] == [{"include_usage": True}] + assert [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] == [ + ("def ", None), + ("main", None), + (None, "length"), + ] + assert all(frame.usage.total_tokens == 0 for frame in frames) + + +def test_native_text_completion_stream_keeps_the_usage_chunk_the_client_asked_for(): + frames: Final = list( + litellm.text_completion( + model="text-completion-openai/code-model", + prompt="hi", + client=_native_completions_upstream([]), + api_key="transport-only", + stream=True, + stream_options={"include_usage": True}, + ) + ) + + assert len(frames) == 4 + assert all(_carries_text_or_finish(frame) for frame in frames[:3]) + assert (frames[-1].usage.prompt_tokens, frames[-1].usage.completion_tokens) == (10, 5) + + def test_mock_completion_stream_with_model_response(): """Test that mock_completion correctly handles stream=True with a ModelResponse as mock_response.""" from litellm import completion @@ -3319,7 +3423,6 @@ class TestCallTypesOCR: def test_stream_chunk_builder_text_completion_combines_text_and_usage(): from litellm.main import stream_chunk_builder_text_completion - from litellm.types.utils import TextCompletionResponse chunks = [ TextCompletionResponse( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 8a7c9825f0a..26f4745c0c9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -9,7 +9,7 @@ import logging import os import queue import threading -from collections.abc import Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Iterator, Mapping from concurrent.futures import Future, ThreadPoolExecutor from datetime import datetime, timedelta, timezone from pathlib import PurePath @@ -70,6 +70,7 @@ from litellm.types.utils import ( RerankResponse, StandardCallbackDynamicParams, StreamingChoices, + TextCompletionResponse, TranscriptionResponse, Usage, all_litellm_params, @@ -2200,6 +2201,55 @@ def test_reasoning_content_preserved_in_text_completion_wrapper(): assert choice["reasoning_content"] == "Here's my chain of thought..." +def _text_stream_chunks() -> tuple[ModelResponseStream, ...]: + return ( + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Hi"), finish_reason=None)]), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="length")]), + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(), finish_reason=None)], + usage=Usage(prompt_tokens=19, completion_tokens=5, total_tokens=24), + ), + ) + + +def _text_frames(frames: list[TextCompletionResponse]) -> list[tuple[str | None, str | None]]: + return [(frame.choices[0].text, frame.choices[0].finish_reason) for frame in frames] + + +def test_text_completion_stream_wrapper_hides_the_usage_chunk_the_client_did_not_ask_for(): + wrapper: Final = TextCompletionStreamWrapper(completion_stream=iter(_text_stream_chunks()), model="gpt-5.4-mini") + + frames: Final = list(wrapper) + + assert _text_frames(frames) == [("Hi", None), (None, "length")] + assert all(frame.usage.total_tokens == 0 for frame in frames) + + +def test_text_completion_stream_wrapper_keeps_the_usage_chunk_the_client_asked_for(): + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=iter(_text_stream_chunks()), model="gpt-5.4-mini", stream_options={"include_usage": True} + ) + + frames: Final = list(wrapper) + + assert _text_frames(frames) == [("Hi", None), (None, "length"), (None, None)] + assert (frames[-1].usage.prompt_tokens, frames[-1].usage.completion_tokens) == (19, 5) + + +@pytest.mark.asyncio +async def test_async_text_completion_stream_wrapper_hides_the_usage_chunk_the_client_did_not_ask_for(): + async def chunks() -> AsyncIterator[ModelResponseStream]: + for chunk in _text_stream_chunks(): + yield chunk + + wrapper: Final = TextCompletionStreamWrapper(completion_stream=chunks(), model="gpt-5.4-mini") + + frames: Final = [frame async for frame in wrapper] + + assert _text_frames(frames) == [("Hi", None), (None, "length")] + assert all(frame.usage.total_tokens == 0 for frame in frames) + + def test_anthropic_claude_4_invoke_chat_provider_config(): """Test that the Anthropic Claude 4 Invoke chat provider config is correct.""" from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( From f9ee31f2b235408bd8b5b985be28b79a15d6c3c2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:25:11 -0700 Subject: [PATCH 2/2] test(proxy): mark the text completion stream usage test variable Final --- tests/unit/proxy/test_common_request_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 8d514e32b34..bd602cd0b9b 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9251,7 +9251,7 @@ class TestApplyStreamUsageTracking: assert data["_litellm_stream_usage_injected"] is True def test_default_injects_usage_and_marks_strip_for_text_completions(self): - data = {"stream": True, "model": "gpt-5.4-nano"} + data: Final = {"stream": True, "model": "gpt-5.4-nano"} _apply_stream_usage_tracking(data=data, general_settings={}, route_type="atext_completion")