mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(streaming): keep provider fields on the final usage chunk
Provider fields sent on the usage-only chunk at the end of a stream (for example GreenPT's impact object) never reached the client. The stream wrapper rebuilt the final usage chunk without them, and the proxy dropped them in its fast serializer and when stripping the usage chunk it injected for spend tracking. Record the usage-only chunk's non-OpenAI fields and stamp them onto the final usage chunk, reusing the selection rule from preserve_upstream_non_openai_attributes. In the proxy, fall back to the canonical serializer when a chunk carries such fields, and keep an injected usage chunk that carries them, minus the usage. Fixes #46613 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
cfe0100e1f
commit
45d1c82eb4
5 changed files with 171 additions and 10 deletions
|
|
@ -634,19 +634,36 @@ def process_response_headers(
|
|||
return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})}
|
||||
|
||||
|
||||
def _upstream_non_openai_keys(
|
||||
response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream"
|
||||
) -> tuple[str, ...]:
|
||||
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
|
||||
expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"})
|
||||
return tuple(
|
||||
key
|
||||
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
|
||||
if key not in expected_keys
|
||||
)
|
||||
|
||||
|
||||
def _dump_keys(original_chunk: "ModelResponseStream", keys: tuple[str, ...]) -> Mapping[str, object]:
|
||||
return original_chunk.model_dump(include=frozenset(keys))
|
||||
|
||||
|
||||
def upstream_non_openai_attributes(
|
||||
response_type: "type[ModelResponseStream]", original_chunk: "ModelResponseStream"
|
||||
) -> Mapping[str, object]:
|
||||
keys: Final = _upstream_non_openai_keys(response_type, original_chunk)
|
||||
return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {})
|
||||
|
||||
|
||||
def preserve_upstream_non_openai_attributes(
|
||||
model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
|
||||
):
|
||||
"""
|
||||
Preserve non-OpenAI attributes from the original chunk.
|
||||
"""
|
||||
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
|
||||
expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"})
|
||||
candidate_keys: Final = tuple(
|
||||
key
|
||||
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
|
||||
if key not in expected_keys
|
||||
)
|
||||
candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk)
|
||||
if not candidate_keys:
|
||||
return
|
||||
candidate_values: Final[tuple[tuple[str, object], ...]] = tuple(
|
||||
|
|
@ -662,7 +679,7 @@ def preserve_upstream_non_openai_attributes(
|
|||
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
|
||||
_assign_attributes(model_response, candidate_values)
|
||||
return
|
||||
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
|
||||
_assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items())
|
||||
|
||||
|
||||
def _assign_attributes(target: object, values: Iterable[tuple[str, object]]) -> None:
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ from litellm.types.utils import (
|
|||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
|
||||
from ..exceptions import OpenAIError, Timeout
|
||||
from .core_helpers import map_finish_reason, process_response_headers
|
||||
from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes
|
||||
from .exception_mapping_utils import exception_type, with_deployment_api_base
|
||||
from .llm_response_utils.get_api_base import get_api_base
|
||||
from .rules import Rules
|
||||
|
|
@ -138,6 +138,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
|
|||
return status_code is not None and 400 <= status_code < 500 and status_code != 429
|
||||
|
||||
|
||||
def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None:
|
||||
for key, value in attributes.items():
|
||||
if value is not None:
|
||||
setattr(response, key, value)
|
||||
|
||||
|
||||
def is_async_iterable(obj: object) -> bool:
|
||||
"""
|
||||
Check if an object is an async iterable (can be used with 'async for').
|
||||
|
|
@ -376,6 +382,7 @@ class CustomStreamWrapper:
|
|||
self.stream_options = stream_options or getattr(logging_obj, "stream_options", None)
|
||||
self.messages = getattr(logging_obj, "messages", None)
|
||||
self.sent_stream_usage = False
|
||||
self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({})
|
||||
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
|
||||
self.caller_requested_stream_usage: Final = (
|
||||
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
|
||||
|
|
@ -1696,6 +1703,9 @@ class CustomStreamWrapper:
|
|||
model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta()
|
||||
plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta
|
||||
else:
|
||||
self._usage_chunk_upstream_attributes = upstream_non_openai_attributes(
|
||||
ModelResponseStream, original_chunk
|
||||
)
|
||||
if self.send_stream_usage is True:
|
||||
model_response.choices = []
|
||||
return model_response
|
||||
|
|
@ -2161,6 +2171,7 @@ class CustomStreamWrapper:
|
|||
|
||||
if self.sent_stream_usage is False and self.send_stream_usage is True:
|
||||
self.sent_stream_usage = True
|
||||
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
|
||||
return response
|
||||
self.close()
|
||||
raise # Re-raise StopIteration
|
||||
|
|
@ -2362,6 +2373,7 @@ class CustomStreamWrapper:
|
|||
|
||||
if self.sent_stream_usage is False and self.send_stream_usage is True:
|
||||
self.sent_stream_usage = True
|
||||
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
|
||||
return response
|
||||
|
||||
_deferred_cb: Final = getattr(
|
||||
|
|
|
|||
|
|
@ -10068,6 +10068,7 @@ def _fast_serialize_simple_model_response_stream(
|
|||
getattr(chunk, "provider_specific_fields", None) is not None
|
||||
or getattr(chunk, "system_fingerprint", None) is not None
|
||||
or getattr(chunk, "usage", None) is not None
|
||||
or _has_provider_extras(chunk)
|
||||
):
|
||||
return None
|
||||
|
||||
|
|
@ -10143,6 +10144,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
|
|||
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
|
||||
|
||||
|
||||
_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"})
|
||||
|
||||
|
||||
def _has_provider_extras(chunk: ModelResponseStream) -> bool:
|
||||
extras: Final = chunk.model_extra or {}
|
||||
return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS)
|
||||
|
||||
|
||||
def _without_injected_usage(chunk: object) -> ModelResponseStream | None:
|
||||
if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk):
|
||||
return None
|
||||
return chunk.model_copy(update={"usage": None})
|
||||
|
||||
|
||||
def _is_injected_stream_usage_artifact(chunk: object) -> bool:
|
||||
if not isinstance(chunk, ModelResponseStream):
|
||||
return False
|
||||
|
|
@ -10552,7 +10567,9 @@ async def async_data_generator(
|
|||
fallback_errors=fallback_errors,
|
||||
)
|
||||
fallback_metadata_event_sent = True
|
||||
continue
|
||||
if (kept_chunk := _without_injected_usage(chunk)) is None:
|
||||
continue
|
||||
chunk = kept_chunk
|
||||
|
||||
if error_state is not None:
|
||||
error_state.observe_chunk(chunk)
|
||||
|
|
|
|||
|
|
@ -875,6 +875,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes(
|
|||
assert getattr(returned_chunk, "custom_field", None) == {"key": "value"}
|
||||
|
||||
|
||||
def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]:
|
||||
def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl-usage-fields",
|
||||
created=1742056047,
|
||||
model=None,
|
||||
object="chat.completion.chunk",
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30))
|
||||
setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5})
|
||||
return [
|
||||
chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]),
|
||||
chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]),
|
||||
usage_chunk,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool):
|
||||
response: Final = CustomStreamWrapper(
|
||||
completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()),
|
||||
model="gpt-5.5",
|
||||
custom_llm_provider="openai",
|
||||
logging_obj=Logging(
|
||||
model="gpt-5.5",
|
||||
messages=[{"role": "user", "content": "Hey"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="usage-fields",
|
||||
function_id="usage-fields",
|
||||
),
|
||||
stream_options={"include_usage": True},
|
||||
)
|
||||
|
||||
chunks: Final = list(response) if sync_mode else [chunk async for chunk in response]
|
||||
|
||||
assert chunks[-1].usage is not None
|
||||
assert chunks[-1].usage.total_tokens == 30
|
||||
assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5}
|
||||
|
||||
|
||||
def test_set_response_id_propagation_empty_to_valid(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
|
|
|
|||
|
|
@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv
|
|||
assert _fast_serialize_simple_model_response_stream(chunk) is None
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_keeps_provider_fields():
|
||||
chunk = _simple_chunk()
|
||||
setattr(chunk, "provider_metrics", {"energy_wh": 0.5})
|
||||
|
||||
payload = json.loads(_serialize_streaming_chunk(chunk))
|
||||
|
||||
assert payload["provider_metrics"] == {"energy_wh": 0.5}
|
||||
assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"}
|
||||
|
||||
|
||||
def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path():
|
||||
chunk = _simple_chunk()
|
||||
setattr(chunk, "citations", None)
|
||||
|
||||
assert isinstance(_serialize_streaming_chunk(chunk), bytes)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _serialize_streaming_chunk
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False):
|
|||
)
|
||||
|
||||
|
||||
def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream:
|
||||
chunk: Final = ModelResponseStream(
|
||||
id="chatcmpl-test",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion.chunk",
|
||||
usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
|
||||
)
|
||||
for key, value in provider_fields.items():
|
||||
setattr(chunk, key, value)
|
||||
return chunk
|
||||
|
||||
|
||||
def _sse_payload(line: str | bytes) -> dict[str, object]:
|
||||
text: Final = line.decode() if isinstance(line, bytes) else line
|
||||
return json.loads(text.removeprefix("data: ").removesuffix("\n\n"))
|
||||
|
||||
|
||||
async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]:
|
||||
out: Final = [
|
||||
line
|
||||
async for line in async_data_generator(
|
||||
response=_async_iter([_simple_chunk(content="hello"), usage_chunk]),
|
||||
user_api_key_dict=_user_auth(),
|
||||
request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True},
|
||||
)
|
||||
]
|
||||
assert out[-1] == "data: [DONE]\n\n"
|
||||
return [_sse_payload(line) for line in out[:-1]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
||||
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5}))
|
||||
|
||||
assert len(payloads) == 2
|
||||
assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5}
|
||||
assert "usage" not in payloads[-1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
||||
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None))
|
||||
|
||||
assert len(payloads) == 1
|
||||
assert "usage" not in payloads[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch):
|
||||
_patch_logging_flags(monkeypatch)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue