This commit is contained in:
Ed Summers 2026-10-30 17:24:24 +00:00 • committed by GitHub
commit a99618031f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 168 additions and 10 deletions

View file

@ -680,19 +680,32 @@ def process_response_headers(
return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})}
def _upstream_non_openai_keys(response_type: type[BaseModel], original_chunk: BaseModel) -> tuple[str, ...]:
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
expected_keys: Final = frozenset(response_type.model_fields) | frozenset({"usage"})
return tuple(
key
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
if key not in expected_keys
)
def _dump_keys(original_chunk: BaseModel, keys: tuple[str, ...]) -> Mapping[str, object]:
return original_chunk.model_dump(include=MappingProxyType(dict.fromkeys(keys, True)))
def upstream_non_openai_attributes(response_type: type[BaseModel], original_chunk: BaseModel) -> Mapping[str, object]:
keys: Final = _upstream_non_openai_keys(response_type, original_chunk)
return MappingProxyType(_dump_keys(original_chunk, keys) if keys else {})
def preserve_upstream_non_openai_attributes(
model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
):
"""
Preserve non-OpenAI attributes from the original chunk.
"""
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
expected_keys: Final = frozenset(type(model_response).model_fields) | frozenset({"usage"})
candidate_keys: Final = tuple(
key
for key in (*type(original_chunk).model_fields, *(original_chunk.__pydantic_extra__ or {}))
if key not in expected_keys
)
candidate_keys: Final = _upstream_non_openai_keys(type(model_response), original_chunk)
if not candidate_keys:
return
candidate_values: Final[tuple[tuple[str, object], ...]] = tuple(
@ -708,7 +721,7 @@ def preserve_upstream_non_openai_attributes(
if all(value is None or isinstance(value, (str, int, float, bool)) for _, value in candidate_values):
_assign_attributes(model_response, candidate_values)
return
_assign_attributes(model_response, original_chunk.model_dump(include=frozenset(candidate_keys)).items())
_assign_attributes(model_response, _dump_keys(original_chunk, candidate_keys).items())
def _assign_attributes(target: BaseModel, values: Iterable[tuple[str, object]]) -> None:

View file

@ -57,7 +57,7 @@ from litellm.types.utils import (
from litellm.types.utils import GenericStreamingChunk as GChunk
from ..exceptions import OpenAIError, Timeout
from .core_helpers import map_finish_reason, process_response_headers
from .core_helpers import map_finish_reason, process_response_headers, upstream_non_openai_attributes
from .exception_mapping_utils import exception_type, with_deployment_api_base
from .llm_response_utils.get_api_base import get_api_base
from .rules import Rules
@ -152,6 +152,12 @@ def _skips_stream_fallback(status_code: int | None) -> bool:
return status_code is not None and 400 <= status_code < 500 and status_code != 429
def _stamp_upstream_attributes(response: ModelResponseStream, attributes: Mapping[str, object]) -> None:
for key, value in attributes.items():
if value is not None:
setattr(response, key, value)
def is_async_iterable(obj: object) -> bool:
"""
Check if an object is an async iterable (can be used with 'async for').
@ -390,6 +396,7 @@ class CustomStreamWrapper:
self.stream_options = stream_options or getattr(logging_obj, "stream_options", None)
self.messages = getattr(logging_obj, "messages", None)
self.sent_stream_usage = False
self._usage_chunk_upstream_attributes: Mapping[str, object] = MappingProxyType({})
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
self.caller_requested_stream_usage: Final = (
self.check_send_stream_usage(getattr(logging_obj, "stream_options", None))
@ -1718,6 +1725,10 @@ class CustomStreamWrapper:
model_response.choices[0].delta = plain_delta if plain_delta is not None else Delta()
plain_delta_created = plain_delta is not None # rebind-ok: the first chunk builds its delta
else:
if isinstance(original_chunk, BaseModel):
self._usage_chunk_upstream_attributes = upstream_non_openai_attributes(
ModelResponseStream, original_chunk
)
if self.send_stream_usage is True:
model_response.choices = []
return model_response
@ -2208,6 +2219,7 @@ class CustomStreamWrapper:
if self.sent_stream_usage is False and self.send_stream_usage is True:
self.sent_stream_usage = True
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
return response
self.close()
raise # Re-raise StopIteration
@ -2409,6 +2421,7 @@ class CustomStreamWrapper:
if self.sent_stream_usage is False and self.send_stream_usage is True:
self.sent_stream_usage = True
_stamp_upstream_attributes(response, self._usage_chunk_upstream_attributes)
return response
_deferred_cb: Final = getattr(

View file

@ -10171,6 +10171,7 @@ def _fast_serialize_simple_model_response_stream(
getattr(chunk, "provider_specific_fields", None) is not None
or getattr(chunk, "system_fingerprint", None) is not None
or getattr(chunk, "usage", None) is not None
or _has_provider_extras(chunk)
):
return None
@ -10246,6 +10247,20 @@ def _serialize_streaming_chunk(chunk: BaseModel) -> str | bytes:
return chunk.model_dump_json(exclude_none=True, exclude_unset=True)
_FAST_SERIALIZED_EXTRAS: Final = frozenset({"service_tier", "usage"})
def _has_provider_extras(chunk: ModelResponseStream) -> bool:
extras: Final = chunk.model_extra or {}
return any(value is not None for key, value in extras.items() if key not in _FAST_SERIALIZED_EXTRAS)
def _without_injected_usage(chunk: object) -> ModelResponseStream | None:
if not isinstance(chunk, ModelResponseStream) or not _has_provider_extras(chunk):
return None
return chunk.model_copy(update={"usage": None})
async def _apply_streaming_chunk_hooks(
*,
chunk: Any,
@ -10632,7 +10647,9 @@ async def async_data_generator(
fallback_errors=fallback_errors,
)
fallback_metadata_event_sent = True
continue
if (kept_chunk := _without_injected_usage(chunk)) is None:
continue
chunk = kept_chunk
if error_state is not None:
error_state.observe_chunk(chunk)

View file

@ -994,6 +994,52 @@ def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes(
assert getattr(returned_chunk, "custom_field", None) == {"key": "value"}
def _stream_ending_with_provider_usage_chunk() -> list[ModelResponseStream]:
def chunk(choices: list[StreamingChoices], usage: Usage | None = None) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-usage-fields",
created=1742056047,
model=None,
object="chat.completion.chunk",
choices=choices,
usage=usage,
)
usage_chunk: Final = chunk([], Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30))
setattr(usage_chunk, "provider_metrics", {"energy_wh": 0.5})
return [
chunk([StreamingChoices(index=0, delta=Delta(content="Hello!", role="assistant"))]),
chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]),
usage_chunk,
]
@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
@pytest.mark.asyncio
async def test_final_usage_chunk_keeps_provider_fields_from_usage_only_chunk(sync_mode: bool):
response: Final = CustomStreamWrapper(
completion_stream=ModelResponseListIterator(model_responses=_stream_ending_with_provider_usage_chunk()),
model="gpt-5.5",
custom_llm_provider="openai",
logging_obj=Logging(
model="gpt-5.5",
messages=[{"role": "user", "content": "Hey"}],
stream=True,
call_type="completion",
start_time=datetime.datetime.now(),
litellm_call_id="usage-fields",
function_id="usage-fields",
),
stream_options={"include_usage": True},
)
chunks: Final = list(response) if sync_mode else [chunk async for chunk in response]
assert chunks[-1].usage is not None
assert chunks[-1].usage.total_tokens == 30
assert getattr(chunks[-1], "provider_metrics", None) == {"energy_wh": 0.5}
def test_set_response_id_propagation_empty_to_valid(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):

View file

@ -553,6 +553,23 @@ def test_fast_serialize_simple_model_response_stream_with_usage_returns_none_inv
assert _fast_serialize_simple_model_response_stream(chunk) is None
def test_serialize_streaming_chunk_keeps_provider_fields():
chunk = _simple_chunk()
setattr(chunk, "provider_metrics", {"energy_wh": 0.5})
payload = json.loads(_serialize_streaming_chunk(chunk))
assert payload["provider_metrics"] == {"energy_wh": 0.5}
assert payload["choices"][0]["delta"] == {"role": "assistant", "content": "hi"}
def test_serialize_streaming_chunk_ignores_unset_provider_fields_for_fast_path():
chunk = _simple_chunk()
setattr(chunk, "citations", None)
assert isinstance(_serialize_streaming_chunk(chunk), bytes)
# ---------------------------------------------------------------------------
# _serialize_streaming_chunk
# ---------------------------------------------------------------------------
@ -686,6 +703,58 @@ def _patch_logging_flags(monkeypatch, needs_wrap=False, needs_per_chunk=False):
)
def _injected_usage_chunk(**provider_fields: object) -> ModelResponseStream:
chunk: Final = ModelResponseStream(
id="chatcmpl-test",
created=0,
model="gpt-4",
object="chat.completion.chunk",
usage=Usage(prompt_tokens=3, completion_tokens=2, total_tokens=5),
)
for key, value in provider_fields.items():
setattr(chunk, key, value)
return chunk
def _sse_payload(line: str | bytes) -> dict[str, object]:
text: Final = line.decode() if isinstance(line, bytes) else line
return json.loads(text.removeprefix("data: ").removesuffix("\n\n"))
async def _stream_with_injected_usage(usage_chunk: ModelResponseStream) -> list[dict[str, object]]:
out: Final = [
line
async for line in async_data_generator(
response=_async_iter([_simple_chunk(content="hello"), usage_chunk]),
user_api_key_dict=_user_auth(),
request_data={"model": "gpt-4", "_litellm_strip_stream_usage": True},
)
]
assert out[-1] == "data: [DONE]\n\n"
return [_sse_payload(line) for line in out[:-1]]
@pytest.mark.asyncio
async def test_async_data_generator_keeps_provider_fields_from_stripped_usage_chunk(monkeypatch):
_patch_logging_flags(monkeypatch)
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(provider_metrics={"energy_wh": 0.5}))
assert len(payloads) == 2
assert payloads[-1]["provider_metrics"] == {"energy_wh": 0.5}
assert "usage" not in payloads[-1]
@pytest.mark.asyncio
async def test_async_data_generator_drops_stripped_usage_chunk_without_provider_fields(monkeypatch):
_patch_logging_flags(monkeypatch)
payloads: Final = await _stream_with_injected_usage(_injected_usage_chunk(citations=None))
assert len(payloads) == 1
assert "usage" not in payloads[0]
@pytest.mark.asyncio
async def test_async_data_generator_yields_sse_chunks_and_done(monkeypatch):
_patch_logging_flags(monkeypatch)