From 1dda02d65b0a338417cd658633efb44fe5a97fb1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:20:31 -0700 Subject: [PATCH] fix(a2a): keep one reply text per stream and bill every artifact and status reply --- litellm/a2a_protocol/main.py | 1 - litellm/a2a_protocol/streaming_iterator.py | 21 +++-- litellm/a2a_protocol/utils.py | 7 +- .../test_a2a_streaming_iterator.py | 82 +++++++++++++++++-- 4 files changed, 89 insertions(+), 22 deletions(-) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 2cd5137526a..23a6168f35f 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -746,7 +746,6 @@ async def asend_message_streaming( request=request, logging_obj=bridge_logging_obj, agent_name=agent_name or "unknown", - output_text=A2ARequestUtils.extract_text_from_stream_event, ): yield chunk return diff --git a/litellm/a2a_protocol/streaming_iterator.py b/litellm/a2a_protocol/streaming_iterator.py index 10770ac6944..7900eea86e2 100644 --- a/litellm/a2a_protocol/streaming_iterator.py +++ b/litellm/a2a_protocol/streaming_iterator.py @@ -3,7 +3,7 @@ A2A Streaming Iterator with token tracking and logging support. """ import asyncio -from collections.abc import AsyncIterator, Callable, Mapping +from collections.abc import AsyncIterator, Mapping from datetime import datetime from typing import TYPE_CHECKING, Final @@ -53,18 +53,16 @@ class A2AStreamingIterator: request: "SendStreamingMessageRequest", logging_obj: LiteLLMLoggingObj, agent_name: str = "unknown", - output_text: Callable[[Mapping[str, object]], str] = A2ARequestUtils.extract_text_from_response, ): self.stream = stream self.request = request self.logging_obj = logging_obj self.agent_name = agent_name - self.output_text = output_text self.start_time = logging_obj.start_time # Collect chunks for token counting self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = [] - self.collected_text_parts: list[str] = [] + self.latest_message_text = "" self.artifact_texts: Mapping[str, str] = {} self.saw_error = False self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None @@ -100,22 +98,25 @@ class A2AStreamingIterator: raise def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None: - """Extract text from a streaming chunk and add to collected parts.""" + """Record the reply text a streaming chunk carries.""" try: - text: Final = self.output_text(payload) + text: Final = A2ARequestUtils.extract_text_from_stream_event(payload) if not text: return artifact_update: Final = _artifact_update(payload) if artifact_update is None: - self.collected_text_parts.append(text) + self.latest_message_text = text return artifact_id, append = artifact_update artifact_text: Final = self.artifact_texts.get(artifact_id, "") + text if append else text self.artifact_texts = {**self.artifact_texts, artifact_id: artifact_text} - self.collected_text_parts.append(artifact_text) except Exception: verbose_logger.debug("Failed to extract text from A2A streaming chunk") + def _reply_text(self) -> str: + artifact_text: Final = " ".join(text for text in self.artifact_texts.values() if text) + return artifact_text or self.latest_message_text + def _is_completed_chunk(self, payload: Mapping[str, object]) -> bool: """Check if chunk indicates stream completion.""" try: @@ -138,9 +139,7 @@ class A2AStreamingIterator: input_text: Final = A2ARequestUtils.extract_text_from_message(input_message) prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text) - # Use the last (most complete) text from chunks - output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else "" - completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text) + completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(self._reply_text()) total_tokens: Final = prompt_tokens + completion_tokens diff --git a/litellm/a2a_protocol/utils.py b/litellm/a2a_protocol/utils.py index dd5ddaa7170..a9ec0de116e 100644 --- a/litellm/a2a_protocol/utils.py +++ b/litellm/a2a_protocol/utils.py @@ -100,8 +100,13 @@ class A2ARequestUtils: @staticmethod def extract_text_from_stream_event(event: Mapping[str, object]) -> str: result: Final = as_str_mapping(event.get("result")) - if result is not None and result.get("kind") == "artifact-update": + if result is None: + return "" + if result.get("kind") == "artifact-update": return A2ARequestUtils.extract_text_from_message(result.get("artifact")) + if result.get("kind") == "status-update": + status: Final = as_str_mapping(result.get("status")) + return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else "" return A2ARequestUtils.extract_text_from_response(event) @staticmethod diff --git a/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py b/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py index ce2f241b3d9..60f5c5a338b 100644 --- a/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py +++ b/tests/unit/a2a_protocol/test_a2a_streaming_iterator.py @@ -12,6 +12,7 @@ from typing import Final from types import SimpleNamespace import pytest +from a2a.compat.v0_3.types import SendStreamingMessageResponse import litellm from litellm.a2a_protocol import streaming_iterator as a2a_streaming_iterator_module @@ -178,7 +179,9 @@ async def test_stream_logs_its_first_chunk_time_measured_from_the_call_start(): iterator: Final = A2AStreamingIterator( stream=_stream(), - request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})), + request=SimpleNamespace( + params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]}) + ), logging_obj=logging_obj, agent_name="test-agent", ) @@ -199,16 +202,36 @@ _REPLY: Final = ( ) -def _artifact_event(text: str, append: bool | None) -> Mapping[str, object]: +def _artifact_event(text: str, append: bool | None, artifact_id: str = "a1") -> Mapping[str, object]: result: Final = { "kind": "artifact-update", "taskId": "t1", "contextId": "c1", - "artifact": {"artifactId": "a1", "parts": [{"kind": "text", "text": text}]}, + "artifact": {"artifactId": artifact_id, "parts": [{"kind": "text", "text": text}]}, } return {"jsonrpc": "2.0", "id": "r1", "result": result if append is None else {**result, "append": append}} +def _status_event(state: str, text: str | None = None) -> Mapping[str, object]: + message: Final = {"kind": "message", "role": "agent", "messageId": "m1", "parts": [{"kind": "text", "text": text}]} + status: Final = {"state": state} if text is None else {"state": state, "message": message} + return { + "jsonrpc": "2.0", + "id": "r1", + "result": { + "kind": "status-update", + "taskId": "t1", + "contextId": "c1", + "final": state == "completed", + "status": status, + }, + } + + +_FIRST_ARTIFACT: Final = "Paris is the capital of France and sits on the Seine." +_SECOND_ARTIFACT: Final = "It has been the seat of the French government for most of the last thousand years." + + async def _snapshot_stream() -> AsyncIterator[Mapping[str, object]]: for end in range(40, len(_REPLY) + 40, 40): yield _artifact_event(_REPLY[:end], append=None) @@ -223,13 +246,55 @@ async def _watsonx_stream() -> AsyncIterator[Mapping[str, object]]: yield event +async def _two_artifact_stream() -> AsyncIterator[Mapping[str, object]]: + yield _artifact_event(_FIRST_ARTIFACT, append=None, artifact_id="a1") + yield _artifact_event(_SECOND_ARTIFACT, append=None, artifact_id="a2") + yield _status_event("completed") + + +async def _status_message_stream() -> AsyncIterator[Mapping[str, object]]: + yield _status_event("working", "Looking that up") + yield _status_event("working", _REPLY) + yield _status_event("completed") + + +async def _artifact_without_id_stream() -> AsyncIterator[Mapping[str, object]]: + yield { + "jsonrpc": "2.0", + "id": "r1", + "result": { + "kind": "artifact-update", + "taskId": "t1", + "contextId": "c1", + "artifact": {"parts": [{"kind": "text", "text": _REPLY}]}, + }, + } + yield _status_event("completed") + + +async def _url_agent_sdk_stream() -> AsyncIterator[SendStreamingMessageResponse]: + for event in ( + _status_event("working", "Looking that up"), + _artifact_event(_REPLY, append=False), + _status_event("completed"), + ): + yield SendStreamingMessageResponse.model_validate(event) + + @pytest.mark.parametrize( - "agent_stream", - (pytest.param(_snapshot_stream, id="replacing_snapshots"), pytest.param(_watsonx_stream, id="watsonx_chunks")), + ("agent_stream", "reply"), + ( + pytest.param(_snapshot_stream, _REPLY, id="replacing_snapshots"), + pytest.param(_watsonx_stream, _REPLY, id="watsonx_chunks"), + pytest.param(_two_artifact_stream, f"{_FIRST_ARTIFACT} {_SECOND_ARTIFACT}", id="two_artifacts"), + pytest.param(_status_message_stream, _REPLY, id="status_message_reply"), + pytest.param(_artifact_without_id_stream, _REPLY, id="artifact_without_id"), + pytest.param(_url_agent_sdk_stream, _REPLY, id="url_agent_sdk_events"), + ), ) @pytest.mark.asyncio -async def test_stream_counts_the_whole_artifact_once( - agent_stream: Callable[[], AsyncIterator[Mapping[str, object]]], monkeypatch: pytest.MonkeyPatch +async def test_stream_counts_the_whole_reply_once( + agent_stream: Callable[[], AsyncIterator[object]], reply: str, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(litellm, "success_callback", []) monkeypatch.setattr(litellm, "_async_success_callback", []) @@ -247,9 +312,8 @@ async def test_stream_counts_the_whole_artifact_once( request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": []})), logging_obj=logging_obj, agent_name="trip-planner", - output_text=A2ARequestUtils.extract_text_from_stream_event, ) _ = [chunk async for chunk in iterator] - assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(_REPLY) + assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(reply)