fix(a2a): keep one reply text per stream and bill every artifact and status reply

This commit is contained in:
mateo-berri 2026-10-29 23:20:31 -07:00
parent 4512624633
commit 1dda02d65b
4 changed files with 89 additions and 22 deletions

View file

@ -746,7 +746,6 @@ async def asend_message_streaming(
request=request,
logging_obj=bridge_logging_obj,
agent_name=agent_name or "unknown",
output_text=A2ARequestUtils.extract_text_from_stream_event,
):
yield chunk
return

View file

@ -3,7 +3,7 @@ A2A Streaming Iterator with token tracking and logging support.
"""
import asyncio
from collections.abc import AsyncIterator, Callable, Mapping
from collections.abc import AsyncIterator, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final
@ -53,18 +53,16 @@ class A2AStreamingIterator:
request: "SendStreamingMessageRequest",
logging_obj: LiteLLMLoggingObj,
agent_name: str = "unknown",
output_text: Callable[[Mapping[str, object]], str] = A2ARequestUtils.extract_text_from_response,
):
self.stream = stream
self.request = request
self.logging_obj = logging_obj
self.agent_name = agent_name
self.output_text = output_text
self.start_time = logging_obj.start_time
# Collect chunks for token counting
self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = []
self.collected_text_parts: list[str] = []
self.latest_message_text = ""
self.artifact_texts: Mapping[str, str] = {}
self.saw_error = False
self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None
@ -100,22 +98,25 @@ class A2AStreamingIterator:
raise
def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None:
"""Extract text from a streaming chunk and add to collected parts."""
"""Record the reply text a streaming chunk carries."""
try:
text: Final = self.output_text(payload)
text: Final = A2ARequestUtils.extract_text_from_stream_event(payload)
if not text:
return
artifact_update: Final = _artifact_update(payload)
if artifact_update is None:
self.collected_text_parts.append(text)
self.latest_message_text = text
return
artifact_id, append = artifact_update
artifact_text: Final = self.artifact_texts.get(artifact_id, "") + text if append else text
self.artifact_texts = {**self.artifact_texts, artifact_id: artifact_text}
self.collected_text_parts.append(artifact_text)
except Exception:
verbose_logger.debug("Failed to extract text from A2A streaming chunk")
def _reply_text(self) -> str:
artifact_text: Final = " ".join(text for text in self.artifact_texts.values() if text)
return artifact_text or self.latest_message_text
def _is_completed_chunk(self, payload: Mapping[str, object]) -> bool:
"""Check if chunk indicates stream completion."""
try:
@ -138,9 +139,7 @@ class A2AStreamingIterator:
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
# Use the last (most complete) text from chunks
output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(self._reply_text())
total_tokens: Final = prompt_tokens + completion_tokens

View file

@ -100,8 +100,13 @@ class A2ARequestUtils:
@staticmethod
def extract_text_from_stream_event(event: Mapping[str, object]) -> str:
result: Final = as_str_mapping(event.get("result"))
if result is not None and result.get("kind") == "artifact-update":
if result is None:
return ""
if result.get("kind") == "artifact-update":
return A2ARequestUtils.extract_text_from_message(result.get("artifact"))
if result.get("kind") == "status-update":
status: Final = as_str_mapping(result.get("status"))
return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
return A2ARequestUtils.extract_text_from_response(event)
@staticmethod

View file

@ -12,6 +12,7 @@ from typing import Final
from types import SimpleNamespace
import pytest
from a2a.compat.v0_3.types import SendStreamingMessageResponse
import litellm
from litellm.a2a_protocol import streaming_iterator as a2a_streaming_iterator_module
@ -178,7 +179,9 @@ async def test_stream_logs_its_first_chunk_time_measured_from_the_call_start():
iterator: Final = A2AStreamingIterator(
stream=_stream(),
request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})),
request=SimpleNamespace(
params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})
),
logging_obj=logging_obj,
agent_name="test-agent",
)
@ -199,16 +202,36 @@ _REPLY: Final = (
)
def _artifact_event(text: str, append: bool | None) -> Mapping[str, object]:
def _artifact_event(text: str, append: bool | None, artifact_id: str = "a1") -> Mapping[str, object]:
result: Final = {
"kind": "artifact-update",
"taskId": "t1",
"contextId": "c1",
"artifact": {"artifactId": "a1", "parts": [{"kind": "text", "text": text}]},
"artifact": {"artifactId": artifact_id, "parts": [{"kind": "text", "text": text}]},
}
return {"jsonrpc": "2.0", "id": "r1", "result": result if append is None else {**result, "append": append}}
def _status_event(state: str, text: str | None = None) -> Mapping[str, object]:
message: Final = {"kind": "message", "role": "agent", "messageId": "m1", "parts": [{"kind": "text", "text": text}]}
status: Final = {"state": state} if text is None else {"state": state, "message": message}
return {
"jsonrpc": "2.0",
"id": "r1",
"result": {
"kind": "status-update",
"taskId": "t1",
"contextId": "c1",
"final": state == "completed",
"status": status,
},
}
_FIRST_ARTIFACT: Final = "Paris is the capital of France and sits on the Seine."
_SECOND_ARTIFACT: Final = "It has been the seat of the French government for most of the last thousand years."
async def _snapshot_stream() -> AsyncIterator[Mapping[str, object]]:
for end in range(40, len(_REPLY) + 40, 40):
yield _artifact_event(_REPLY[:end], append=None)
@ -223,13 +246,55 @@ async def _watsonx_stream() -> AsyncIterator[Mapping[str, object]]:
yield event
async def _two_artifact_stream() -> AsyncIterator[Mapping[str, object]]:
yield _artifact_event(_FIRST_ARTIFACT, append=None, artifact_id="a1")
yield _artifact_event(_SECOND_ARTIFACT, append=None, artifact_id="a2")
yield _status_event("completed")
async def _status_message_stream() -> AsyncIterator[Mapping[str, object]]:
yield _status_event("working", "Looking that up")
yield _status_event("working", _REPLY)
yield _status_event("completed")
async def _artifact_without_id_stream() -> AsyncIterator[Mapping[str, object]]:
yield {
"jsonrpc": "2.0",
"id": "r1",
"result": {
"kind": "artifact-update",
"taskId": "t1",
"contextId": "c1",
"artifact": {"parts": [{"kind": "text", "text": _REPLY}]},
},
}
yield _status_event("completed")
async def _url_agent_sdk_stream() -> AsyncIterator[SendStreamingMessageResponse]:
for event in (
_status_event("working", "Looking that up"),
_artifact_event(_REPLY, append=False),
_status_event("completed"),
):
yield SendStreamingMessageResponse.model_validate(event)
@pytest.mark.parametrize(
"agent_stream",
(pytest.param(_snapshot_stream, id="replacing_snapshots"), pytest.param(_watsonx_stream, id="watsonx_chunks")),
("agent_stream", "reply"),
(
pytest.param(_snapshot_stream, _REPLY, id="replacing_snapshots"),
pytest.param(_watsonx_stream, _REPLY, id="watsonx_chunks"),
pytest.param(_two_artifact_stream, f"{_FIRST_ARTIFACT} {_SECOND_ARTIFACT}", id="two_artifacts"),
pytest.param(_status_message_stream, _REPLY, id="status_message_reply"),
pytest.param(_artifact_without_id_stream, _REPLY, id="artifact_without_id"),
pytest.param(_url_agent_sdk_stream, _REPLY, id="url_agent_sdk_events"),
),
)
@pytest.mark.asyncio
async def test_stream_counts_the_whole_artifact_once(
agent_stream: Callable[[], AsyncIterator[Mapping[str, object]]], monkeypatch: pytest.MonkeyPatch
async def test_stream_counts_the_whole_reply_once(
agent_stream: Callable[[], AsyncIterator[object]], reply: str, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "success_callback", [])
monkeypatch.setattr(litellm, "_async_success_callback", [])
@ -247,9 +312,8 @@ async def test_stream_counts_the_whole_artifact_once(
request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": []})),
logging_obj=logging_obj,
agent_name="trip-planner",
output_text=A2ARequestUtils.extract_text_from_stream_event,
)
_ = [chunk async for chunk in iterator]
assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(_REPLY)
assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(reply)