mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(a2a): keep one reply text per stream and bill every artifact and status reply
This commit is contained in:
parent
4512624633
commit
1dda02d65b
4 changed files with 89 additions and 22 deletions
|
|
@ -746,7 +746,6 @@ async def asend_message_streaming(
|
|||
request=request,
|
||||
logging_obj=bridge_logging_obj,
|
||||
agent_name=agent_name or "unknown",
|
||||
output_text=A2ARequestUtils.extract_text_from_stream_event,
|
||||
):
|
||||
yield chunk
|
||||
return
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ A2A Streaming Iterator with token tracking and logging support.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
|
|
@ -53,18 +53,16 @@ class A2AStreamingIterator:
|
|||
request: "SendStreamingMessageRequest",
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
agent_name: str = "unknown",
|
||||
output_text: Callable[[Mapping[str, object]], str] = A2ARequestUtils.extract_text_from_response,
|
||||
):
|
||||
self.stream = stream
|
||||
self.request = request
|
||||
self.logging_obj = logging_obj
|
||||
self.agent_name = agent_name
|
||||
self.output_text = output_text
|
||||
self.start_time = logging_obj.start_time
|
||||
|
||||
# Collect chunks for token counting
|
||||
self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = []
|
||||
self.collected_text_parts: list[str] = []
|
||||
self.latest_message_text = ""
|
||||
self.artifact_texts: Mapping[str, str] = {}
|
||||
self.saw_error = False
|
||||
self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None
|
||||
|
|
@ -100,22 +98,25 @@ class A2AStreamingIterator:
|
|||
raise
|
||||
|
||||
def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None:
|
||||
"""Extract text from a streaming chunk and add to collected parts."""
|
||||
"""Record the reply text a streaming chunk carries."""
|
||||
try:
|
||||
text: Final = self.output_text(payload)
|
||||
text: Final = A2ARequestUtils.extract_text_from_stream_event(payload)
|
||||
if not text:
|
||||
return
|
||||
artifact_update: Final = _artifact_update(payload)
|
||||
if artifact_update is None:
|
||||
self.collected_text_parts.append(text)
|
||||
self.latest_message_text = text
|
||||
return
|
||||
artifact_id, append = artifact_update
|
||||
artifact_text: Final = self.artifact_texts.get(artifact_id, "") + text if append else text
|
||||
self.artifact_texts = {**self.artifact_texts, artifact_id: artifact_text}
|
||||
self.collected_text_parts.append(artifact_text)
|
||||
except Exception:
|
||||
verbose_logger.debug("Failed to extract text from A2A streaming chunk")
|
||||
|
||||
def _reply_text(self) -> str:
|
||||
artifact_text: Final = " ".join(text for text in self.artifact_texts.values() if text)
|
||||
return artifact_text or self.latest_message_text
|
||||
|
||||
def _is_completed_chunk(self, payload: Mapping[str, object]) -> bool:
|
||||
"""Check if chunk indicates stream completion."""
|
||||
try:
|
||||
|
|
@ -138,9 +139,7 @@ class A2AStreamingIterator:
|
|||
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
|
||||
prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
|
||||
|
||||
# Use the last (most complete) text from chunks
|
||||
output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
|
||||
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
|
||||
completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(self._reply_text())
|
||||
|
||||
total_tokens: Final = prompt_tokens + completion_tokens
|
||||
|
||||
|
|
|
|||
|
|
@ -100,8 +100,13 @@ class A2ARequestUtils:
|
|||
@staticmethod
|
||||
def extract_text_from_stream_event(event: Mapping[str, object]) -> str:
|
||||
result: Final = as_str_mapping(event.get("result"))
|
||||
if result is not None and result.get("kind") == "artifact-update":
|
||||
if result is None:
|
||||
return ""
|
||||
if result.get("kind") == "artifact-update":
|
||||
return A2ARequestUtils.extract_text_from_message(result.get("artifact"))
|
||||
if result.get("kind") == "status-update":
|
||||
status: Final = as_str_mapping(result.get("status"))
|
||||
return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
|
||||
return A2ARequestUtils.extract_text_from_response(event)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from typing import Final
|
|||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from a2a.compat.v0_3.types import SendStreamingMessageResponse
|
||||
|
||||
import litellm
|
||||
from litellm.a2a_protocol import streaming_iterator as a2a_streaming_iterator_module
|
||||
|
|
@ -178,7 +179,9 @@ async def test_stream_logs_its_first_chunk_time_measured_from_the_call_start():
|
|||
|
||||
iterator: Final = A2AStreamingIterator(
|
||||
stream=_stream(),
|
||||
request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})),
|
||||
request=SimpleNamespace(
|
||||
params=SimpleNamespace(message={"role": "user", "parts": [{"kind": "text", "text": "hi"}]})
|
||||
),
|
||||
logging_obj=logging_obj,
|
||||
agent_name="test-agent",
|
||||
)
|
||||
|
|
@ -199,16 +202,36 @@ _REPLY: Final = (
|
|||
)
|
||||
|
||||
|
||||
def _artifact_event(text: str, append: bool | None) -> Mapping[str, object]:
|
||||
def _artifact_event(text: str, append: bool | None, artifact_id: str = "a1") -> Mapping[str, object]:
|
||||
result: Final = {
|
||||
"kind": "artifact-update",
|
||||
"taskId": "t1",
|
||||
"contextId": "c1",
|
||||
"artifact": {"artifactId": "a1", "parts": [{"kind": "text", "text": text}]},
|
||||
"artifact": {"artifactId": artifact_id, "parts": [{"kind": "text", "text": text}]},
|
||||
}
|
||||
return {"jsonrpc": "2.0", "id": "r1", "result": result if append is None else {**result, "append": append}}
|
||||
|
||||
|
||||
def _status_event(state: str, text: str | None = None) -> Mapping[str, object]:
|
||||
message: Final = {"kind": "message", "role": "agent", "messageId": "m1", "parts": [{"kind": "text", "text": text}]}
|
||||
status: Final = {"state": state} if text is None else {"state": state, "message": message}
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": "r1",
|
||||
"result": {
|
||||
"kind": "status-update",
|
||||
"taskId": "t1",
|
||||
"contextId": "c1",
|
||||
"final": state == "completed",
|
||||
"status": status,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
_FIRST_ARTIFACT: Final = "Paris is the capital of France and sits on the Seine."
|
||||
_SECOND_ARTIFACT: Final = "It has been the seat of the French government for most of the last thousand years."
|
||||
|
||||
|
||||
async def _snapshot_stream() -> AsyncIterator[Mapping[str, object]]:
|
||||
for end in range(40, len(_REPLY) + 40, 40):
|
||||
yield _artifact_event(_REPLY[:end], append=None)
|
||||
|
|
@ -223,13 +246,55 @@ async def _watsonx_stream() -> AsyncIterator[Mapping[str, object]]:
|
|||
yield event
|
||||
|
||||
|
||||
async def _two_artifact_stream() -> AsyncIterator[Mapping[str, object]]:
|
||||
yield _artifact_event(_FIRST_ARTIFACT, append=None, artifact_id="a1")
|
||||
yield _artifact_event(_SECOND_ARTIFACT, append=None, artifact_id="a2")
|
||||
yield _status_event("completed")
|
||||
|
||||
|
||||
async def _status_message_stream() -> AsyncIterator[Mapping[str, object]]:
|
||||
yield _status_event("working", "Looking that up")
|
||||
yield _status_event("working", _REPLY)
|
||||
yield _status_event("completed")
|
||||
|
||||
|
||||
async def _artifact_without_id_stream() -> AsyncIterator[Mapping[str, object]]:
|
||||
yield {
|
||||
"jsonrpc": "2.0",
|
||||
"id": "r1",
|
||||
"result": {
|
||||
"kind": "artifact-update",
|
||||
"taskId": "t1",
|
||||
"contextId": "c1",
|
||||
"artifact": {"parts": [{"kind": "text", "text": _REPLY}]},
|
||||
},
|
||||
}
|
||||
yield _status_event("completed")
|
||||
|
||||
|
||||
async def _url_agent_sdk_stream() -> AsyncIterator[SendStreamingMessageResponse]:
|
||||
for event in (
|
||||
_status_event("working", "Looking that up"),
|
||||
_artifact_event(_REPLY, append=False),
|
||||
_status_event("completed"),
|
||||
):
|
||||
yield SendStreamingMessageResponse.model_validate(event)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"agent_stream",
|
||||
(pytest.param(_snapshot_stream, id="replacing_snapshots"), pytest.param(_watsonx_stream, id="watsonx_chunks")),
|
||||
("agent_stream", "reply"),
|
||||
(
|
||||
pytest.param(_snapshot_stream, _REPLY, id="replacing_snapshots"),
|
||||
pytest.param(_watsonx_stream, _REPLY, id="watsonx_chunks"),
|
||||
pytest.param(_two_artifact_stream, f"{_FIRST_ARTIFACT} {_SECOND_ARTIFACT}", id="two_artifacts"),
|
||||
pytest.param(_status_message_stream, _REPLY, id="status_message_reply"),
|
||||
pytest.param(_artifact_without_id_stream, _REPLY, id="artifact_without_id"),
|
||||
pytest.param(_url_agent_sdk_stream, _REPLY, id="url_agent_sdk_events"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_counts_the_whole_artifact_once(
|
||||
agent_stream: Callable[[], AsyncIterator[Mapping[str, object]]], monkeypatch: pytest.MonkeyPatch
|
||||
async def test_stream_counts_the_whole_reply_once(
|
||||
agent_stream: Callable[[], AsyncIterator[object]], reply: str, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [])
|
||||
|
|
@ -247,9 +312,8 @@ async def test_stream_counts_the_whole_artifact_once(
|
|||
request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": []})),
|
||||
logging_obj=logging_obj,
|
||||
agent_name="trip-planner",
|
||||
output_text=A2ARequestUtils.extract_text_from_stream_event,
|
||||
)
|
||||
|
||||
_ = [chunk async for chunk in iterator]
|
||||
|
||||
assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(_REPLY)
|
||||
assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(reply)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue