fix(a2a): bill streams the client drops and skip the bridge's working placeholder

A stream the client closes or cancels mid-reply now logs its spend row at the agent's prices on both the bridge and URL paths, and a stream dropped before the agent sends anything still logs nothing. The bridge's "Processing request..." status no longer counts as reply text, so an empty bridge reply bills no completion tokens, as send does. Adds proxy-level tests that a bridge stream bills under the agent's card name.
This commit is contained in:
mateo-berri 2026-10-30 09:59:55 -07:00
parent 8a989cc795
commit 6684cbfece
6 changed files with 245 additions and 25 deletions

View file

@ -22,7 +22,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
A2AStreamingContext,
)
from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS
from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS, A2A_COMPLETION_BRIDGE_WORKING_MESSAGE
from litellm.interactions.agents.utils import merge_agent_headers
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.types.utils import ModelResponse
@ -268,7 +268,7 @@ class A2ACompletionBridgeHandler:
ctx=ctx,
state="working",
final=False,
message_text="Processing request...",
message_text=A2A_COMPLETION_BRIDGE_WORKING_MESSAGE,
)
yield working_event

View file

@ -741,13 +741,18 @@ async def asend_message_streaming(
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
async for chunk in A2AStreamingIterator(
bridge_iterator: Final = A2AStreamingIterator(
stream=bridge_stream,
request=request,
logging_obj=bridge_logging_obj,
agent_name=agent_name or "unknown",
):
yield chunk
)
try:
async for chunk in bridge_iterator:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
await bridge_iterator.abill_on_disconnect()
raise
return
if request is None:
@ -805,13 +810,18 @@ async def asend_message_streaming(
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
async for chunk in A2AStreamingIterator(
url_agent_iterator: Final = A2AStreamingIterator(
stream=stream,
request=request,
logging_obj=logging_obj,
agent_name=card_agent_name,
):
yield chunk
)
try:
async for chunk in url_agent_iterator:
yield chunk
except (GeneratorExit, asyncio.CancelledError):
await url_agent_iterator.abill_on_disconnect()
raise
async def create_a2a_client(

View file

@ -7,12 +7,14 @@ from collections.abc import AsyncIterator, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final
import anyio
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import A2A_COMPLETION_BRIDGE_WORKING_MESSAGE
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -91,12 +93,15 @@ class A2AStreamingIterator:
return chunk
except StopAsyncIteration:
# Stream ended - handle logging
if self.final_chunk is None and self.chunks:
self.final_chunk = self.chunks[-1]
await self._handle_stream_complete()
raise
async def abill_on_disconnect(self) -> None:
if not self.chunks:
return
with anyio.CancelScope(shield=True):
await self._handle_stream_complete()
def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None:
"""Record the reply text a streaming chunk carries."""
try:
@ -104,6 +109,8 @@ class A2AStreamingIterator:
if not text:
return
artifact_update: Final = _artifact_update(payload)
if artifact_update is None and text == A2A_COMPLETION_BRIDGE_WORKING_MESSAGE:
return
if artifact_update is None:
self.latest_message_text = text
return
@ -131,6 +138,8 @@ class A2AStreamingIterator:
async def _handle_stream_complete(self) -> None:
"""Handle logging and token counting when stream completes."""
if self.final_chunk is None and self.chunks:
self.final_chunk = self.chunks[-1]
try:
end_time: Final = datetime.now()

View file

@ -886,6 +886,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
)
A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
A2A_COMPLETION_BRIDGE_WORKING_MESSAGE: Final = "Processing request..."
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
"openai",

View file

@ -1,9 +1,10 @@
"""Tests for litellm/a2a_protocol/main.py non-streaming send behavior."""
import asyncio
import contextlib
import datetime
import json
from collections.abc import Callable, Iterator, Mapping
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterator, Mapping
from typing import Final, TypeAlias
import httpx
@ -1026,23 +1027,25 @@ async def test_agentcore_stream_bills_every_appended_reply_chunk(
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, _per_token_cost, _LONG_AGENT_REPLY)
_PYDANTIC_COMPLETED_TASK: Final[Mapping[str, object]] = {
"jsonrpc": "2.0",
"id": "bridge-stream",
"result": {
"kind": "task",
"id": "t1",
"contextId": "c1",
"status": {"state": "completed"},
"artifacts": [{"artifactId": "a1", "parts": [{"kind": "text", "text": _LONG_AGENT_REPLY}]}],
},
}
@pytest.mark.asyncio
async def test_pydantic_ai_stream_bills_the_whole_chunked_reply(
a2a_spend_recorder: _A2ASpendRecorder, respx_reachable_httpx: None
) -> None:
completed_task: Final = {
"jsonrpc": "2.0",
"id": "bridge-stream",
"result": {
"kind": "task",
"id": "t1",
"contextId": "c1",
"status": {"state": "completed"},
"artifacts": [{"artifactId": "a1", "parts": [{"kind": "text", "text": _LONG_AGENT_REPLY}]}],
},
}
with respx.mock(assert_all_called=True) as router:
router.post("http://127.0.0.1:9/").mock(return_value=httpx.Response(200, json=completed_task))
router.post("http://127.0.0.1:9/").mock(return_value=httpx.Response(200, json=_PYDANTIC_COMPLETED_TASK))
await _drain_bridge_stream({"custom_llm_provider": "pydantic_ai_agents", **_PER_TOKEN_PRICES})
spend: Final = await a2a_spend_recorder.logged_spend()
@ -1050,6 +1053,125 @@ async def test_pydantic_ai_stream_bills_the_whole_chunked_reply(
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, _per_token_cost, _LONG_AGENT_REPLY)
async def _close_after_first_event(stream: AsyncGenerator[object, None]) -> None:
_ = await stream.__anext__()
await stream.aclose()
async def _cancel_reader_after_first_event(stream: AsyncGenerator[object, None]) -> None:
first_event: Final = asyncio.Event()
async def _read() -> None:
async for _ in stream:
first_event.set()
reader: Final = asyncio.create_task(_read())
await first_event.wait()
reader.cancel()
with contextlib.suppress(asyncio.CancelledError):
await reader
def _pydantic_bridge_agent_stream() -> AsyncGenerator[object, None]:
from litellm.a2a_protocol.main import asend_message_streaming
return asend_message_streaming(
request=SendStreamingMessageRequest(id="bridge-stream", params=_bridge_message_params()),
api_base="http://127.0.0.1:9",
litellm_params={"custom_llm_provider": "pydantic_ai_agents", "cost_per_query": 0.25},
agent_id="agent-1",
agent_name="trip-planner",
)
def _url_agent_stream() -> AsyncGenerator[object, None]:
from litellm.a2a_protocol.main import asend_message_streaming
return asend_message_streaming(
request=SendStreamingMessageRequest(id="reply", params=_bridge_message_params()),
api_base="http://127.0.0.1:9",
litellm_params={"cost_per_query": 0.25},
agent_id="agent-1",
)
@pytest.mark.parametrize(
("open_stream", "billed_model"),
(
pytest.param(_pydantic_bridge_agent_stream, "a2a_agent/trip-planner", id="bridge_agent"),
pytest.param(_url_agent_stream, f"a2a_agent/{_AGENT_CARD['name']}", id="url_agent"),
),
)
@pytest.mark.parametrize(
"drop_stream",
(
pytest.param(_close_after_first_event, id="proxy_closes_the_stream"),
pytest.param(_cancel_reader_after_first_event, id="reader_cancelled_mid_stream"),
),
)
@pytest.mark.asyncio
async def test_stream_the_client_drops_still_bills_the_agents_price(
open_stream: Callable[[], AsyncGenerator[object, None]],
billed_model: str,
drop_stream: Callable[[AsyncGenerator[object, None]], Coroutine[object, object, None]],
a2a_spend_recorder: _A2ASpendRecorder,
isolated_client_cache: LLMClientCache,
respx_reachable_httpx: None,
) -> None:
_ = await _seed_shared_a2a_client(rpc_stream_events=_LANGGRAPH_STREAM_EVENTS)
with respx.mock(assert_all_called=False) as router:
router.post("http://127.0.0.1:9/").mock(return_value=httpx.Response(200, json=_PYDANTIC_COMPLETED_TASK))
await drop_stream(open_stream())
spend: Final = await a2a_spend_recorder.logged_spend()
assert spend.response_cost == pytest.approx(0.25)
assert spend.model == billed_model
class _AgentThatNeverAnswers(_RequestRecorder):
def __init__(self) -> None:
super().__init__()
self.message_sent: Final = asyncio.Event()
def __call__(self, request: httpx.Request) -> httpx.Response | Coroutine[object, object, httpx.Response]:
if request.method == "GET":
return super().__call__(request)
return self._never_answer()
async def _never_answer(self) -> httpx.Response:
self.message_sent.set()
reply: Final[asyncio.Future[httpx.Response]] = asyncio.get_running_loop().create_future()
return await reply
@pytest.mark.asyncio
async def test_stream_dropped_before_the_agent_answers_bills_nothing(
a2a_spend_recorder: _A2ASpendRecorder, isolated_client_cache: LLMClientCache
) -> None:
agent: Final = _AgentThatNeverAnswers()
_ = await _seed_shared_a2a_client(recorder=agent)
stream: Final = _url_agent_stream()
reader: Final = asyncio.create_task(anext(stream))
await asyncio.wait_for(agent.message_sent.wait(), timeout=10)
reader.cancel()
with contextlib.suppress(asyncio.CancelledError):
await reader
with pytest.raises(TimeoutError):
await asyncio.wait_for(a2a_spend_recorder.logged.wait(), timeout=2)
@pytest.mark.asyncio
async def test_completion_bridge_stream_with_an_empty_reply_bills_no_reply_tokens(
a2a_spend_recorder: _A2ASpendRecorder,
) -> None:
await _drain_bridge_stream({**_bridge_agent_params(_PER_TOKEN_PRICES), "mock_response": ""})
spend: Final = await a2a_spend_recorder.logged_spend()
assert spend.completion_tokens == 0
assert spend.response_cost == pytest.approx(_per_token_cost(spend.prompt_tokens, 0))
@pytest.mark.parametrize(
"upstream_reply",
(

View file

@ -5,6 +5,7 @@ Tests that invoke_agent_a2a properly integrates with add_litellm_data_to_request
"""
import asyncio
import importlib.util
import json
import socket
import sys
@ -17,8 +18,10 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import respx
from pydantic import BaseModel, ConfigDict
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.agents import AgentCaller
@ -44,6 +47,7 @@ def _sse_upstream_response(*lines: str, first_line_delay: float = 0.0) -> httpx.
class CapturedAgentCall:
request_id: object
agent_extra_headers: dict[str, str] | None
agent_name: object
@pytest.mark.asyncio
@ -627,7 +631,11 @@ async def _invoke_message_method(
kwargs: Final = downstream.call_args.kwargs
request_id: Final = kwargs["request"].__dict__["id"] if is_send else kwargs["request_id"]
return CapturedAgentCall(request_id=request_id, agent_extra_headers=kwargs.get("agent_extra_headers"))
return CapturedAgentCall(
request_id=request_id,
agent_extra_headers=kwargs.get("agent_extra_headers"),
agent_name=kwargs.get("agent_name"),
)
@pytest.mark.asyncio
@ -1684,6 +1692,76 @@ def test_build_message_send_params_proto_fallback_ignores_unknown_fields():
assert result.message.role.value == "user"
@pytest.mark.asyncio
async def test_message_stream_names_the_agent_by_its_card_name():
agent: Final = _make_agent_mock()
agent.agent_card_params = {"url": "http://backend-agent:10001", "name": "trip-planner"}
captured: Final = await _invoke_message_method(
"message/stream",
_make_request_mock("message/stream", _HELLO_MESSAGE_PARAMS),
UserAPIKeyAuth(api_key="sk-test", user_id="u1"),
agent=agent,
)
assert captured.agent_name == "trip-planner"
class _StreamedSpend(BaseModel):
model_config = ConfigDict(frozen=True)
call_type: str
model: str
response_cost: float
class _StreamedSpendRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.logged: Final = asyncio.Event()
self.spend: _StreamedSpend | None = None
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
spend: Final = _StreamedSpend.model_validate(kwargs["standard_logging_object"])
if spend.call_type != "asend_message_streaming":
return
self.spend = spend
self.logged.set()
@pytest.mark.skipif(importlib.util.find_spec("a2a") is None, reason="needs the optional a2a-sdk")
@pytest.mark.asyncio
async def test_handle_stream_message_bills_a_bridge_agent_under_its_name(monkeypatch: pytest.MonkeyPatch):
from litellm.proxy.agent_endpoints.a2a_endpoints import _handle_stream_message
recorder: Final = _StreamedSpendRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
monkeypatch.setattr(litellm, "success_callback", [recorder])
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
response: Final = await _handle_stream_message(
api_base=None,
request_id="req-1",
params=dict(_HELLO_MESSAGE_PARAMS),
litellm_params={
"custom_llm_provider": "langgraph",
"model": "agent",
"mock_response": "pong",
"cost_per_query": 0.25,
},
agent_id="agent-1",
agent_name="trip-planner",
)
_ = [chunk async for chunk in response.body_iterator]
await asyncio.wait_for(recorder.logged.wait(), timeout=10)
assert recorder.spend == _StreamedSpend(
call_type="asend_message_streaming", model="a2a_agent/trip-planner", response_cost=0.25
)
@pytest.mark.asyncio
async def test_handle_stream_message_rejects_invalid_params_with_32602():
from litellm.proxy.agent_endpoints.a2a_endpoints import _handle_stream_message