fix(a2a): bill completion-bridge agents at their own prices on /a2a send and stream

message/send on an agent served through the completion bridge (custom_llm_provider
agents such as LangGraph or databricks_agent) logged a $0, 0-token row, and
message/stream logged no row at all, so cost_per_query and per-token prices never
reached spend.

The bridge send path now records counted usage, the agent's pricing keys, and the
agent id on its logging object, the same way URL agents do. The bridge stream is
wrapped in the A2A streaming logger, which now reads the bridge's artifact event
for the reply text and gets the agent name from the proxy. Streamed calls also
count prompt tokens again, since the message text reader now reads pydantic
messages as dicts.
This commit is contained in:
mateo-berri 2026-10-29 02:03:51 -07:00
parent ff22f3de06
commit e2d40a830d
5 changed files with 195 additions and 54 deletions

View file

@ -191,6 +191,22 @@ def _set_litellm_params_on_logging_obj(
}
async def _record_send_message_billing(
kwargs: Mapping[str, object],
request: "SendMessageRequest",
response_dict: Mapping[str, object],
litellm_params: Mapping[str, object],
agent_id: str | None,
) -> None:
prompt_tokens, completion_tokens, _ = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
request=request,
response_dict=response_dict,
)
_set_usage_on_logging_obj(kwargs=kwargs, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
def _get_a2a_model_info(a2a_client: "A2AClientType", kwargs: Mapping[str, object]) -> str:
"""
Extract agent info and set model/custom_llm_provider for cost tracking.
@ -483,13 +499,21 @@ async def asend_message(
if custom_llm_provider:
if request is None:
raise ValueError("request is required for completion bridge")
return await _send_message_via_completion_bridge(
bridge_response: Final = await _send_message_via_completion_bridge(
request=request,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
litellm_params=litellm_params,
agent_extra_headers=agent_extra_headers,
)
await _record_send_message_billing(
kwargs=kwargs,
request=request,
response_dict=bridge_response.model_dump(mode="json", exclude_none=True),
litellm_params=litellm_params,
agent_id=agent_id,
)
return bridge_response
# Standard A2A client flow
if request is None:
@ -538,30 +562,14 @@ async def asend_message(
# Wrap in LiteLLM response type for _hidden_params support
response: Final = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id))
# Calculate token usage from request and response
response_dict: Final[dict[str, object]] = a2a_response.root.model_dump(mode="json", exclude_none=True)
(
prompt_tokens,
completion_tokens,
_,
) = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
request=request,
response_dict=response_dict,
)
# Set usage on logging obj for standard logging payload
_set_usage_on_logging_obj(
await _record_send_message_billing(
kwargs=kwargs,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
request=request,
response_dict=a2a_response.root.model_dump(mode="json", exclude_none=True),
litellm_params=litellm_params,
agent_id=agent_id,
)
# Merge agent pricing params into the logging obj so cost is calculated
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
# Set agent_id on logging obj for SpendLogs tracking
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
return response
@ -646,6 +654,7 @@ async def asend_message_streaming(
metadata: dict[str, object] | None = None,
proxy_server_request: dict[str, object] | None = None,
agent_extra_headers: dict[str, str] | None = None,
agent_name: str | None = None,
**kwargs: object,
) -> AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"]:
"""
@ -661,6 +670,7 @@ async def asend_message_streaming(
agent_id: Optional agent ID for tracking in SpendLogs
metadata: Optional metadata dict (contains user_api_key, user_id, team_id, etc.)
proxy_server_request: Optional proxy server request data
agent_name: Agent name logged as a2a_agent/<agent_name> for completion bridge agents
Yields:
SendStreamingMessageResponse chunks from the agent
@ -685,6 +695,7 @@ async def asend_message_streaming(
print(chunk)
```
"""
called_at: Final = datetime.datetime.now()
litellm_params = litellm_params or {}
custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
@ -705,12 +716,36 @@ async def asend_message_streaming(
request.params.model_dump(mode="json") if hasattr(request.params, "model_dump") else dict(request.params)
)
async for chunk in A2ACompletionBridgeHandler.handle_streaming(
bridge_stream: Final = A2ACompletionBridgeHandler.handle_streaming(
request_id=str(request.id),
params=params,
litellm_params=litellm_params,
api_base=api_base,
agent_extra_headers=agent_extra_headers,
)
caller_logging_obj: Final = kwargs.get("litellm_logging_obj")
bridge_logging_obj: Final = (
caller_logging_obj
if isinstance(caller_logging_obj, Logging)
else _build_streaming_logging_obj(
request=request,
agent_name=agent_name or "unknown",
agent_id=agent_id,
litellm_params=litellm_params,
metadata=metadata,
proxy_server_request=proxy_server_request,
start_time=called_at,
)
)
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
async for chunk in A2AStreamingIterator(
stream=bridge_stream,
request=request,
logging_obj=bridge_logging_obj,
agent_name=agent_name or "unknown",
output_text=A2ARequestUtils.extract_text_from_stream_event,
):
yield chunk
return
@ -718,7 +753,6 @@ async def asend_message_streaming(
if request is None:
raise ValueError("request is required")
called_at: Final = datetime.datetime.now()
_raw_logging_obj: Final = kwargs.get("litellm_logging_obj")
logging_obj: Logging | None = _raw_logging_obj if isinstance(_raw_logging_obj, Logging) else None
@ -742,12 +776,12 @@ async def asend_message_streaming(
assert a2a_client is not None
agent_name: Final = _get_a2a_model_info(a2a_client, kwargs)
card_agent_name: Final = _get_a2a_model_info(a2a_client, kwargs)
if logging_obj is None:
logging_obj = _build_streaming_logging_obj(
request=request,
agent_name=agent_name,
agent_name=card_agent_name,
agent_id=agent_id,
litellm_params=litellm_params,
metadata=metadata,
@ -755,7 +789,7 @@ async def asend_message_streaming(
start_time=called_at,
)
verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name)
verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, card_agent_name)
agent_card: Final = _get_a2a_client_agent_card(a2a_client)
card_url: Final = get_agent_card_url(agent_card) if agent_card else None
@ -766,7 +800,7 @@ async def asend_message_streaming(
agent_card=agent_card,
card_url=card_url,
api_base=api_base,
agent_name=agent_name,
agent_name=card_agent_name,
)
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
@ -775,7 +809,7 @@ async def asend_message_streaming(
stream=stream,
request=request,
logging_obj=logging_obj,
agent_name=agent_name,
agent_name=card_agent_name,
):
yield chunk

View file

@ -3,7 +3,7 @@ A2A Streaming Iterator with token tracking and logging support.
"""
import asyncio
from collections.abc import AsyncIterator
from collections.abc import AsyncIterator, Callable, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final
@ -23,6 +23,12 @@ if TYPE_CHECKING:
_OPAQUE_PAYLOAD: Final = TypeAdapter(object)
def _chunk_payload(chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> Mapping[str, object]:
if isinstance(chunk, Mapping):
return chunk
return chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
class A2AStreamingIterator:
"""
Async iterator for A2A streaming responses with token tracking.
@ -32,26 +38,28 @@ class A2AStreamingIterator:
def __init__(
self,
stream: AsyncIterator["SendStreamingMessageResponse"],
stream: AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"],
request: "SendStreamingMessageRequest",
logging_obj: LiteLLMLoggingObj,
agent_name: str = "unknown",
output_text: Callable[[Mapping[str, object]], str] = A2ARequestUtils.extract_text_from_response,
):
self.stream = stream
self.request = request
self.logging_obj = logging_obj
self.agent_name = agent_name
self.output_text = output_text
self.start_time = logging_obj.start_time
# Collect chunks for token counting
self.chunks: list[SendStreamingMessageResponse] = []
self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = []
self.collected_text_parts: list[str] = []
self.final_chunk: SendStreamingMessageResponse | None = None
self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None
def __aiter__(self):
return self
async def __anext__(self) -> "SendStreamingMessageResponse":
async def __anext__(self) -> "SendStreamingMessageResponse | Mapping[str, object]":
try:
chunk: Final = await self.stream.__anext__()
if self.logging_obj.completion_start_time is None:
@ -76,21 +84,19 @@ class A2AStreamingIterator:
await self._handle_stream_complete()
raise
def _collect_text_from_chunk(self, chunk: "SendStreamingMessageResponse") -> None:
def _collect_text_from_chunk(self, chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> None:
"""Extract text from a streaming chunk and add to collected parts."""
try:
chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
text: Final = A2ARequestUtils.extract_text_from_response(chunk_dict)
text: Final = self.output_text(_chunk_payload(chunk))
if text:
self.collected_text_parts.append(text)
except Exception:
verbose_logger.debug("Failed to extract text from A2A streaming chunk")
def _is_completed_chunk(self, chunk: "SendStreamingMessageResponse") -> bool:
def _is_completed_chunk(self, chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> bool:
"""Check if chunk indicates stream completion."""
try:
chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
result: Final = chunk_dict.get("result", {})
result: Final = _chunk_payload(chunk).get("result", {})
if isinstance(result, dict):
status: Final = result.get("status", {})
if isinstance(status, dict):
@ -167,9 +173,7 @@ class A2AStreamingIterator:
# Add final chunk result if available
if self.final_chunk:
try:
chunk_dict: Final = as_str_mapping(
_OPAQUE_PAYLOAD.validate_python(self.final_chunk.model_dump(mode="json", exclude_none=True))
)
chunk_dict: Final = as_str_mapping(_OPAQUE_PAYLOAD.validate_python(_chunk_payload(self.final_chunk)))
if chunk_dict is not None:
result["result"] = chunk_dict.get("result", {})
except Exception:

View file

@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Final, Protocol
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attributes
if TYPE_CHECKING:
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
@ -34,6 +34,9 @@ class A2ARequestUtils:
"""
if message is None:
return ""
# a2a-sdk Part RootModels keep kind/text on part.root, so read pydantic messages as dicts
if has_attributes(message, _SupportsModelDump, "model_dump"):
return A2ARequestUtils.extract_text_from_message(message.model_dump(mode="json"))
# Handle both dict and object access
if isinstance(message, dict):
@ -76,6 +79,13 @@ class A2ARequestUtils:
message: Final = result.get("message", {})
return A2ARequestUtils.extract_text_from_message(message)
@staticmethod
def extract_text_from_stream_event(event: Mapping[str, object]) -> str:
result: Final = as_str_mapping(event.get("result"))
if result is not None and result.get("kind") == "artifact-update":
return A2ARequestUtils.extract_text_from_message(result.get("artifact"))
return A2ARequestUtils.extract_text_from_response(event)
@staticmethod
def get_input_message_from_request(
request: "SendMessageRequest | SendStreamingMessageRequest",
@ -128,17 +138,8 @@ class A2ARequestUtils:
Returns:
Tuple of (prompt_tokens, completion_tokens, total_tokens)
"""
# Count input tokens. Dump the message to a dict first so extraction hits
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
# kind/text live on part.root, which the object branch cannot read. This
# mirrors how the response side already works (it operates on model_dump).
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
input_text: Final = A2ARequestUtils.extract_text_from_message(
input_message.model_dump(mode="json")
if has_attributes(input_message, _SupportsModelDump, "model_dump")
else input_message
)
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
prompt_tokens: Final = A2ARequestUtils.count_tokens(A2ARequestUtils.extract_text_from_message(input_message))
# Count output tokens
output_text: Final = A2ARequestUtils.extract_text_from_response(response_dict)

View file

@ -431,6 +431,7 @@ async def _handle_stream_message(
proxy_server_request: dict[str, object] | None = None,
*,
agent_extra_headers: dict[str, str] | None = None,
agent_name: str | None = None,
user_api_key_dict: UserAPIKeyAuth | None = None,
request_data: dict[str, object] | None = None,
proxy_logging_obj: ProxyLogging | None = None,
@ -504,6 +505,7 @@ async def _handle_stream_message(
metadata=metadata,
proxy_server_request=proxy_server_request,
agent_extra_headers=agent_extra_headers,
agent_name=agent_name,
)
if (
@ -932,6 +934,7 @@ async def invoke_agent_a2a(
metadata=data.get("metadata", {}),
proxy_server_request=data.get("proxy_server_request"),
agent_extra_headers=agent_extra_headers,
agent_name=str(agent_name),
user_api_key_dict=user_api_key_auth_for_call,
request_data=data,
proxy_logging_obj=proxy_logging_obj,

View file

@ -798,6 +798,105 @@ def test_streaming_logging_obj_keeps_agent_credentials_out_of_logging_params():
assert logging_obj.model_call_details["litellm_params"] == expected
class _A2ASpendRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.logged = asyncio.Event()
self.payload = None
self.agent_id = None
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
payload = kwargs["standard_logging_object"]
if payload["call_type"] not in ("asend_message", "asend_message_streaming"):
return
self.payload = payload
self.agent_id = kwargs.get("agent_id")
self.logged.set()
@pytest.fixture
def a2a_spend_recorder(monkeypatch):
recorder = _A2ASpendRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
monkeypatch.setattr(litellm, "success_callback", [recorder])
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
return recorder
_BRIDGE_AGENT_PRICES: Final = (
pytest.param({"cost_per_query": 0.25}, lambda prompt_tokens, completion_tokens: 0.25, id="cost_per_query"),
pytest.param(
{"input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
lambda prompt_tokens, completion_tokens: prompt_tokens * 0.001 + completion_tokens * 0.002,
id="per_token",
),
)
def _bridge_agent_params(prices):
return {
"custom_llm_provider": "langgraph",
"model": "agent",
"mock_response": "Paris is the capital of France.",
**prices,
}
def _bridge_message_params() -> MessageSendParams:
return MessageSendParams(
message={
"messageId": "m1",
"role": "user",
"parts": [{"kind": "text", "text": "What is the capital of France?"}],
}
)
def _assert_billed_at_agent_prices(recorder, expected_cost):
payload = recorder.payload
assert payload["prompt_tokens"] > 0
assert payload["completion_tokens"] > 0
assert payload["response_cost"] == pytest.approx(
expected_cost(payload["prompt_tokens"], payload["completion_tokens"])
)
assert recorder.agent_id == "agent-1"
@pytest.mark.parametrize(("prices", "expected_cost"), _BRIDGE_AGENT_PRICES)
@pytest.mark.asyncio
async def test_completion_bridge_send_bills_the_agents_prices(prices, expected_cost, a2a_spend_recorder):
await asend_message(
request=SendMessageRequest(id="bridge-send", params=_bridge_message_params()),
api_base="http://127.0.0.1:9",
litellm_params=_bridge_agent_params(prices),
agent_id="agent-1",
)
await asyncio.wait_for(a2a_spend_recorder.logged.wait(), timeout=10)
_assert_billed_at_agent_prices(a2a_spend_recorder, expected_cost)
@pytest.mark.parametrize(("prices", "expected_cost"), _BRIDGE_AGENT_PRICES)
@pytest.mark.asyncio
async def test_completion_bridge_stream_bills_the_agents_prices(prices, expected_cost, a2a_spend_recorder):
from litellm.a2a_protocol.main import asend_message_streaming
_ = [
chunk
async for chunk in asend_message_streaming(
request=SendStreamingMessageRequest(id="bridge-stream", params=_bridge_message_params()),
api_base="http://127.0.0.1:9",
litellm_params=_bridge_agent_params(prices),
agent_id="agent-1",
agent_name="trip-planner",
)
]
await asyncio.wait_for(a2a_spend_recorder.logged.wait(), timeout=10)
_assert_billed_at_agent_prices(a2a_spend_recorder, expected_cost)
assert a2a_spend_recorder.payload["model"] == "a2a_agent/trip-planner"
class MockA2AResponse:
def __init__(self, text: str):
self._payload = {