mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(a2a): count chunked and Task replies once and skip agent prices on error replies
Bridge streams now follow the A2A append flag per artifact, so chunked pydantic-ai, watsonx, and AgentCore replies count the whole reply once. Send replies that are a Task count their artifact text. A JSON-RPC error reply is logged at cost 0 instead of the agent's price.
This commit is contained in:
parent
e2d40a830d
commit
4512624633
8 changed files with 347 additions and 57 deletions
|
|
@ -203,7 +203,8 @@ async def _record_send_message_billing(
|
|||
response_dict=response_dict,
|
||||
)
|
||||
_set_usage_on_logging_obj(kwargs=kwargs, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
|
||||
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
|
||||
if "error" not in response_dict:
|
||||
_set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
|
||||
_set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -515,6 +515,8 @@ class PydanticAITransformation:
|
|||
"contextId": context_id,
|
||||
"kind": "artifact-update",
|
||||
"taskId": task_id,
|
||||
"append": i > 0,
|
||||
"lastChunk": is_last_chunk,
|
||||
"artifact": {
|
||||
"artifactId": artifact_id,
|
||||
"parts": [
|
||||
|
|
|
|||
|
|
@ -206,6 +206,8 @@ class WatsonxOrchestrateTransformation:
|
|||
"contextId": context_id,
|
||||
"kind": "artifact-update",
|
||||
"taskId": task_id,
|
||||
"append": i > 0,
|
||||
"lastChunk": is_last,
|
||||
"artifact": {
|
||||
"artifactId": artifact_id,
|
||||
"parts": [{"kind": "text", "text": chunk_text}],
|
||||
|
|
|
|||
|
|
@ -29,6 +29,17 @@ def _chunk_payload(chunk: "SendStreamingMessageResponse | Mapping[str, object]")
|
|||
return chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
|
||||
|
||||
|
||||
def _artifact_update(payload: Mapping[str, object]) -> tuple[str, bool] | None:
|
||||
result: Final = as_str_mapping(payload.get("result"))
|
||||
if result is None or result.get("kind") != "artifact-update":
|
||||
return None
|
||||
artifact: Final = as_str_mapping(result.get("artifact"))
|
||||
artifact_id: Final = artifact.get("artifactId") if artifact is not None else None
|
||||
if not isinstance(artifact_id, str):
|
||||
return None
|
||||
return artifact_id, result.get("append") is True
|
||||
|
||||
|
||||
class A2AStreamingIterator:
|
||||
"""
|
||||
Async iterator for A2A streaming responses with token tracking.
|
||||
|
|
@ -54,6 +65,8 @@ class A2AStreamingIterator:
|
|||
# Collect chunks for token counting
|
||||
self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = []
|
||||
self.collected_text_parts: list[str] = []
|
||||
self.artifact_texts: Mapping[str, str] = {}
|
||||
self.saw_error = False
|
||||
self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None
|
||||
|
||||
def __aiter__(self):
|
||||
|
|
@ -67,12 +80,14 @@ class A2AStreamingIterator:
|
|||
|
||||
# Store chunk
|
||||
self.chunks.append(chunk)
|
||||
payload: Final = _chunk_payload(chunk)
|
||||
self.saw_error = self.saw_error or "error" in payload
|
||||
|
||||
# Extract text from chunk for token counting
|
||||
self._collect_text_from_chunk(chunk)
|
||||
self._collect_text_from_chunk(payload)
|
||||
|
||||
# Check if this is the final chunk (completed status)
|
||||
if self._is_completed_chunk(chunk):
|
||||
if self._is_completed_chunk(payload):
|
||||
self.final_chunk = chunk
|
||||
|
||||
return chunk
|
||||
|
|
@ -84,19 +99,27 @@ class A2AStreamingIterator:
|
|||
await self._handle_stream_complete()
|
||||
raise
|
||||
|
||||
def _collect_text_from_chunk(self, chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> None:
|
||||
def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None:
|
||||
"""Extract text from a streaming chunk and add to collected parts."""
|
||||
try:
|
||||
text: Final = self.output_text(_chunk_payload(chunk))
|
||||
if text:
|
||||
text: Final = self.output_text(payload)
|
||||
if not text:
|
||||
return
|
||||
artifact_update: Final = _artifact_update(payload)
|
||||
if artifact_update is None:
|
||||
self.collected_text_parts.append(text)
|
||||
return
|
||||
artifact_id, append = artifact_update
|
||||
artifact_text: Final = self.artifact_texts.get(artifact_id, "") + text if append else text
|
||||
self.artifact_texts = {**self.artifact_texts, artifact_id: artifact_text}
|
||||
self.collected_text_parts.append(artifact_text)
|
||||
except Exception:
|
||||
verbose_logger.debug("Failed to extract text from A2A streaming chunk")
|
||||
|
||||
def _is_completed_chunk(self, chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> bool:
|
||||
def _is_completed_chunk(self, payload: Mapping[str, object]) -> bool:
|
||||
"""Check if chunk indicates stream completion."""
|
||||
try:
|
||||
result: Final = _chunk_payload(chunk).get("result", {})
|
||||
result: Final = payload.get("result", {})
|
||||
if isinstance(result, dict):
|
||||
status: Final = result.get("status", {})
|
||||
if isinstance(status, dict):
|
||||
|
|
@ -134,7 +157,7 @@ class A2AStreamingIterator:
|
|||
self.logging_obj.model_call_details["stream"] = False
|
||||
|
||||
# Calculate cost using A2ACostCalculator
|
||||
response_cost: Final = A2ACostCalculator.calculate_a2a_cost(self.logging_obj)
|
||||
response_cost: Final = 0.0 if self.saw_error else A2ACostCalculator.calculate_a2a_cost(self.logging_obj)
|
||||
self.logging_obj.model_call_details["response_cost"] = response_cost
|
||||
|
||||
# Build result for logging
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import hashlib
|
|||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attributes
|
||||
|
|
@ -13,6 +15,8 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attribut
|
|||
if TYPE_CHECKING:
|
||||
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
|
||||
|
||||
_OBJECT_ITEMS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
class _SupportsModelDump(Protocol):
|
||||
def model_dump(self, *, mode: str) -> object: ...
|
||||
|
|
@ -66,8 +70,8 @@ class A2ARequestUtils:
|
|||
Returns:
|
||||
Text from response message parts
|
||||
"""
|
||||
result: Final = response_dict.get("result", {})
|
||||
if not isinstance(result, dict):
|
||||
result: Final = as_str_mapping(response_dict.get("result"))
|
||||
if result is None:
|
||||
return ""
|
||||
|
||||
# Direct message format (A2A spec): detect by explicit kind tag only.
|
||||
|
|
@ -75,10 +79,24 @@ class A2ARequestUtils:
|
|||
# type that happens to include a "parts" field.
|
||||
if result.get("kind") == "message":
|
||||
return A2ARequestUtils.extract_text_from_message(result)
|
||||
if result.get("kind") == "task":
|
||||
return A2ARequestUtils.extract_text_from_task(result)
|
||||
|
||||
message: Final = result.get("message", {})
|
||||
return A2ARequestUtils.extract_text_from_message(message)
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_task(task: Mapping[str, object]) -> str:
|
||||
artifacts: Final = task.get("artifacts")
|
||||
artifact_items: Final = _OBJECT_ITEMS.validate_python(artifacts) if isinstance(artifacts, list) else ()
|
||||
artifact_texts: Final = tuple(
|
||||
A2ARequestUtils.extract_text_from_message(artifact) for artifact in artifact_items
|
||||
)
|
||||
if any(artifact_texts):
|
||||
return " ".join(text for text in artifact_texts if text)
|
||||
status: Final = as_str_mapping(task.get("status"))
|
||||
return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_stream_event(event: Mapping[str, object]) -> str:
|
||||
result: Final = as_str_mapping(event.get("result"))
|
||||
|
|
|
|||
|
|
@ -148,6 +148,8 @@ async def test_handle_streaming_runs_the_agent_and_ignores_unrelated_keywords(re
|
|||
"kind": "artifact-update",
|
||||
"taskId": ANY,
|
||||
"artifact": {"artifactId": ANY, "parts": [{"kind": "text", "text": "wxo says hi"}]},
|
||||
"append": False,
|
||||
"lastChunk": True,
|
||||
}
|
||||
assert chunks[3]["result"] == {
|
||||
"contextId": ANY,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ async_success_handler (cross-thread pydantic mutation segfaults pydantic-core).
|
|||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Callable, Mapping
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from types import SimpleNamespace
|
||||
|
|
@ -15,6 +16,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm.a2a_protocol import streaming_iterator as a2a_streaming_iterator_module
|
||||
from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
|
||||
from litellm.a2a_protocol.utils import A2ARequestUtils
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils import thread_pool_executor as thread_pool_executor_module
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
|
||||
|
|
@ -189,3 +191,65 @@ async def test_stream_logs_its_first_chunk_time_measured_from_the_call_start():
|
|||
|
||||
assert first_chunk_at is not None
|
||||
assert recorder.logged == [(call_start, first_chunk_at)]
|
||||
|
||||
|
||||
_REPLY: Final = (
|
||||
"Paris is the capital of France. It sits on the Seine, holds about two million people, "
|
||||
"and has been the seat of the French government for most of the last thousand years."
|
||||
)
|
||||
|
||||
|
||||
def _artifact_event(text: str, append: bool | None) -> Mapping[str, object]:
|
||||
result: Final = {
|
||||
"kind": "artifact-update",
|
||||
"taskId": "t1",
|
||||
"contextId": "c1",
|
||||
"artifact": {"artifactId": "a1", "parts": [{"kind": "text", "text": text}]},
|
||||
}
|
||||
return {"jsonrpc": "2.0", "id": "r1", "result": result if append is None else {**result, "append": append}}
|
||||
|
||||
|
||||
async def _snapshot_stream() -> AsyncIterator[Mapping[str, object]]:
|
||||
for end in range(40, len(_REPLY) + 40, 40):
|
||||
yield _artifact_event(_REPLY[:end], append=None)
|
||||
|
||||
|
||||
async def _watsonx_stream() -> AsyncIterator[Mapping[str, object]]:
|
||||
from litellm.a2a_protocol.providers.watsonx_orchestrate.transformation import (
|
||||
WatsonxOrchestrateTransformation,
|
||||
)
|
||||
|
||||
async for event in WatsonxOrchestrateTransformation.fake_streaming_from_text(_REPLY, "r1", delay_ms=0):
|
||||
yield event
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"agent_stream",
|
||||
(pytest.param(_snapshot_stream, id="replacing_snapshots"), pytest.param(_watsonx_stream, id="watsonx_chunks")),
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_counts_the_whole_artifact_once(
|
||||
agent_stream: Callable[[], AsyncIterator[Mapping[str, object]]], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "success_callback", [])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [])
|
||||
logging_obj: Final = LitellmLogging(
|
||||
model="a2a_agent/trip-planner",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=False,
|
||||
call_type="asend_message_streaming",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="artifact-count",
|
||||
function_id="artifact-count",
|
||||
)
|
||||
iterator: Final = A2AStreamingIterator(
|
||||
stream=agent_stream(),
|
||||
request=SimpleNamespace(params=SimpleNamespace(message={"role": "user", "parts": []})),
|
||||
logging_obj=logging_obj,
|
||||
agent_name="trip-planner",
|
||||
output_text=A2ARequestUtils.extract_text_from_stream_event,
|
||||
)
|
||||
|
||||
_ = [chunk async for chunk in iterator]
|
||||
|
||||
assert logging_obj.model_call_details["usage"].completion_tokens == A2ARequestUtils.count_tokens(_REPLY)
|
||||
|
|
|
|||
|
|
@ -3,10 +3,13 @@
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
from typing import Final
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
pytest.importorskip("a2a.compat.v0_3.conversions")
|
||||
|
||||
|
|
@ -27,6 +30,7 @@ from litellm.a2a_protocol.main import (
|
|||
create_a2a_client,
|
||||
)
|
||||
from litellm.a2a_protocol.exceptions import A2AError
|
||||
from litellm.a2a_protocol.utils import A2ARequestUtils
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
|
|
@ -798,46 +802,95 @@ def test_streaming_logging_obj_keeps_agent_credentials_out_of_logging_params():
|
|||
assert logging_obj.model_call_details["litellm_params"] == expected
|
||||
|
||||
|
||||
class _LoggedSpend(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
call_type: str
|
||||
model: str
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
response_cost: float
|
||||
|
||||
|
||||
class _A2ASpendRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.logged = asyncio.Event()
|
||||
self.payload = None
|
||||
self.agent_id = None
|
||||
self.logged: Final = asyncio.Event()
|
||||
self.spend: _LoggedSpend | None = None
|
||||
self.agent_id: object = None
|
||||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
payload = kwargs["standard_logging_object"]
|
||||
if payload["call_type"] not in ("asend_message", "asend_message_streaming"):
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
spend: Final = _LoggedSpend.model_validate(kwargs["standard_logging_object"])
|
||||
if spend.call_type not in ("asend_message", "asend_message_streaming"):
|
||||
return
|
||||
self.payload = payload
|
||||
self.spend = spend
|
||||
self.agent_id = kwargs.get("agent_id")
|
||||
self.logged.set()
|
||||
|
||||
async def logged_spend(self) -> _LoggedSpend:
|
||||
await asyncio.wait_for(self.logged.wait(), timeout=10)
|
||||
assert self.spend is not None
|
||||
return self.spend
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def a2a_spend_recorder(monkeypatch):
|
||||
recorder = _A2ASpendRecorder()
|
||||
def a2a_spend_recorder(monkeypatch: pytest.MonkeyPatch) -> _A2ASpendRecorder:
|
||||
recorder: Final = _A2ASpendRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
monkeypatch.setattr(litellm, "success_callback", [recorder])
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [recorder])
|
||||
return recorder
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def respx_reachable_httpx(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
|
||||
|
||||
_AgentPrices: TypeAlias = Mapping[str, float]
|
||||
_ExpectedCost: TypeAlias = Callable[[int, int], float]
|
||||
|
||||
_BRIDGE_PROMPT: Final = "What is the capital of France?"
|
||||
_BRIDGE_REPLY: Final = "Paris is the capital of France."
|
||||
_LONG_AGENT_REPLY: Final = (
|
||||
"Paris is the capital of France. It sits on the Seine, holds about two million people, "
|
||||
"and has been the seat of the French government for most of the last thousand years."
|
||||
)
|
||||
_PER_TOKEN_PRICES: Final[_AgentPrices] = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002}
|
||||
_AGENTCORE_ARN: Final = "arn:aws:bedrock-agentcore:us-west-2:123456789:runtime/trip_planner"
|
||||
_AGENTCORE_PARAMS: Final[Mapping[str, object]] = {
|
||||
"custom_llm_provider": "bedrock",
|
||||
"model": f"bedrock/agentcore/{_AGENTCORE_ARN}",
|
||||
"api_key": "test-jwt-token",
|
||||
}
|
||||
_JSON_RPC_ERROR: Final[Mapping[str, object]] = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": "agentcore-call",
|
||||
"error": {"code": -32603, "message": "agent crashed"},
|
||||
}
|
||||
|
||||
|
||||
def _per_token_cost(prompt_tokens: int, completion_tokens: int) -> float:
|
||||
return prompt_tokens * 0.001 + completion_tokens * 0.002
|
||||
|
||||
|
||||
_BRIDGE_AGENT_PRICES: Final = (
|
||||
pytest.param({"cost_per_query": 0.25}, lambda prompt_tokens, completion_tokens: 0.25, id="cost_per_query"),
|
||||
pytest.param(
|
||||
{"input_cost_per_token": 0.001, "output_cost_per_token": 0.002},
|
||||
lambda prompt_tokens, completion_tokens: prompt_tokens * 0.001 + completion_tokens * 0.002,
|
||||
id="per_token",
|
||||
),
|
||||
pytest.param(_PER_TOKEN_PRICES, _per_token_cost, id="per_token"),
|
||||
)
|
||||
|
||||
|
||||
def _bridge_agent_params(prices):
|
||||
def _bridge_agent_params(prices: _AgentPrices) -> dict[str, object]:
|
||||
return {
|
||||
"custom_llm_provider": "langgraph",
|
||||
"model": "agent",
|
||||
"mock_response": "Paris is the capital of France.",
|
||||
"mock_response": _BRIDGE_REPLY,
|
||||
**prices,
|
||||
}
|
||||
|
||||
|
|
@ -847,38 +900,21 @@ def _bridge_message_params() -> MessageSendParams:
|
|||
message={
|
||||
"messageId": "m1",
|
||||
"role": "user",
|
||||
"parts": [{"kind": "text", "text": "What is the capital of France?"}],
|
||||
"parts": [{"kind": "text", "text": _BRIDGE_PROMPT}],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _assert_billed_at_agent_prices(recorder, expected_cost):
|
||||
payload = recorder.payload
|
||||
assert payload["prompt_tokens"] > 0
|
||||
assert payload["completion_tokens"] > 0
|
||||
assert payload["response_cost"] == pytest.approx(
|
||||
expected_cost(payload["prompt_tokens"], payload["completion_tokens"])
|
||||
)
|
||||
def _assert_billed_at_agent_prices(
|
||||
spend: _LoggedSpend, recorder: _A2ASpendRecorder, expected_cost: _ExpectedCost, reply: str
|
||||
) -> None:
|
||||
assert spend.prompt_tokens == A2ARequestUtils.count_tokens(_BRIDGE_PROMPT)
|
||||
assert spend.completion_tokens == A2ARequestUtils.count_tokens(reply)
|
||||
assert spend.response_cost == pytest.approx(expected_cost(spend.prompt_tokens, spend.completion_tokens))
|
||||
assert recorder.agent_id == "agent-1"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("prices", "expected_cost"), _BRIDGE_AGENT_PRICES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_bridge_send_bills_the_agents_prices(prices, expected_cost, a2a_spend_recorder):
|
||||
await asend_message(
|
||||
request=SendMessageRequest(id="bridge-send", params=_bridge_message_params()),
|
||||
api_base="http://127.0.0.1:9",
|
||||
litellm_params=_bridge_agent_params(prices),
|
||||
agent_id="agent-1",
|
||||
)
|
||||
await asyncio.wait_for(a2a_spend_recorder.logged.wait(), timeout=10)
|
||||
|
||||
_assert_billed_at_agent_prices(a2a_spend_recorder, expected_cost)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("prices", "expected_cost"), _BRIDGE_AGENT_PRICES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_bridge_stream_bills_the_agents_prices(prices, expected_cost, a2a_spend_recorder):
|
||||
async def _drain_bridge_stream(litellm_params: Mapping[str, object]) -> None:
|
||||
from litellm.a2a_protocol.main import asend_message_streaming
|
||||
|
||||
_ = [
|
||||
|
|
@ -886,15 +922,157 @@ async def test_completion_bridge_stream_bills_the_agents_prices(prices, expected
|
|||
async for chunk in asend_message_streaming(
|
||||
request=SendStreamingMessageRequest(id="bridge-stream", params=_bridge_message_params()),
|
||||
api_base="http://127.0.0.1:9",
|
||||
litellm_params=_bridge_agent_params(prices),
|
||||
litellm_params=dict(litellm_params),
|
||||
agent_id="agent-1",
|
||||
agent_name="trip-planner",
|
||||
)
|
||||
]
|
||||
await asyncio.wait_for(a2a_spend_recorder.logged.wait(), timeout=10)
|
||||
|
||||
_assert_billed_at_agent_prices(a2a_spend_recorder, expected_cost)
|
||||
assert a2a_spend_recorder.payload["model"] == "a2a_agent/trip-planner"
|
||||
|
||||
def _sse(*events: Mapping[str, object]) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
content="".join(f"data: {json.dumps(event)}\n\n" for event in events),
|
||||
)
|
||||
|
||||
|
||||
def _agentcore_appended_chunks(text: str, chunk_size: int) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": "agentcore-call",
|
||||
"result": {
|
||||
"kind": "artifact-update",
|
||||
"taskId": "t1",
|
||||
"contextId": "c1",
|
||||
"append": start > 0,
|
||||
"artifact": {"artifactId": "a1", "parts": [{"kind": "text", "text": text[start : start + chunk_size]}]},
|
||||
},
|
||||
}
|
||||
for start in range(0, len(text), chunk_size)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("prices", "expected_cost"), _BRIDGE_AGENT_PRICES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_bridge_send_bills_the_agents_prices(
|
||||
prices: _AgentPrices, expected_cost: _ExpectedCost, a2a_spend_recorder: _A2ASpendRecorder
|
||||
) -> None:
|
||||
await asend_message(
|
||||
request=SendMessageRequest(id="bridge-send", params=_bridge_message_params()),
|
||||
api_base="http://127.0.0.1:9",
|
||||
litellm_params=_bridge_agent_params(prices),
|
||||
agent_id="agent-1",
|
||||
)
|
||||
|
||||
spend: Final = await a2a_spend_recorder.logged_spend()
|
||||
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, expected_cost, _BRIDGE_REPLY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("prices", "expected_cost"), _BRIDGE_AGENT_PRICES)
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_bridge_stream_bills_the_agents_prices(
|
||||
prices: _AgentPrices, expected_cost: _ExpectedCost, a2a_spend_recorder: _A2ASpendRecorder
|
||||
) -> None:
|
||||
await _drain_bridge_stream(_bridge_agent_params(prices))
|
||||
|
||||
spend: Final = await a2a_spend_recorder.logged_spend()
|
||||
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, expected_cost, _BRIDGE_REPLY)
|
||||
assert spend.model == "a2a_agent/trip-planner"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agentcore_send_bills_a_task_reply_at_the_agents_token_prices(
|
||||
a2a_spend_recorder: _A2ASpendRecorder, respx_reachable_httpx: None
|
||||
) -> None:
|
||||
task_reply: Final = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": "bridge-send",
|
||||
"result": {
|
||||
"kind": "task",
|
||||
"id": "t1",
|
||||
"contextId": "c1",
|
||||
"status": {"state": "completed"},
|
||||
"artifacts": [{"artifactId": "a1", "parts": [{"kind": "text", "text": _LONG_AGENT_REPLY}]}],
|
||||
},
|
||||
}
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post(url__regex=r".*/invocations.*").mock(return_value=httpx.Response(200, json=task_reply))
|
||||
await asend_message(
|
||||
request=SendMessageRequest(id="bridge-send", params=_bridge_message_params()),
|
||||
litellm_params={**_AGENTCORE_PARAMS, **_PER_TOKEN_PRICES},
|
||||
agent_id="agent-1",
|
||||
)
|
||||
|
||||
spend: Final = await a2a_spend_recorder.logged_spend()
|
||||
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, _per_token_cost, _LONG_AGENT_REPLY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agentcore_stream_bills_every_appended_reply_chunk(
|
||||
a2a_spend_recorder: _A2ASpendRecorder, respx_reachable_httpx: None
|
||||
) -> None:
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post(url__regex=r".*/invocations.*").mock(
|
||||
return_value=_sse(*_agentcore_appended_chunks(_LONG_AGENT_REPLY, chunk_size=40))
|
||||
)
|
||||
await _drain_bridge_stream({**_AGENTCORE_PARAMS, **_PER_TOKEN_PRICES})
|
||||
|
||||
spend: Final = await a2a_spend_recorder.logged_spend()
|
||||
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, _per_token_cost, _LONG_AGENT_REPLY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pydantic_ai_stream_bills_the_whole_chunked_reply(
|
||||
a2a_spend_recorder: _A2ASpendRecorder, respx_reachable_httpx: None
|
||||
) -> None:
|
||||
completed_task: Final = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": "bridge-stream",
|
||||
"result": {
|
||||
"kind": "task",
|
||||
"id": "t1",
|
||||
"contextId": "c1",
|
||||
"status": {"state": "completed"},
|
||||
"artifacts": [{"artifactId": "a1", "parts": [{"kind": "text", "text": _LONG_AGENT_REPLY}]}],
|
||||
},
|
||||
}
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post("http://127.0.0.1:9/").mock(return_value=httpx.Response(200, json=completed_task))
|
||||
await _drain_bridge_stream({"custom_llm_provider": "pydantic_ai_agents", **_PER_TOKEN_PRICES})
|
||||
|
||||
spend: Final = await a2a_spend_recorder.logged_spend()
|
||||
assert len(_LONG_AGENT_REPLY) > 50
|
||||
_assert_billed_at_agent_prices(spend, a2a_spend_recorder, _per_token_cost, _LONG_AGENT_REPLY)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"upstream_reply",
|
||||
(
|
||||
pytest.param(httpx.Response(200, json=_JSON_RPC_ERROR), id="send"),
|
||||
pytest.param(_sse(_JSON_RPC_ERROR), id="stream"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_agentcore_error_reply_is_not_billed_at_the_agents_price(
|
||||
upstream_reply: httpx.Response, a2a_spend_recorder: _A2ASpendRecorder, respx_reachable_httpx: None
|
||||
) -> None:
|
||||
agent_params: Final = {**_AGENTCORE_PARAMS, "cost_per_query": 0.25}
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post(url__regex=r".*/invocations.*").mock(return_value=upstream_reply)
|
||||
if upstream_reply.headers["content-type"] == "text/event-stream":
|
||||
await _drain_bridge_stream(agent_params)
|
||||
else:
|
||||
response = await asend_message(
|
||||
request=SendMessageRequest(id="agentcore-call", params=_bridge_message_params()),
|
||||
litellm_params=dict(agent_params),
|
||||
agent_id="agent-1",
|
||||
)
|
||||
assert response.model_dump(mode="json", exclude_none=True)["error"]["message"] == "agent crashed"
|
||||
|
||||
spend: Final = await a2a_spend_recorder.logged_spend()
|
||||
assert spend.response_cost == 0.0
|
||||
|
||||
|
||||
class MockA2AResponse:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue