mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-08 08:08:37 +00:00
test(e2e): tighten toolset wire-name, pricing-reload precondition, and stream-ordering assertions
Addresses the three Greptile P2 findings on this PR. - MCP toolset test: assert the served tool's exact documented wire name, built from the registered server alias and the '-' separator, instead of accepting any served name that merely ends in the bare tool. A wrong prefix or separator now fails, matching what a client calling the documented name actually gets. - Pricing override test: assert the override reads as cleared before Reload Price Data, so the post-reload check proves reload preserved the clear rather than a later ordinary router refresh doing the clearing. - Responses stream test: replace the mutable set accumulator with an immutable, order-preserving check of which item ids were announced strictly before each text delta. This also drops the fragile events.index(event) first-match lookup in the failure message for the enumerate index.
This commit is contained in:
parent
322483525d
commit
4c40f5dc38
3 changed files with 40 additions and 21 deletions
|
|
@ -16,7 +16,7 @@ issue describes.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from typing import Final, Literal
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -84,6 +84,19 @@ class _StreamEvent(BaseModel):
|
|||
return self.item.id if self.item else None
|
||||
|
||||
|
||||
_ANNOUNCE_TYPES: Final = ("response.output_item.added", "response.content_part.added")
|
||||
|
||||
|
||||
def _announced_before(events: list[_StreamEvent], index: int) -> frozenset[str]:
|
||||
"""Item ids announced by an output_item.added / content_part.added event
|
||||
strictly before `index`: the ids a text delta at `index` is allowed to target."""
|
||||
return frozenset(
|
||||
item_id
|
||||
for event in events[:index]
|
||||
if event.type in _ANNOUNCE_TYPES and (item_id := event.announced_item_id()) is not None
|
||||
)
|
||||
|
||||
|
||||
def _stream_post_tool_text(proxy: ProxyClient, key: str, model: str) -> StreamingResponse:
|
||||
call_id = f"call_{unique_marker()}"
|
||||
return proxy.transport.stream(
|
||||
|
|
@ -123,18 +136,16 @@ class TestResponsesStreamTextStart:
|
|||
deltas = [event for event in events if event.type == "response.output_text.delta"]
|
||||
assert deltas, f"the post-tool round streamed no text deltas: {[e.type for e in events]}"
|
||||
|
||||
announced: set[str] = set()
|
||||
for event in events:
|
||||
for index, event in enumerate(events):
|
||||
if event.type != "response.output_text.delta":
|
||||
continue
|
||||
item_id = event.announced_item_id()
|
||||
if event.type in ("response.output_item.added", "response.content_part.added") and item_id:
|
||||
announced.add(item_id)
|
||||
if event.type == "response.output_text.delta":
|
||||
assert item_id in announced, (
|
||||
f"text delta for item {item_id!r} arrived before any output_item.added/"
|
||||
f"content_part.added announced it; OpenAI-SDK parsers fail with "
|
||||
f"'text part ... not found' (#27671). Events so far: "
|
||||
f"{[e.type for e in events[: events.index(event) + 1]]}"
|
||||
)
|
||||
assert item_id in _announced_before(events, index), (
|
||||
f"text delta for item {item_id!r} arrived before any output_item.added/"
|
||||
f"content_part.added announced it; OpenAI-SDK parsers fail with "
|
||||
f"'text part ... not found' (#27671). Events so far: "
|
||||
f"{[e.type for e in events[: index + 1]]}"
|
||||
)
|
||||
assert any(event.type == "response.completed" for event in events), (
|
||||
f"stream never completed: {[e.type for e in events][-5:]}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -122,7 +122,12 @@ class TestPricingOverrideClearAndReload:
|
|||
response_type=NoBody,
|
||||
)
|
||||
)
|
||||
_poll_override_cleared(proxy, model_name)
|
||||
cleared_entry = _poll_override_cleared(proxy, model_name)
|
||||
assert cleared_entry.litellm_params.input_cost_per_token is None, (
|
||||
f"override must be cleared before the reload, else a later ordinary router refresh "
|
||||
f"could clear it and let this test pass without proving reload preserved the clear: "
|
||||
f"{cleared_entry.litellm_params.input_cost_per_token}"
|
||||
)
|
||||
|
||||
unwrap(
|
||||
proxy.transport.post(
|
||||
|
|
|
|||
|
|
@ -126,12 +126,15 @@ class TestMcpToolsetToolNames:
|
|||
)
|
||||
)
|
||||
served_names = [tool.name for tool in served.tools]
|
||||
prefixed = [
|
||||
name
|
||||
for name in served_names
|
||||
if name.endswith(SEARCH_LOGS_TOOL) and name != SEARCH_LOGS_TOOL
|
||||
]
|
||||
assert prefixed, (
|
||||
f"a toolset-scoped key must be served {SEARCH_LOGS_TOOL} under its "
|
||||
f"'{{server_prefix}}{{separator}}{SEARCH_LOGS_TOOL}' wire name; got {served_names}"
|
||||
alias = next(
|
||||
(row.alias for row in client.registered_servers() if row.server_id == server_id),
|
||||
None,
|
||||
)
|
||||
assert alias is not None, f"registered server {server_id} absent from /v1/mcp/server listing"
|
||||
expected_name = f"{alias}-{SEARCH_LOGS_TOOL}"
|
||||
assert expected_name in served_names, (
|
||||
f"a toolset-scoped key must be served {SEARCH_LOGS_TOOL} under its exact documented "
|
||||
f"wire name {expected_name!r} ('<alias>-<tool>'). A wrong prefix or separator still "
|
||||
f"ends in {SEARCH_LOGS_TOOL} but breaks every client calling the documented name with "
|
||||
f"'tool not found'; got {served_names}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue