From 4c40f5dc38090944531fdcdc25eb5fb55dc47a88 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Tue, 11 Aug 2026 17:00:04 -0700 Subject: [PATCH] test(e2e): tighten toolset wire-name, pricing-reload precondition, and stream-ordering assertions Addresses the three Greptile P2 findings on this PR. - MCP toolset test: assert the served tool's exact documented wire name, built from the registered server alias and the '-' separator, instead of accepting any served name that merely ends in the bare tool. A wrong prefix or separator now fails, matching what a client calling the documented name actually gets. - Pricing override test: assert the override reads as cleared before Reload Price Data, so the post-reload check proves reload preserved the clear rather than a later ordinary router refresh doing the clearing. - Responses stream test: replace the mutable set accumulator with an immutable, order-preserving check of which item ids were announced strictly before each text delta. This also drops the fragile events.index(event) first-match lookup in the failure message for the enumerate index. --- .../test_responses_stream_text_start_e2e.py | 35 ++++++++++++------- .../test_model_pricing_override_e2e.py | 7 +++- tests/e2e/mcp/test_mcp_toolset_e2e.py | 19 +++++----- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/tests/e2e/llm_translation/test_responses_stream_text_start_e2e.py b/tests/e2e/llm_translation/test_responses_stream_text_start_e2e.py index 9387eb1c845..e3930105dbc 100644 --- a/tests/e2e/llm_translation/test_responses_stream_text_start_e2e.py +++ b/tests/e2e/llm_translation/test_responses_stream_text_start_e2e.py @@ -16,7 +16,7 @@ issue describes. from __future__ import annotations -from typing import Literal +from typing import Final, Literal import pytest from pydantic import BaseModel @@ -84,6 +84,19 @@ class _StreamEvent(BaseModel): return self.item.id if self.item else None +_ANNOUNCE_TYPES: Final = ("response.output_item.added", "response.content_part.added") + + +def _announced_before(events: list[_StreamEvent], index: int) -> frozenset[str]: + """Item ids announced by an output_item.added / content_part.added event + strictly before `index`: the ids a text delta at `index` is allowed to target.""" + return frozenset( + item_id + for event in events[:index] + if event.type in _ANNOUNCE_TYPES and (item_id := event.announced_item_id()) is not None + ) + + def _stream_post_tool_text(proxy: ProxyClient, key: str, model: str) -> StreamingResponse: call_id = f"call_{unique_marker()}" return proxy.transport.stream( @@ -123,18 +136,16 @@ class TestResponsesStreamTextStart: deltas = [event for event in events if event.type == "response.output_text.delta"] assert deltas, f"the post-tool round streamed no text deltas: {[e.type for e in events]}" - announced: set[str] = set() - for event in events: + for index, event in enumerate(events): + if event.type != "response.output_text.delta": + continue item_id = event.announced_item_id() - if event.type in ("response.output_item.added", "response.content_part.added") and item_id: - announced.add(item_id) - if event.type == "response.output_text.delta": - assert item_id in announced, ( - f"text delta for item {item_id!r} arrived before any output_item.added/" - f"content_part.added announced it; OpenAI-SDK parsers fail with " - f"'text part ... not found' (#27671). Events so far: " - f"{[e.type for e in events[: events.index(event) + 1]]}" - ) + assert item_id in _announced_before(events, index), ( + f"text delta for item {item_id!r} arrived before any output_item.added/" + f"content_part.added announced it; OpenAI-SDK parsers fail with " + f"'text part ... not found' (#27671). Events so far: " + f"{[e.type for e in events[: index + 1]]}" + ) assert any(event.type == "response.completed" for event in events), ( f"stream never completed: {[e.type for e in events][-5:]}" ) diff --git a/tests/e2e/management/test_model_pricing_override_e2e.py b/tests/e2e/management/test_model_pricing_override_e2e.py index 5cb9d036d38..9e4a8d0e021 100644 --- a/tests/e2e/management/test_model_pricing_override_e2e.py +++ b/tests/e2e/management/test_model_pricing_override_e2e.py @@ -122,7 +122,12 @@ class TestPricingOverrideClearAndReload: response_type=NoBody, ) ) - _poll_override_cleared(proxy, model_name) + cleared_entry = _poll_override_cleared(proxy, model_name) + assert cleared_entry.litellm_params.input_cost_per_token is None, ( + f"override must be cleared before the reload, else a later ordinary router refresh " + f"could clear it and let this test pass without proving reload preserved the clear: " + f"{cleared_entry.litellm_params.input_cost_per_token}" + ) unwrap( proxy.transport.post( diff --git a/tests/e2e/mcp/test_mcp_toolset_e2e.py b/tests/e2e/mcp/test_mcp_toolset_e2e.py index 4e6247064a4..3e27ef209e8 100644 --- a/tests/e2e/mcp/test_mcp_toolset_e2e.py +++ b/tests/e2e/mcp/test_mcp_toolset_e2e.py @@ -126,12 +126,15 @@ class TestMcpToolsetToolNames: ) ) served_names = [tool.name for tool in served.tools] - prefixed = [ - name - for name in served_names - if name.endswith(SEARCH_LOGS_TOOL) and name != SEARCH_LOGS_TOOL - ] - assert prefixed, ( - f"a toolset-scoped key must be served {SEARCH_LOGS_TOOL} under its " - f"'{{server_prefix}}{{separator}}{SEARCH_LOGS_TOOL}' wire name; got {served_names}" + alias = next( + (row.alias for row in client.registered_servers() if row.server_id == server_id), + None, + ) + assert alias is not None, f"registered server {server_id} absent from /v1/mcp/server listing" + expected_name = f"{alias}-{SEARCH_LOGS_TOOL}" + assert expected_name in served_names, ( + f"a toolset-scoped key must be served {SEARCH_LOGS_TOOL} under its exact documented " + f"wire name {expected_name!r} ('-'). A wrong prefix or separator still " + f"ends in {SEARCH_LOGS_TOOL} but breaks every client calling the documented name with " + f"'tool not found'; got {served_names}" )