From 2f76aa1b1bcd64bd5e3101592fbd9be48f4b011b Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 19:51:26 -0700 Subject: [PATCH] test(integration): move Xiaomi MiMo coverage from live e2e to the providers wire shard (#42395) --- .../coverage_registry/llm_conversational.yaml | 3 - .../llm_translation/test_xiaomi_mimo_e2e.py | 214 --------------- tests/integration/contracts.json | 12 + .../providers/test_xiaomi_mimo_wire.py | 258 ++++++++++++++++++ 4 files changed, 270 insertions(+), 217 deletions(-) delete mode 100644 tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py create mode 100644 tests/integration/providers/test_xiaomi_mimo_wire.py diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 64335aa560c..49d4d92ff0b 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -89,9 +89,6 @@ - {id: llm.chat_completions.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip"} - {id: llm.chat_completions.together_ai.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together cost header and spend row match the registry price"} - {id: llm.chat_completions.together_ai.thinking.nonstream.effort_none_disables, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [effort_none_disables], source: "llm_translation/test_together_ai_e2e.py", rationale: "reasoning_effort=none maps to Together's reasoning disable toggle on hybrid models"} -- {id: llm.chat_completions.xiaomi_mimo.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "Native MiMo v2.6 rows price the cost header and spend row from the cost map"} -- {id: llm.chat_completions.xiaomi_mimo.thinking.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: thinking, streaming: stream, assertions: [works], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "MiMo reasoning deltas stream as reasoning_content"} -- {id: llm.chat_completions.xiaomi_mimo.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: xiaomi_mimo, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_xiaomi_mimo_e2e.py", rationale: "MiMo tool calls are not dropped"} - {id: llm.chat_completions.together_ai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: structured_output, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "response_format json_schema reaches Together and constrains the reply"} - {id: llm.chat_completions.together_ai.prompt_cache_5m.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: prompt_cache_5m, streaming: nonstream, assertions: [cache_hit, cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together prefix-cache reads bill at cache_read_input_token_cost, not full input price"} - {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"} diff --git a/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py b/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py deleted file mode 100644 index efca216634b..00000000000 --- a/tests/e2e/llm_translation/test_xiaomi_mimo_e2e.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Live e2e: Xiaomi MiMo v2.6 through the gateway on /chat/completions. - -Both native ``xiaomi_mimo/`` v2.6 rows (pro and flash) are registered via -``/model/new`` and driven against Xiaomi's own endpoint. What the gateway owes -us is that the reasoning chain surfaces as ``reasoning_content``, tool calls -survive translation, and the cost header plus spend row follow the proxy's own -cost-map price for the row (read back from ``/model/info``, never pinned here). -Requires XIAOMI_MIMO_API_KEY on the proxy; no skip gate. -""" - -from __future__ import annotations - -from typing import Final - -import pytest -from e2e_config import unique_marker -from e2e_http import StreamingResponse, require_successful_call, unwrap -from lifecycle import ResourceManager -from models import ( - ChatBody, - ChatMessage, - ChatResponse, - ChatTool, - ChatToolFunction, - CostMapEntry, - LiteLLMParamsBody, - OutMessage, - SpendLogRow, -) -from passthrough_client import PassthroughClient -from pydantic import BaseModel - -pytestmark = pytest.mark.e2e - -BACKENDS: Final = ("xiaomi_mimo/mimo-v2.6-pro", "xiaomi_mimo/mimo-v2.6-flash") -ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number." -WEATHER_PROMPT = "What is the weather in Paris? Use the tool." -COUNTING_PROMPT = "Count from 1 to 50, one number per line." - -WEATHER_TOOL = ChatTool( - function=ChatToolFunction( - name="get_weather", - description="Get the current weather for a location.", - parameters={ - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"], - }, - ) -) - - -class _WeatherArgs(BaseModel): - location: str - - -class _StreamDelta(BaseModel): - content: str | None = None - reasoning_content: str | None = None - - -class _StreamChoice(BaseModel): - delta: _StreamDelta | None = None - - -class _StreamChunk(BaseModel): - choices: list[_StreamChoice] = [] - - -def _approx_equal(actual: float, expected: float) -> bool: - return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) - - -@pytest.fixture(scope="module") -def registry(client: PassthroughClient) -> dict[str, CostMapEntry]: - return client.proxy.model_cost_map() - - -def _register(client: PassthroughClient, resources: ResourceManager, backend: str) -> tuple[str, str]: - model = f"e2e-xiaomi-{unique_marker()}" - model_id = client.proxy.create_model( - model, LiteLLMParamsBody(model=backend, api_key="os.environ/XIAOMI_MIMO_API_KEY") - ) - resources.defer(lambda: client.proxy.delete_model(model_id)) - return model, resources.key() - - -def _message(response: ChatResponse) -> OutMessage: - assert response.choices, f"Xiaomi returned no choices: {response}" - message = response.choices[0].message - assert message is not None, f"Xiaomi choice has no message: {response}" - return message - - -def _deltas(result: StreamingResponse) -> list[_StreamDelta]: - require_successful_call(result) - assert result.is_streaming, f"response was not streamed: {result.headers}" - assert not result.stream_error, f"stream errored: {result.stream_error}" - assert result.stream_done, f"stream never reached [DONE]: {result.stream_events[-3:]}" - return [ - choice.delta - for event in result.stream_events - for choice in _StreamChunk.model_validate_json(event).choices - if choice.delta is not None - ] - - -@pytest.mark.parametrize("backend", BACKENDS) -class TestXiaomiMimoChatCompletions: - @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.basic.nonstream.cost_logged") - def test_cost_header_and_spend_row_match_the_registry_price( - self, - client: PassthroughClient, - resources: ResourceManager, - registry: dict[str, CostMapEntry], - backend: str, - ) -> None: - price = registry.get(backend) - assert price is not None, f"{backend} has no row in the proxy's cost map, so native calls would bill $0" - assert price.litellm_provider == "xiaomi_mimo", f"{backend} is filed under the wrong provider: {price}" - assert price.input_cost_per_token and price.output_cost_per_token, f"{backend} carries no price: {price}" - model, key = _register(client, resources, backend) - - result = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(key), - json=ChatBody( - model=model, - messages=[ChatMessage(role="user", content=f"{ARITHMETIC_PROMPT} {unique_marker()}")], - max_tokens=1024, - ), - ) - require_successful_call(result) - response = ChatResponse.model_validate_json(result.body) - message = _message(response) - assert message.content and "43" in message.content, f"answer lost: {message}" - assert message.reasoning_content, f"{backend} reasons, but no reasoning_content came back: {message}" - - usage = response.usage - assert usage is not None and usage.prompt_tokens and usage.completion_tokens, ( - f"response carries no usage, so the cost cannot be real: {result.body[:300]}" - ) - header_cost = result.response_cost - assert header_cost is not None and header_cost > 0, ( - f"x-litellm-response-cost header missing or non-positive: {result.headers}" - ) - cached = (usage.prompt_tokens_details.cached_tokens or 0) if usage.prompt_tokens_details else 0 - expected = ( - (usage.prompt_tokens - cached) * price.input_cost_per_token - + cached * (price.cache_read_input_token_cost or 0.0) - + usage.completion_tokens * price.output_cost_per_token - ) - assert _approx_equal(header_cost, expected), ( - f"header cost {header_cost} disagrees with the registry price for {backend} at {usage}: expected {expected}" - ) - - def _priced(rows: list[SpendLogRow]) -> bool: - return any(row.spend is not None and row.spend > 0 for row in rows) - - rows = client.proxy.poll_logs_for_key(key, predicate=_priced) - priced = [row for row in rows if row.spend is not None and row.spend > 0] - assert priced, f"no priced spend row landed for key {key}; got {rows}" - row = priced[0] - assert row.custom_llm_provider == "xiaomi_mimo", f"spend row misattributed: {row}" - assert row.spend is not None and _approx_equal(row.spend, header_cost), ( - f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}" - ) - - @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.thinking.stream.works") - def test_reasoning_and_answer_stream_as_deltas( - self, client: PassthroughClient, resources: ResourceManager, backend: str - ) -> None: - model, key = _register(client, resources, backend) - - deltas = _deltas( - client.proxy.chat_stream( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content=COUNTING_PROMPT)], - max_tokens=2048, - stream=True, - ), - ) - ) - reasoning = "".join(delta.reasoning_content or "" for delta in deltas) - content = "".join(delta.content or "" for delta in deltas) - assert reasoning, f"stream carried no reasoning_content deltas: {deltas[:5]}" - assert "50" in content, f"streamed answer lost: {content[:300]!r}" - - @pytest.mark.covers("llm.chat_completions.xiaomi_mimo.tool_use.nonstream.works") - def test_tool_call_is_returned(self, client: PassthroughClient, resources: ResourceManager, backend: str) -> None: - model, key = _register(client, resources, backend) - - message = _message( - unwrap( - client.proxy.chat( - key, - ChatBody( - model=model, - messages=[ChatMessage(role="user", content=WEATHER_PROMPT)], - tools=[WEATHER_TOOL], - max_tokens=1024, - ), - ) - ) - ) - assert message.tool_calls, f"{backend} dropped the tool call: {message}" - call = message.tool_calls[0] - assert call.id, f"tool call carries no id, so a tool result cannot answer it: {call}" - assert call.function.name == "get_weather", f"wrong tool called: {call}" - assert call.function.arguments, f"tool call carries no arguments: {call}" - args = _WeatherArgs.model_validate_json(call.function.arguments) - assert "paris" in args.location.lower(), f"tool arguments lost the location: {args}" diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 2cdd4c17e39..6629b1fa1f4 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -190,6 +190,18 @@ "tests/integration/providers/test_fal_ai_chat_wire.py::test_fal_moondream3_chat_sends_prompt_image_and_reasoning": [ "other.provider_wire.fal_ai.moondream3_chat_query_wire_and_token_pricing" ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-pro]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price[mimo-v2.6-flash]": [ + "other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas": [ + "other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas" + ], + "tests/integration/providers/test_xiaomi_mimo_wire.py::test_xiaomi_mimo_tool_call_is_forwarded_and_returned": [ + "other.provider_wire.xiaomi_mimo.tool_call_survives_translation" + ], "tests/integration/providers/test_fal_ai_video_wire.py::test_fal_h3_video_create_uses_canonical_body_and_status_path": [ "other.provider_wire.fal_ai.video_queue_create_status_and_content_download" ], diff --git a/tests/integration/providers/test_xiaomi_mimo_wire.py b/tests/integration/providers/test_xiaomi_mimo_wire.py new file mode 100644 index 00000000000..96b9dc17bdf --- /dev/null +++ b/tests/integration/providers/test_xiaomi_mimo_wire.py @@ -0,0 +1,258 @@ +import json +import uuid +from collections.abc import Mapping +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +_BACKENDS: Final = ("mimo-v2.6-pro", "mimo-v2.6-flash") +_API_KEY: Final = "synthetic-xiaomi-key" +_ARITHMETIC_PROMPT: Final = "What is 17 + 26? Answer with just the number." +_WEATHER_PROMPT: Final = "What is the weather in Paris? Use the tool." +_COUNTING_PROMPT: Final = "Count from 1 to 5, one number per line." +_WEATHER_TOOL: Final[JsonValue] = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +} +_COST_MAP_PATH: Final = Path(__file__).resolve().parents[3] / "model_prices_and_context_window.json" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_COST_MAP: Final = TypeAdapter(dict[str, dict[str, object]]) + + +class _Delta(BaseModel): + model_config = ConfigDict(extra="ignore") + content: str | None = None + reasoning_content: str | None = None + + +class _Choice(BaseModel): + model_config = ConfigDict(extra="ignore") + delta: _Delta + finish_reason: str | None = None + + +class _Chunk(BaseModel): + model_config = ConfigDict(extra="ignore") + id: str + choices: tuple[_Choice, ...] + + +def _catalog_cost(backend: str, field: str) -> float: + cost_map: Final = _COST_MAP.validate_json(_COST_MAP_PATH.read_bytes()) + cost_value: Final = cost_map[f"xiaomi_mimo/{backend}"][field] + assert isinstance(cost_value, (int, float)) + return float(cost_value) + + +def _approx(value: float) -> object: + return pytest.approx(value, rel=1e-6) # pyright: ignore[reportUnknownMemberType] # pytest lacks typed approx stubs + + +def _completion(identity: str, backend: str, message: Mapping[str, object], finish: str) -> bytes: + return json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": backend, + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}, + } + ).encode() + + +def _frame(identity: str, backend: str, delta: Mapping[str, object], finish: str | None = None) -> bytes: + value: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": backend, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish}], + } + return b"data: " + json.dumps(value).encode() + b"\n\n" + + +def _assert_provider_request(request: Request, backend: str, prompt: str) -> dict[str, JsonValue]: + assert request.method == "POST" + assert request.target == "/chat/completions" + assert request.headers["authorization"] == f"Bearer {_API_KEY}" + assert request.headers["content-type"] == "application/json" + body: Final = _JSON_OBJECT.validate_json(request.body) + assert body["model"] == backend + assert body["messages"] == [{"role": "user", "content": prompt}] + return body + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.reasoning_content_and_registry_pricing") +@pytest.mark.parametrize("backend", _BACKENDS) +def test_xiaomi_mimo_nonstream_surfaces_reasoning_and_charges_registry_price(gateway: Gateway, backend: str) -> None: + identity: Final = f"xiaomi-cost-{uuid.uuid4().hex}" + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _ARITHMETIC_PROMPT) + assert body["max_tokens"] == 256 + assert "max_completion_tokens" not in body + return Reply( + body=_completion( + identity, + backend, + {"role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43."}, + "stop", + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _ARITHMETIC_PROMPT}], + "max_completion_tokens": 256, + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["id"] == identity + assert payload["choices"] == [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "43", + "reasoning_content": "17 plus 26 is 43.", + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert payload["usage"] == {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64} + expected_cost: Final = 23 * _catalog_cost(backend, "input_cost_per_token") + 41 * _catalog_cost( + backend, "output_cost_per_token" + ) + assert float(response.headers["x-litellm-response-cost"]) == _approx(expected_cost) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + rows: Final = eventually( + lambda: read_rows( + 'SELECT spend, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (identity,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == (23, 41) + spend: Final = rows[0]["spend"] + assert isinstance(spend, (int, float, str)) + assert float(spend) == _approx(expected_cost) + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.reasoning_and_answer_stream_as_deltas") +def test_xiaomi_mimo_stream_delivers_reasoning_then_answer_deltas(gateway: Gateway) -> None: + backend: Final = _BACKENDS[0] + identity: Final = f"xiaomi-stream-{uuid.uuid4().hex}" + frames: Final = ( + _frame(identity, backend, {"role": "assistant", "reasoning_content": "Count "}), + _frame(identity, backend, {"reasoning_content": "up by one."}), + _frame(identity, backend, {"content": "1\n2\n"}), + _frame(identity, backend, {"content": "3\n4\n5"}), + _frame(identity, backend, {}, finish="stop"), + b"data: [DONE]\n\n", + ) + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _COUNTING_PROMPT) + assert body["stream"] is True + return Reply(content_type="text/event-stream", chunks=frames) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + with gateway.client.stream( + "POST", + "/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": _COUNTING_PROMPT}], "stream": True}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: ")) + assert lines[-1] == "data: [DONE]" + chunks: Final = tuple(_Chunk.model_validate_json(line.removeprefix("data: ")) for line in lines[:-1]) + assert {chunk.id for chunk in chunks} == {identity} + choices: Final = tuple(choice for chunk in chunks for choice in chunk.choices) + assert "".join(choice.delta.reasoning_content or "" for choice in choices) == "Count up by one." + assert "".join(choice.delta.content or "" for choice in choices) == "1\n2\n3\n4\n5" + assert tuple(choice.finish_reason for choice in choices if choice.finish_reason) == ("stop",) + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")] + + +@pytest.mark.covers("other.provider_wire.xiaomi_mimo.tool_call_survives_translation") +def test_xiaomi_mimo_tool_call_is_forwarded_and_returned(gateway: Gateway) -> None: + backend: Final = _BACKENDS[1] + identity: Final = f"xiaomi-tool-{uuid.uuid4().hex}" + tool_call: Final = { + "id": "call_paris", + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps({"city": "Paris"})}, + } + + def respond(request: Request) -> Reply: + body: Final = _assert_provider_request(request, backend, _WEATHER_PROMPT) + assert body["tools"] == [_WEATHER_TOOL] + assert body["tool_choice"] == "auto" + return Reply( + body=_completion( + identity, + backend, + { + "role": "assistant", + "content": None, + "reasoning_content": "Need the tool.", + "tool_calls": [tool_call], + }, + "tool_calls", + ) + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"xiaomi_mimo/{backend}", api_base=wire.url, api_key=_API_KEY) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _WEATHER_PROMPT}], + "tools": [_WEATHER_TOOL], + "tool_choice": "auto", + }, + ) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + assert payload["choices"] == [ + { + "finish_reason": "tool_calls", + "index": 0, + "message": { + "role": "assistant", + "content": None, + "reasoning_content": "Need the tool.", + "tool_calls": [tool_call], + "provider_specific_fields": {"refusal": None}, + }, + "provider_specific_fields": {}, + } + ] + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]