diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index 1d4e1e028ca..fc3803e5790 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -75,3 +75,14 @@ - {id: llm.responses.vertex.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Vertex"} - {id: llm.responses.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Azure OpenAI (smoke)"} - {id: llm.responses.azure_openai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Azure OpenAI"} +- {id: llm.chat_completions.together_ai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together reasoning surfaces as reasoning_content (LIT-5960)"} +- {id: llm.chat_completions.together_ai.thinking.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together reasoning deltas stream as reasoning_content"} +- {id: llm.chat_completions.together_ai.thinking.nonstream.template_kwargs_forwarded, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [template_kwargs_forwarded], source: "llm_translation/test_together_ai_e2e.py", rationale: "chat_template_kwargs reaches Together and turns thinking off"} +- {id: llm.chat_completions.together_ai.thinking.nonstream.replayed_reasoning_forwarded, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [replayed_reasoning_forwarded], source: "llm_translation/test_together_ai_e2e.py", rationale: "Replayed reasoning_content survives the Together message transform"} +- {id: llm.chat_completions.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls are not dropped"} +- {id: llm.chat_completions.together_ai.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over streaming"} +- {id: llm.chat_completions.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip"} +- {id: llm.chat_completions.together_ai.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together cost header and spend row match the registry price"} +- {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"} +- {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"} +- {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"} diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py new file mode 100644 index 00000000000..0a446b629de --- /dev/null +++ b/tests/e2e/llm_translation/test_together_ai_e2e.py @@ -0,0 +1,566 @@ +"""Live e2e: Together AI through the gateway, driven the way its users drive it. + +Covers the Together overhaul (LIT-5960): reasoning surfaces as ``reasoning_content`` +non-streaming and streamed, tool calls survive the transform and round-trip a tool +result on both ``/chat/completions`` and ``/v1/messages``, ``chat_template_kwargs`` +reaches Together, and the cost header and spend row agree with the registry price. + +Model selection is registry-driven so the matrix does not rot: the reasoning and +tool-calling deployment is the cheapest live ``together_ai/`` chat model whose cost +map row (read from the proxy's own ``/public/litellm_model_cost_map``) carries +``supports_function_calling`` and ``supports_reasoning``. The one pinned model, +``TEMPLATE_KWARGS_BACKEND``, is pinned on purpose: ``enable_thinking`` is a Qwen +chat-template contract the registry has no flag for. ``REASONING_REPLAY_BACKEND`` is +pinned for the same reason: it is the serverless Together model whose template +renders a replayed ``reasoning_content`` back into the prompt (its prompt_tokens grow +with the replay and it answers from it; Qwen and DeepSeek silently drop it), which is +what makes a stripped replay observable end to end. + +Requires TOGETHER_API_KEY on the proxy (tests/e2e/.env). No skip gate: once the +proxy is up, a failure here is real, per the suite's hard-fail contract. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date + +import pytest +from e2e_config import unique_marker +from e2e_http import StreamingResponse, require_successful_call, unwrap +from lifecycle import ResourceManager +from models import ( + AnthropicAssistantTurn, + AnthropicContentBlock, + AnthropicCustomTool, + AnthropicMessagesBody, + AnthropicToolResultBlock, + AnthropicToolResultTurn, + ChatAssistantTurn, + ChatBody, + ChatMessage, + ChatResponse, + ChatTool, + ChatToolFunction, + ChatToolResultTurn, + CostMapEntry, + JsonSchemaProperty, + LiteLLMParamsBody, + OutMessage, + SpendLogRow, + ToolCall, + ToolInputSchema, +) +from passthrough_client import PassthroughClient +from pydantic import BaseModel + +pytestmark = pytest.mark.e2e + +TEMPLATE_KWARGS_BACKEND = "together_ai/Qwen/Qwen3.5-9B" +REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3" + +SECRET_PROMPT = "Remember this for later and reply with just OK." +SECRET_REASONING = "The user told me their favorite color is chartreuse. I must remember it." +SECRET_QUESTION = "What is my favorite color? Answer with one word." + +ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number." +WEATHER_PROMPT = "What is the weather in Paris? Use the tool." +WEATHER_REPORT = "Paris: 22 degrees Celsius, clear skies, wind from the northwest at 9 km/h" +COUNTING_PROMPT = "Count from 1 to 20, one number per line." + +WEATHER_TOOL = ChatTool( + function=ChatToolFunction( + name="get_weather", + description="Get the current weather for a location.", + parameters={ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + ) +) + +MESSAGES_WEATHER_TOOL = AnthropicCustomTool( + name="get_weather", + description="Get the current weather for a location.", + input_schema=ToolInputSchema( + properties={"location": JsonSchemaProperty(type="string")}, + required=["location"], + ), +) + + +@dataclass(frozen=True, slots=True) +class _Needs: + function_calling: bool = False + reasoning: bool = False + + +class _WeatherArgs(BaseModel): + location: str + + +class _StreamToolCallFunction(BaseModel): + name: str | None = None + arguments: str | None = None + + +class _StreamToolCall(BaseModel): + function: _StreamToolCallFunction | None = None + + +class _StreamDelta(BaseModel): + content: str | None = None + reasoning_content: str | None = None + tool_calls: list[_StreamToolCall] | None = None + + +class _StreamChoice(BaseModel): + delta: _StreamDelta | None = None + + +class _StreamChunk(BaseModel): + choices: list[_StreamChoice] = [] + + +class _MessagesEventDelta(BaseModel): + type: str | None = None + text: str = "" + + +class _MessagesStreamEvent(BaseModel): + type: str + delta: _MessagesEventDelta | None = None + + +def _approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +def _cheapest_together_chat_model(registry: Mapping[str, CostMapEntry], needs: _Needs) -> str: + today = date.today().isoformat() + + def qualifies(name: str, entry: CostMapEntry) -> bool: + return ( + name.startswith("together_ai/") + and entry.litellm_provider == "together_ai" + and entry.mode == "chat" + and (entry.deprecation_date is None or entry.deprecation_date > today) + and (entry.input_cost_per_token or 0.0) > 0 + and (entry.output_cost_per_token or 0.0) > 0 + and (not needs.function_calling or bool(entry.supports_function_calling)) + and (not needs.reasoning or bool(entry.supports_reasoning)) + ) + + candidates = sorted( + (name for name, entry in registry.items() if qualifies(name, entry)), + key=lambda name: ( + registry[name].input_cost_per_token or 0.0, + registry[name].output_cost_per_token or 0.0, + name, + ), + ) + assert candidates, f"no live together_ai chat model in the proxy's cost map satisfies {needs}" + return candidates[0] + + +@pytest.fixture(scope="module") +def registry(client: PassthroughClient) -> dict[str, CostMapEntry]: + return client.proxy.model_cost_map() + + +@pytest.fixture(scope="module") +def reasoning_tool_backend(registry: dict[str, CostMapEntry]) -> str: + return _cheapest_together_chat_model(registry, _Needs(function_calling=True, reasoning=True)) + + +def _register(client: PassthroughClient, resources: ResourceManager, backend: str) -> tuple[str, str]: + model = f"e2e-together-{unique_marker()}" + model_id = client.proxy.create_model( + model, LiteLLMParamsBody(model=backend, api_key="os.environ/TOGETHER_API_KEY") + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model, resources.key() + + +def _message(response: ChatResponse) -> OutMessage: + assert response.choices, f"Together returned no choices: {response}" + message = response.choices[0].message + assert message is not None, f"Together choice has no message: {response}" + return message + + +def _deltas(result: StreamingResponse) -> list[_StreamDelta]: + require_successful_call(result) + assert result.is_streaming, f"response was not streamed: {result.headers}" + assert not result.stream_error, f"stream errored: {result.stream_error}" + assert result.stream_done, f"stream never reached [DONE]: {result.stream_events[-3:]}" + return [ + choice.delta + for event in result.stream_events + for choice in _StreamChunk.model_validate_json(event).choices + if choice.delta is not None + ] + + +def _single_weather_call(message: OutMessage) -> ToolCall: + assert message.tool_calls, f"Together dropped the tool call: {message}" + assert len(message.tool_calls) == 1, f"expected one tool call, got {message.tool_calls}" + call = message.tool_calls[0] + assert call.id, f"tool call carries no id, so a tool result cannot answer it: {call}" + assert call.function.name == "get_weather", f"wrong tool called: {call}" + assert call.function.arguments, f"tool call carries no arguments: {call}" + args = _WeatherArgs.model_validate_json(call.function.arguments) + assert "paris" in args.location.lower(), f"tool arguments lost the location: {args}" + return call + + +def _weather_call(client: PassthroughClient, key: str, model: str) -> OutMessage: + return _message( + unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=WEATHER_PROMPT)], + tools=[WEATHER_TOOL], + max_tokens=512, + ), + ) + ) + ) + + +class TestTogetherChatCompletions: + @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.works") + def test_reasoning_surfaces_as_reasoning_content( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + + message = _message( + unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)], + max_tokens=1024, + ), + ) + ) + ) + assert message.reasoning_content, ( + f"{reasoning_tool_backend} reasons, but no reasoning_content came back: {message}" + ) + assert message.content and "43" in message.content, f"answer lost: {message}" + + @pytest.mark.covers("llm.chat_completions.together_ai.thinking.stream.works") + def test_reasoning_streams_as_reasoning_content_deltas( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + + deltas = _deltas( + client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)], + max_tokens=1024, + stream=True, + ), + ) + ) + reasoning = "".join(delta.reasoning_content or "" for delta in deltas) + content = "".join(delta.content or "" for delta in deltas) + assert reasoning, f"stream carried no reasoning_content deltas: {deltas[:5]}" + assert "43" in content, f"streamed answer lost: {content!r}" + + @pytest.mark.covers("llm.chat_completions.together_ai.tool_use.nonstream.works") + def test_tool_call_is_returned( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + _single_weather_call(_weather_call(client, key, model)) + + @pytest.mark.covers("llm.chat_completions.together_ai.tool_use.stream.works") + def test_tool_call_is_streamed( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + + deltas = _deltas( + client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=WEATHER_PROMPT)], + tools=[WEATHER_TOOL], + max_tokens=512, + stream=True, + ), + ) + ) + calls = [ + call.function + for delta in deltas + for call in delta.tool_calls or [] + if call.function is not None + ] + assert calls, f"stream carried no tool call deltas: {deltas[:5]}" + names = {call.name for call in calls if call.name} + assert names == {"get_weather"}, f"unexpected streamed tool names: {names}" + arguments = "".join(call.arguments or "" for call in calls) + args = _WeatherArgs.model_validate_json(arguments) + assert "paris" in args.location.lower(), f"streamed tool arguments lost the location: {args}" + + @pytest.mark.covers("llm.chat_completions.together_ai.multi_turn.nonstream.works") + def test_tool_result_round_trip( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + first = _weather_call(client, key, model) + call = _single_weather_call(first) + assert call.id is not None + + answer = _message( + unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content=WEATHER_PROMPT), + ChatAssistantTurn( + content=first.content, + reasoning_content=first.reasoning_content, + tool_calls=first.tool_calls, + ), + ChatToolResultTurn(tool_call_id=call.id, content=WEATHER_REPORT), + ], + tools=[WEATHER_TOOL], + max_tokens=512, + ), + ) + ) + ) + assert answer.content and "22" in answer.content, ( + f"the model never saw the tool result: {answer}" + ) + + @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.template_kwargs_forwarded") + def test_chat_template_kwargs_reach_together( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + """Control first: the pinned Qwen model thinks by default, so the treatment's + missing reasoning_content proves ``enable_thinking: false`` reached Together + rather than the model simply not reasoning.""" + model, key = _register(client, resources, TEMPLATE_KWARGS_BACKEND) + + def ask(chat_template_kwargs: dict[str, bool] | None) -> OutMessage: + return _message( + unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)], + max_tokens=1024, + chat_template_kwargs=chat_template_kwargs, + ), + ) + ) + ) + + control = ask(None) + assert control.reasoning_content, ( + f"control: {TEMPLATE_KWARGS_BACKEND} returned no reasoning_content by default, " + f"so the disable assertion below cannot be trusted: {control}" + ) + treatment = ask({"enable_thinking": False}) + assert not treatment.reasoning_content, ( + "chat_template_kwargs={'enable_thinking': False} did not reach Together: " + f"reasoning_content is still present: {treatment}" + ) + assert treatment.content and "43" in treatment.content, f"answer lost: {treatment}" + + @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.replayed_reasoning_forwarded") + def test_replayed_reasoning_content_reaches_together( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + """The only place the fact lives is the replayed assistant turn's + ``reasoning_content``, so the model can answer only if the Together transform + forwarded it (LIT-5960: it used to strip it with the thinking blocks).""" + model, key = _register(client, resources, REASONING_REPLAY_BACKEND) + + answer = _message( + unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage(role="user", content=SECRET_PROMPT), + ChatAssistantTurn(content="OK.", reasoning_content=SECRET_REASONING), + ChatMessage(role="user", content=SECRET_QUESTION), + ], + max_tokens=512, + ), + ) + ) + ) + assert answer.content and "chartreuse" in answer.content.lower(), ( + f"the replayed reasoning_content never reached Together: {answer}" + ) + + @pytest.mark.covers("llm.chat_completions.together_ai.basic.nonstream.cost_logged") + def test_cost_header_and_spend_row_match_the_registry_price( + self, + client: PassthroughClient, + resources: ResourceManager, + registry: dict[str, CostMapEntry], + reasoning_tool_backend: str, + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + + result = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(key), + json=ChatBody( + model=model, + messages=[ChatMessage(role="user", content=f"{ARITHMETIC_PROMPT} {unique_marker()}")], + max_tokens=1024, + ), + ) + require_successful_call(result) + response = ChatResponse.model_validate_json(result.body) + usage = response.usage + assert usage is not None and usage.prompt_tokens and usage.completion_tokens, ( + f"response carries no usage, so the cost cannot be real: {result.body[:300]}" + ) + header_cost = result.response_cost + assert header_cost is not None and header_cost > 0, ( + f"x-litellm-response-cost header missing or non-positive: {result.headers}" + ) + + price = registry[reasoning_tool_backend] + assert price.input_cost_per_token and price.output_cost_per_token + cached = (usage.prompt_tokens_details.cached_tokens or 0) if usage.prompt_tokens_details else 0 + expected = ( + (usage.prompt_tokens - cached) * price.input_cost_per_token + + cached * (price.cache_read_input_token_cost or price.input_cost_per_token) + + usage.completion_tokens * price.output_cost_per_token + ) + assert _approx_equal(header_cost, expected), ( + f"header cost {header_cost} disagrees with the registry price for " + f"{reasoning_tool_backend} at {usage}: expected {expected}" + ) + + def _priced(rows: list[SpendLogRow]) -> bool: + return any(row.spend is not None and row.spend > 0 for row in rows) + + rows = client.proxy.poll_logs_for_key(key, predicate=_priced) + priced = [row for row in rows if row.spend is not None and row.spend > 0] + assert priced, f"no priced spend row landed for key {key}; got {rows}" + row = priced[0] + assert row.custom_llm_provider == "together_ai", f"spend row misattributed: {row}" + assert row.spend is not None and _approx_equal(row.spend, header_cost), ( + f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}" + ) + + +def _tool_use_blocks(content: list[AnthropicContentBlock] | None) -> list[AnthropicContentBlock]: + assert content, f"/v1/messages returned no content blocks: {content}" + return [block for block in content if block.type == "tool_use"] + + +def _messages_weather_call( + client: PassthroughClient, key: str, model: str +) -> tuple[list[AnthropicContentBlock], AnthropicContentBlock]: + response = unwrap( + client.proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=512, + tools=[MESSAGES_WEATHER_TOOL], + messages=[ChatMessage(role="user", content=WEATHER_PROMPT)], + ), + ) + ) + tool_uses = _tool_use_blocks(response.content) + assert len(tool_uses) == 1, f"expected one tool_use block, got {response.content}" + block = tool_uses[0] + assert block.name == "get_weather", f"wrong tool called: {block}" + assert block.id, f"tool_use block carries no id, so a tool_result cannot answer it: {block}" + assert response.content is not None + return response.content, block + + +class TestTogetherMessages: + @pytest.mark.covers("llm.messages.together_ai.tool_use.nonstream.works") + def test_tool_use_block_is_returned( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + _messages_weather_call(client, key, model) + + @pytest.mark.covers("llm.messages.together_ai.multi_turn.nonstream.works") + def test_tool_result_round_trip( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + first_content, block = _messages_weather_call(client, key, model) + assert block.id is not None + + response = unwrap( + client.proxy.messages( + key, + AnthropicMessagesBody( + model=model, + max_tokens=512, + tools=[MESSAGES_WEATHER_TOOL], + messages=[ + ChatMessage(role="user", content=WEATHER_PROMPT), + AnthropicAssistantTurn(content=first_content), + AnthropicToolResultTurn( + content=[AnthropicToolResultBlock(tool_use_id=block.id, content=WEATHER_REPORT)] + ), + ], + ), + ) + ) + assert response.content, f"/v1/messages returned no content blocks: {response}" + text = "".join(block.text or "" for block in response.content if block.type == "text") + assert "22" in text, f"the model never saw the tool result: {response.content}" + + @pytest.mark.covers("llm.messages.together_ai.basic.stream.works") + def test_streams_text_deltas( + self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str + ) -> None: + model, key = _register(client, resources, reasoning_tool_backend) + + result = client.proxy.messages_stream( + key, + AnthropicMessagesBody( + model=model, + max_tokens=512, + stream=True, + messages=[ChatMessage(role="user", content=COUNTING_PROMPT)], + ), + ) + require_successful_call(result) + assert result.is_streaming, f"response was not streamed: {result.headers}" + assert not result.stream_error, f"stream errored: {result.stream_error}" + events = [_MessagesStreamEvent.model_validate_json(event) for event in result.stream_events] + types = [event.type for event in events] + text_deltas = [ + event.delta.text + for event in events + if event.type == "content_block_delta" and event.delta is not None and event.delta.text + ] + assert len(text_deltas) >= 2, f"stream was not incremental: {types}" + assert "20" in "".join(text_deltas), f"streamed text lost the answer: {text_deltas}" + assert "message_stop" in types, f"stream never reached message_stop: {types}" diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 5e2cb90958e..3e8c55b1461 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -217,9 +217,45 @@ class McpChatTool(BaseModel): allowed_tools: list[str] | None = None +class ToolCallFunction(BaseModel): + name: str | None = None + arguments: str | None = None + + +class ToolCall(BaseModel): + """A tool call as the model emitted it. `id` and `type` are what a client + replays in the assistant turn so the tool result can be matched to it.""" + + id: str | None = None + type: str | None = None + function: ToolCallFunction = ToolCallFunction() + + +class ChatAssistantTurn(BaseModel): + """An assistant turn replayed into history as the client received it: the tool + calls a tool result answers, or the reasoning a thinking-preserving chat + template consumes on the next turn.""" + + role: Literal["assistant"] = "assistant" + content: str | None = None + reasoning_content: str | None = None + tool_calls: list[ToolCall] | None = None + + +class ChatToolResultTurn(BaseModel): + """The tool-role answer to a `ToolCall`; `tool_call_id` must be the id the model emitted.""" + + role: Literal["tool"] = "tool" + tool_call_id: str + content: str + + +type ChatTurn = ChatMessage | ChatAssistantTurn | ChatToolResultTurn + + class ChatBody(BaseModel): model: str - messages: list[ChatMessage] + messages: Sequence[ChatTurn] stream: bool = False max_tokens: int | None = None max_completion_tokens: int | None = None @@ -233,6 +269,7 @@ class ChatBody(BaseModel): tool_choice: str | None = None guardrails: list[str] | None = None response_format: dict[str, object] | None = None + chat_template_kwargs: dict[str, bool] | None = None cache: dict[str, bool] | None = {"no-cache": True} @@ -259,15 +296,6 @@ class ReliabilityChatBody(ChatBody): router_settings_override: RouterSettingsOverride | None = None -class ToolCallFunction(BaseModel): - name: str | None = None - arguments: str | None = None - - -class ToolCall(BaseModel): - function: ToolCallFunction = ToolCallFunction() - - class McpToolFunctionRef(BaseModel): name: str @@ -400,6 +428,7 @@ class AnthropicContentBlock(BaseModel): type: str | None = None text: str | None = None id: str | None = None + name: str | None = None class AnthropicToolResultBlock(BaseModel): @@ -689,6 +718,27 @@ class ModelInfoResponse(BaseModel): data: list[ModelInfoEntry] = [] +class CostMapEntry(BaseModel): + """One `/public/litellm_model_cost_map` row: the price and capability flags the + proxy resolves for a `/` key. Tests pick a deployment from + these flags instead of hardcoding a model name that rots.""" + + model_config = ConfigDict(extra="ignore") + litellm_provider: str | None = None + mode: str | None = None + deprecation_date: str | None = None + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + supports_function_calling: bool | None = None + supports_reasoning: bool | None = None + supports_response_schema: bool | None = None + + +class CostMap(RootModel[dict[str, CostMapEntry]]): + pass + + class FileEntry(BaseModel): id: str diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py index d12364e1794..c56a0649f58 100644 --- a/tests/e2e/proxy_client.py +++ b/tests/e2e/proxy_client.py @@ -29,6 +29,8 @@ from models import ( AnthropicMessagesResponse, ChatBody, ChatResponse, + CostMap, + CostMapEntry, CountTokensBody, CountTokensResponse, CredentialCreateBody, @@ -251,6 +253,19 @@ class ProxyClient: ) ).data + def model_cost_map(self) -> dict[str, CostMapEntry]: + """The registry the proxy prices with and gates capabilities on, keyed + `/`, read from the proxy itself so a test sees exactly the + map the deployment under test resolves against.""" + return unwrap( + self.transport.get( + "/public/litellm_model_cost_map", + headers=self.transport.master, + params=NoBody(), + response_type=CostMap, + ) + ).root + def list_files(self, key: str) -> Result[FileListResponse]: return self.transport.get( "/v1/files",