scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 178

test(e2e): cover Together reasoning_effort=none, json_schema, and cache-read pricing

Browse source
This commit is contained in:
mateo-berri 2026-08-28 12:49:09 -07:00
parent 936e07b0a5
commit 2a15bc4fde
2 changed files with 203 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
tests/e2e/coverage_registry/llm_conversational.yaml
View file

@ -87,6 +87,9 @@
- {id: llm.chat_completions.together_ai.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over streaming"}
- {id: llm.chat_completions.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip"}
- {id: llm.chat_completions.together_ai.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together cost header and spend row match the registry price"}
- {id: llm.chat_completions.together_ai.thinking.nonstream.effort_none_disables, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [effort_none_disables], source: "llm_translation/test_together_ai_e2e.py", rationale: "reasoning_effort=none maps to Together's reasoning disable toggle on hybrid models"}
- {id: llm.chat_completions.together_ai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: structured_output, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "response_format json_schema reaches Together and constrains the reply"}
- {id: llm.chat_completions.together_ai.prompt_cache_5m.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: prompt_cache_5m, streaming: nonstream, assertions: [cache_hit, cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together prefix-cache reads bill at cache_read_input_token_cost, not full input price"}
- {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"}
- {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"}
- {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"}

208
tests/e2e/llm_translation/test_together_ai_e2e.py
View file

@ -1,11 +1,14 @@
"""Live e2e: Together AI through the gateway on /chat/completions and /v1/messages.
The reasoning and tool-calling backend is the cheapest live ``together_ai/`` chat row
in the proxy's own cost map that carries both capability flags. Two backends are
pinned because the registry has no flag for what they prove: ``enable_thinking`` is a
Qwen chat-template contract, and MiniMax-M3 is the serverless model whose template
renders a replayed ``reasoning_content`` back into the prompt (Qwen and DeepSeek
silently drop it). MiniMax-M3 honors that replayed field on nearly every call, not
in the proxy's own cost map that carries both capability flags; the structured-output
and cache-pricing backends are likewise the cheapest rows carrying
``supports_response_schema`` and a ``cache_read_input_token_cost``. Two backends are
pinned because the registry has no flag for what they prove: ``enable_thinking`` and
the ``{"reasoning": {"enabled": false}}`` toggle that ``reasoning_effort="none"`` maps
to are Qwen hybrid-model contracts, and MiniMax-M3 is the serverless model whose
template renders a replayed ``reasoning_content`` back into the prompt (Qwen and
DeepSeek silently drop it). MiniMax-M3 honors that replayed field on nearly every call, not
every call (one miss in dozens of otherwise identical calls), so the replay case asks
up to ``REPLAY_ATTEMPTS`` times and fails only when no answer carries the secret, which
a proxy that strips the field guarantees. Requires TOGETHER_API_KEY on the proxy; no
@ -50,7 +53,7 @@ from pydantic import BaseModel
pytestmark = pytest.mark.e2e
TEMPLATE_KWARGS_BACKEND = "together_ai/Qwen/Qwen3.5-9B"
HYBRID_REASONING_BACKEND = "together_ai/Qwen/Qwen3.5-9B"
REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3"
SECRET_PROMPT = "Remember this for later and reply with just OK."
@ -59,6 +62,23 @@ SECRET_QUESTION = "What is my favorite color? Answer with one word."
REPLAY_ATTEMPTS: Final = 3
ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number."
PERSON_PROMPT = "Invent a fictional person."
CACHE_PREFIX_FACTS: Final = 600
CACHE_ATTEMPTS: Final = 3
PERSON_RESPONSE_FORMAT: dict[str, object] = {
"type": "json_schema",
"json_schema": {
"name": "person",
"strict": True,
"schema": {
"type": "object",
"properties": {"name": {"type": "string"}, "age": {"type": "integer"}},
"required": ["name", "age"],
"additionalProperties": False,
},
},
}
WEATHER_PROMPT = "What is the weather in Paris? Use the tool."
WEATHER_REPORT = "Paris: 22 degrees Celsius, clear skies, wind from the northwest at 9 km/h"
COUNTING_PROMPT = "Count from 1 to 20, one number per line."
@ -89,6 +109,13 @@ MESSAGES_WEATHER_TOOL = AnthropicCustomTool(
class _Needs:
function_calling: bool = False
reasoning: bool = False
response_schema: bool = False
cache_read_pricing: bool = False
class _Person(BaseModel):
name: str
age: int
class _WeatherArgs(BaseModel):
@ -145,6 +172,8 @@ def _cheapest_together_chat_model(registry: Mapping[str, CostMapEntry], needs: _
and (entry.output_cost_per_token or 0.0) > 0
and (not needs.function_calling or bool(entry.supports_function_calling))
and (not needs.reasoning or bool(entry.supports_reasoning))
and (not needs.response_schema or bool(entry.supports_response_schema))
and (not needs.cache_read_pricing or (entry.cache_read_input_token_cost or 0.0) > 0)
)
candidates = sorted(
@ -230,6 +259,54 @@ def _weather_call_ids(message: OutMessage) -> tuple[str, ...]:
return tuple(_validated_weather_call_id(call) for call in message.tool_calls)
def _cache_prefix(marker: str) -> str:
facts = " ".join(f"Fact {i}: the {marker} ledger row {i} holds value {i * 7}." for i in range(CACHE_PREFIX_FACTS))
return f"Reference document {marker}:\n{facts}"
def _cached_tokens(response: ChatResponse) -> int:
usage = response.usage
if usage is None or usage.prompt_tokens_details is None:
return 0
return usage.prompt_tokens_details.cached_tokens or 0
def _primed_calls_until_cache_hit(client: PassthroughClient, key: str, model: str) -> Iterator[StreamingResponse]:
"""Together's prefix cache is best-effort, so each attempt primes a brand-new
prefix (fresh marker = fresh cache identity) and re-asks with a different
trailing question; a new marker per attempt keeps a stale attempt's prefix from
polluting the next one."""
for _ in range(CACHE_ATTEMPTS):
prefix = _cache_prefix(unique_marker())
_ = _message(
unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[ChatMessage(role="user", content=f"{prefix}\n\nReply with just OK.")],
max_tokens=16,
),
)
)
)
result = client.proxy.transport.send(
"/chat/completions",
headers=client.proxy.transport.bearer(key),
json=ChatBody(
model=model,
messages=[
ChatMessage(role="user", content=f"{prefix}\n\nWhat is the marker id? Answer with one word.")
],
max_tokens=32,
),
)
require_successful_call(result)
yield result
if _cached_tokens(ChatResponse.model_validate_json(result.body)) > 0:
return
def _weather_call(client: PassthroughClient, key: str, model: str) -> OutMessage:
return _message(
unwrap(
@ -370,7 +447,7 @@ class TestTogetherChatCompletions:
def test_chat_template_kwargs_reach_together(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model, key = _register(client, resources, TEMPLATE_KWARGS_BACKEND)
model, key = _register(client, resources, HYBRID_REASONING_BACKEND)
def ask(chat_template_kwargs: dict[str, bool] | None) -> OutMessage:
return _message(
@ -389,7 +466,7 @@ class TestTogetherChatCompletions:
control = ask(None)
assert control.reasoning_content, (
f"control: {TEMPLATE_KWARGS_BACKEND} returned no reasoning_content by default, "
f"control: {HYBRID_REASONING_BACKEND} returned no reasoning_content by default, "
f"so the disable assertion below cannot be trusted: {control}"
)
treatment = ask({"enable_thinking": False})
@ -474,6 +551,121 @@ class TestTogetherChatCompletions:
f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}"
)
@pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.effort_none_disables")
def test_reasoning_effort_none_reaches_together(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model, key = _register(client, resources, HYBRID_REASONING_BACKEND)
def ask(reasoning_effort: str | None) -> OutMessage:
return _message(
unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)],
max_tokens=1024,
reasoning_effort=reasoning_effort,
),
)
)
)
control = ask(None)
assert control.reasoning_content, (
f"control: {HYBRID_REASONING_BACKEND} returned no reasoning_content by default, "
f"so the disable assertion below cannot be trusted: {control}"
)
treatment = ask("none")
assert not treatment.reasoning_content, (
"reasoning_effort='none' never reached Together as {'reasoning': {'enabled': false}}: "
f"reasoning_content is still present: {treatment}"
)
assert treatment.content and "43" in treatment.content, f"answer lost: {treatment}"
@pytest.mark.covers("llm.chat_completions.together_ai.structured_output.nonstream.works")
def test_response_format_json_schema_shapes_the_reply(
self, client: PassthroughClient, resources: ResourceManager, registry: dict[str, CostMapEntry]
) -> None:
backend = _cheapest_together_chat_model(registry, _Needs(response_schema=True))
model, key = _register(client, resources, backend)
message = _message(
unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[ChatMessage(role="user", content=PERSON_PROMPT)],
max_tokens=1024,
response_format=PERSON_RESPONSE_FORMAT,
),
)
)
)
assert message.content, f"{backend} returned no content: {message}"
person = _Person.model_validate_json(message.content)
assert person.name, f"schema-shaped reply carries an empty name: {message.content!r}"
@pytest.mark.covers("llm.chat_completions.together_ai.prompt_cache_5m.nonstream.cost_logged")
def test_cache_read_tokens_bill_at_the_cache_read_rate(
self,
client: PassthroughClient,
resources: ResourceManager,
registry: dict[str, CostMapEntry],
) -> None:
backend = _cheapest_together_chat_model(registry, _Needs(cache_read_pricing=True))
model, key = _register(client, resources, backend)
price = registry[backend]
assert price.input_cost_per_token and price.output_cost_per_token
cache_read_rate = price.cache_read_input_token_cost
assert cache_read_rate, f"{backend} lost its cache-read price mid-test: {price}"
results = tuple(_primed_calls_until_cache_hit(client, key, model))
result = results[-1]
response = ChatResponse.model_validate_json(result.body)
cached = _cached_tokens(response)
assert cached > 0, (
f"Together reported no cached tokens on {backend} in {len(results)} primed attempts, "
f"so cache-read billing cannot be proven: {response.usage}"
)
usage = response.usage
assert usage is not None and usage.prompt_tokens and usage.completion_tokens, (
f"response carries no usage, so the cost cannot be real: {result.body[:300]}"
)
assert cached <= usage.prompt_tokens, f"cached tokens exceed the prompt: {usage}"
header_cost = result.response_cost
assert header_cost is not None and header_cost > 0, (
f"x-litellm-response-cost header missing or non-positive: {result.headers}"
)
expected = (
(usage.prompt_tokens - cached) * price.input_cost_per_token
+ cached * cache_read_rate
+ usage.completion_tokens * price.output_cost_per_token
)
discount = cached * (price.input_cost_per_token - cache_read_rate)
assert discount > abs(expected) * 1e-2, (
f"the cache-read discount {discount} sits inside the cost tolerance, so this test "
f"could not tell discounted from full-price billing: {usage}"
)
assert _approx_equal(header_cost, expected), (
f"header cost {header_cost} disagrees with the cache-read-discounted registry price for "
f"{backend} at {usage}: expected {expected}"
)
assert response.id, f"response carries no id, so its spend row cannot be found: {result.body[:200]}"
def _priced(rows: list[SpendLogRow]) -> bool:
return any(row.spend is not None for row in rows)
rows = client.proxy.poll_logs_for_request_id(response.id, predicate=_priced)
row = rows[0]
assert row.spend is not None and _approx_equal(row.spend, header_cost), (
f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}"
)
def _tool_use_blocks(content: list[AnthropicContentBlock] | None) -> list[AnthropicContentBlock]:
assert content, f"/v1/messages returned no content blocks: {content}"