fix(mistral): infer finish_reason=length for truncated Conversations responses

The Conversations API returns no finish/stop reason field (verified against the
live API, including a forced max_tokens truncation), so transform_response
hardcoded finish_reason="stop"; a client checking finish_reason == "length" to
detect truncation got a wrong signal when the model stopped on max_tokens. With
nothing on the wire to forward, infer it from the token budget: return "length"
when completion_tokens reached the requested max_tokens, otherwise "stop"
This commit is contained in:
ElDjeee 2026-07-06 14:27:24 +02:00
parent ea340d6d2f
commit 30612b1d16
3 changed files with 61 additions and 1 deletions

View file

@ -260,6 +260,18 @@ class MistralConversationsConfig(MistralConfig):
setattr(usage_obj, "web_search_premium_requests", web_search_premium_requests)
return usage_obj
@staticmethod
def _finish_reason(request_data: dict, usage: Optional[MistralConversationUsage]) -> str:
"""The Conversations API returns no finish/stop reason, so infer truncation
from the token budget: ``length`` when the completion filled the requested
``max_tokens``, otherwise ``stop``."""
args = STR_OBJ_DICT.validate_python(request_data.get("completion_args") or {})
max_tokens = args.get("max_tokens")
completion_tokens = usage.completion_tokens if usage else None
if isinstance(max_tokens, int) and isinstance(completion_tokens, int) and completion_tokens >= max_tokens:
return "length"
return "stop"
def transform_response(
self,
model: str,
@ -286,7 +298,9 @@ class MistralConversationsConfig(MistralConfig):
content=text or None,
annotations=annotations or None,
)
model_response.choices = [Choices(index=0, message=message, finish_reason="stop")]
model_response.choices = [
Choices(index=0, message=message, finish_reason=self._finish_reason(request_data, parsed.usage))
]
model_response.model = model
if parsed.conversation_id:
model_response.id = parsed.conversation_id

View file

@ -89,3 +89,24 @@ class TestMistralCompletion(BaseLLMChatTest):
assert response is not None
assert response.choices[0].message.content
def test_web_search_truncation_reports_length(self):
"""The Conversations API returns no finish reason, so truncation is inferred
from the token budget. Force a real truncation (max_tokens=1 with an
overflowing prompt) and assert the response reports finish_reason='length'."""
if not os.getenv("MISTRAL_API_KEY"):
pytest.skip("MISTRAL_API_KEY not set")
response = litellm.completion(
model="mistral/mistral-medium-latest",
messages=[
{
"role": "user",
"content": "Write a very long, detailed essay about the entire history of France, at least 3000 words.",
}
],
web_search_options={},
max_tokens=1,
)
assert response.choices[0].finish_reason == "length"

View file

@ -249,6 +249,31 @@ def test_transform_response_chunked_content_and_citations():
assert mr.id == "conv_abc"
def _finish_reason_for(request_data: dict) -> str:
# CONVERSATIONS_RESPONSE reports completion_tokens == 8
mr = MistralConversationsConfig().transform_response(
model="mistral-medium-latest",
raw_response=_raw(CONVERSATIONS_RESPONSE),
model_response=ModelResponse(),
logging_obj=_logging_obj(),
request_data=request_data,
messages=[],
optional_params={},
litellm_params={},
encoding=None,
)
return mr.choices[0].finish_reason
def test_transform_response_finish_reason_length_when_completion_hits_max_tokens():
assert _finish_reason_for({"completion_args": {"max_tokens": 8}}) == "length"
def test_transform_response_finish_reason_stop_within_budget_or_unbounded():
assert _finish_reason_for({"completion_args": {"max_tokens": 100}}) == "stop"
assert _finish_reason_for({}) == "stop"
def test_transform_response_counts_web_searches_from_connectors():
mr = _transform(
{