mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(mistral): infer finish_reason=length for truncated Conversations responses
The Conversations API returns no finish/stop reason field (verified against the live API, including a forced max_tokens truncation), so transform_response hardcoded finish_reason="stop"; a client checking finish_reason == "length" to detect truncation got a wrong signal when the model stopped on max_tokens. With nothing on the wire to forward, infer it from the token budget: return "length" when completion_tokens reached the requested max_tokens, otherwise "stop"
This commit is contained in:
parent
ea340d6d2f
commit
30612b1d16
3 changed files with 61 additions and 1 deletions
|
|
@ -260,6 +260,18 @@ class MistralConversationsConfig(MistralConfig):
|
|||
setattr(usage_obj, "web_search_premium_requests", web_search_premium_requests)
|
||||
return usage_obj
|
||||
|
||||
@staticmethod
|
||||
def _finish_reason(request_data: dict, usage: Optional[MistralConversationUsage]) -> str:
|
||||
"""The Conversations API returns no finish/stop reason, so infer truncation
|
||||
from the token budget: ``length`` when the completion filled the requested
|
||||
``max_tokens``, otherwise ``stop``."""
|
||||
args = STR_OBJ_DICT.validate_python(request_data.get("completion_args") or {})
|
||||
max_tokens = args.get("max_tokens")
|
||||
completion_tokens = usage.completion_tokens if usage else None
|
||||
if isinstance(max_tokens, int) and isinstance(completion_tokens, int) and completion_tokens >= max_tokens:
|
||||
return "length"
|
||||
return "stop"
|
||||
|
||||
def transform_response(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -286,7 +298,9 @@ class MistralConversationsConfig(MistralConfig):
|
|||
content=text or None,
|
||||
annotations=annotations or None,
|
||||
)
|
||||
model_response.choices = [Choices(index=0, message=message, finish_reason="stop")]
|
||||
model_response.choices = [
|
||||
Choices(index=0, message=message, finish_reason=self._finish_reason(request_data, parsed.usage))
|
||||
]
|
||||
model_response.model = model
|
||||
if parsed.conversation_id:
|
||||
model_response.id = parsed.conversation_id
|
||||
|
|
|
|||
|
|
@ -89,3 +89,24 @@ class TestMistralCompletion(BaseLLMChatTest):
|
|||
|
||||
assert response is not None
|
||||
assert response.choices[0].message.content
|
||||
|
||||
def test_web_search_truncation_reports_length(self):
|
||||
"""The Conversations API returns no finish reason, so truncation is inferred
|
||||
from the token budget. Force a real truncation (max_tokens=1 with an
|
||||
overflowing prompt) and assert the response reports finish_reason='length'."""
|
||||
if not os.getenv("MISTRAL_API_KEY"):
|
||||
pytest.skip("MISTRAL_API_KEY not set")
|
||||
|
||||
response = litellm.completion(
|
||||
model="mistral/mistral-medium-latest",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Write a very long, detailed essay about the entire history of France, at least 3000 words.",
|
||||
}
|
||||
],
|
||||
web_search_options={},
|
||||
max_tokens=1,
|
||||
)
|
||||
|
||||
assert response.choices[0].finish_reason == "length"
|
||||
|
|
|
|||
|
|
@ -249,6 +249,31 @@ def test_transform_response_chunked_content_and_citations():
|
|||
assert mr.id == "conv_abc"
|
||||
|
||||
|
||||
def _finish_reason_for(request_data: dict) -> str:
|
||||
# CONVERSATIONS_RESPONSE reports completion_tokens == 8
|
||||
mr = MistralConversationsConfig().transform_response(
|
||||
model="mistral-medium-latest",
|
||||
raw_response=_raw(CONVERSATIONS_RESPONSE),
|
||||
model_response=ModelResponse(),
|
||||
logging_obj=_logging_obj(),
|
||||
request_data=request_data,
|
||||
messages=[],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
)
|
||||
return mr.choices[0].finish_reason
|
||||
|
||||
|
||||
def test_transform_response_finish_reason_length_when_completion_hits_max_tokens():
|
||||
assert _finish_reason_for({"completion_args": {"max_tokens": 8}}) == "length"
|
||||
|
||||
|
||||
def test_transform_response_finish_reason_stop_within_budget_or_unbounded():
|
||||
assert _finish_reason_for({"completion_args": {"max_tokens": 100}}) == "stop"
|
||||
assert _finish_reason_for({}) == "stop"
|
||||
|
||||
|
||||
def test_transform_response_counts_web_searches_from_connectors():
|
||||
mr = _transform(
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue