mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-08 08:08:37 +00:00
fix(vertex_ai): return a truncated choice when a Gemini candidate has no content
A content-less candidate (a thinking model that spends its whole token budget on reasoning returns finishReason MAX_TOKENS with no content) was skipped, leaving choices empty so callers hit IndexError on the first choice. Emit an empty-content choice, mapping finishReason through the same path the mainline uses; a missing finishReason is reported as length, never a false stop.
This commit is contained in:
parent
09889e1986
commit
cfcbe20674
2 changed files with 76 additions and 0 deletions
|
|
@ -2219,6 +2219,29 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
for idx, candidate in enumerate(_candidates):
|
||||
if "content" not in candidate:
|
||||
# Content-less candidate (e.g. thinking model out of budget):
|
||||
# emit an empty choice with a mapped truncation reason, not an
|
||||
# empty choices list. Non-streaming only.
|
||||
if isinstance(model_response, ModelResponse):
|
||||
finish_reason = candidate.get("finishReason")
|
||||
mapped_finish_reason = (
|
||||
VertexGeminiConfig._check_finish_reason(None, finish_reason)
|
||||
if finish_reason is not None
|
||||
else "length"
|
||||
)
|
||||
empty_message: ChatCompletionResponseMessage = {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
}
|
||||
model_response.choices.append(
|
||||
litellm.Choices(
|
||||
finish_reason=mapped_finish_reason,
|
||||
index=candidate.get("index", idx),
|
||||
message=empty_message,
|
||||
logprobs=None,
|
||||
enhancements=None,
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
# Extract metadata using helper function
|
||||
|
|
|
|||
|
|
@ -5553,3 +5553,56 @@ def test_accumulated_json_skips_non_dict_leading_value():
|
|||
|
||||
assert len(out) == 1
|
||||
assert out[0].choices[0].delta.content == "a"
|
||||
|
||||
|
||||
def test_vertex_ai_content_less_candidate_rescued_as_length():
|
||||
"""A candidate with no "content" (e.g. a thinking model that spends its whole
|
||||
token budget on reasoning -> finishReason=MAX_TOKENS, no content) must still
|
||||
produce a choice, not be silently dropped. Dropping it leaves
|
||||
model_response.choices empty and the caller hits IndexError on choices[0].
|
||||
The rescued choice must carry empty content and finishReason mapped through
|
||||
the SAME path the mainline uses (MAX_TOKENS -> "length"), never the raw
|
||||
provider value.
|
||||
"""
|
||||
v = VertexGeminiConfig()
|
||||
model_response = ModelResponse()
|
||||
model_response.choices = []
|
||||
|
||||
v._process_candidates(
|
||||
_candidates=[
|
||||
{
|
||||
"finishReason": "MAX_TOKENS",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
model_response=model_response,
|
||||
standard_optional_params={},
|
||||
)
|
||||
|
||||
assert len(model_response.choices) == 1
|
||||
choice = model_response.choices[0]
|
||||
# mapped, not the raw "MAX_TOKENS"
|
||||
assert choice.finish_reason == "length"
|
||||
assert choice.message.content == ""
|
||||
assert choice.message.role == "assistant"
|
||||
|
||||
|
||||
def test_vertex_ai_content_less_candidate_without_finish_reason_not_stop():
|
||||
"""A content-less candidate with NO finishReason at all must not silently
|
||||
become finish_reason="stop" (a false claim the model finished normally).
|
||||
Budget-exhausted-with-no-marker is honestly reported as "length".
|
||||
"""
|
||||
v = VertexGeminiConfig()
|
||||
model_response = ModelResponse()
|
||||
model_response.choices = []
|
||||
|
||||
v._process_candidates(
|
||||
_candidates=[{"index": 0}],
|
||||
model_response=model_response,
|
||||
standard_optional_params={},
|
||||
)
|
||||
|
||||
assert len(model_response.choices) == 1
|
||||
assert model_response.choices[0].finish_reason == "length"
|
||||
assert model_response.choices[0].finish_reason != "stop"
|
||||
assert model_response.choices[0].message.content == ""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue