fix(vertex_ai): return a truncated choice when a Gemini candidate has no content

A content-less candidate (a thinking model that spends its whole token budget
on reasoning returns finishReason MAX_TOKENS with no content) was skipped,
leaving choices empty so callers hit IndexError on the first choice. Emit an
empty-content choice, mapping finishReason through the same path the mainline
uses; a missing finishReason is reported as length, never a false stop.
This commit is contained in:
codechrl 2026-08-14 09:53:05 +07:00
parent 09889e1986
commit cfcbe20674
2 changed files with 76 additions and 0 deletions

View file

@ -2219,6 +2219,29 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
for idx, candidate in enumerate(_candidates):
if "content" not in candidate:
# Content-less candidate (e.g. thinking model out of budget):
# emit an empty choice with a mapped truncation reason, not an
# empty choices list. Non-streaming only.
if isinstance(model_response, ModelResponse):
finish_reason = candidate.get("finishReason")
mapped_finish_reason = (
VertexGeminiConfig._check_finish_reason(None, finish_reason)
if finish_reason is not None
else "length"
)
empty_message: ChatCompletionResponseMessage = {
"role": "assistant",
"content": "",
}
model_response.choices.append(
litellm.Choices(
finish_reason=mapped_finish_reason,
index=candidate.get("index", idx),
message=empty_message,
logprobs=None,
enhancements=None,
)
)
continue
# Extract metadata using helper function

View file

@ -5553,3 +5553,56 @@ def test_accumulated_json_skips_non_dict_leading_value():
assert len(out) == 1
assert out[0].choices[0].delta.content == "a"
def test_vertex_ai_content_less_candidate_rescued_as_length():
"""A candidate with no "content" (e.g. a thinking model that spends its whole
token budget on reasoning -> finishReason=MAX_TOKENS, no content) must still
produce a choice, not be silently dropped. Dropping it leaves
model_response.choices empty and the caller hits IndexError on choices[0].
The rescued choice must carry empty content and finishReason mapped through
the SAME path the mainline uses (MAX_TOKENS -> "length"), never the raw
provider value.
"""
v = VertexGeminiConfig()
model_response = ModelResponse()
model_response.choices = []
v._process_candidates(
_candidates=[
{
"finishReason": "MAX_TOKENS",
"index": 0,
}
],
model_response=model_response,
standard_optional_params={},
)
assert len(model_response.choices) == 1
choice = model_response.choices[0]
# mapped, not the raw "MAX_TOKENS"
assert choice.finish_reason == "length"
assert choice.message.content == ""
assert choice.message.role == "assistant"
def test_vertex_ai_content_less_candidate_without_finish_reason_not_stop():
"""A content-less candidate with NO finishReason at all must not silently
become finish_reason="stop" (a false claim the model finished normally).
Budget-exhausted-with-no-marker is honestly reported as "length".
"""
v = VertexGeminiConfig()
model_response = ModelResponse()
model_response.choices = []
v._process_candidates(
_candidates=[{"index": 0}],
model_response=model_response,
standard_optional_params={},
)
assert len(model_response.choices) == 1
assert model_response.choices[0].finish_reason == "length"
assert model_response.choices[0].finish_reason != "stop"
assert model_response.choices[0].message.content == ""